From bf8b25f374a36b5fa724cd0e90bac9ba876bc6ef Mon Sep 17 00:00:00 2001 From: Lars Nolden Date: Mon, 29 Jun 2026 17:23:02 +0200 Subject: [PATCH] make caddy a L4 proxy to terminate TLS for HiveMQ --- nix/hosts/mqtt-ingest.nix | 9 +++-- nix/modules/gebos-caddy.nix | 65 +++++++++++++++++++++++++++++------ nix/modules/gebos-secrets.nix | 46 +++++++++++++++++++++---- 3 files changed, 100 insertions(+), 20 deletions(-) diff --git a/nix/hosts/mqtt-ingest.nix b/nix/hosts/mqtt-ingest.nix index 8382bdf..4042644 100644 --- a/nix/hosts/mqtt-ingest.nix +++ b/nix/hosts/mqtt-ingest.nix @@ -21,13 +21,16 @@ # GEBOS_POSTGRES_PASSWORD sourced from /run/secrets/gebos-env (sops-nix). }; - # Caddy fronts TLS for `ingest.gebos.online` and forwards to HiveMQ. + # Caddy (layer4) terminates TLS on :8883 and forwards cleartext MQTT to + # HiveMQ's loopback listener on :1883. tcpProxy is the upstream backend. services.gebos.caddy = { enable = true; sites = { - "ingest.gebos.online" = { tcpProxy = "127.0.0.1:8883"; }; + "ingest.gebos.online" = { tcpProxy = "127.0.0.1:1883"; }; }; }; - networking.firewall.allowedTCPPorts = [ 8883 ]; + # 8883: MQTTS for senders. 80/443: ACME challenge so Caddy can obtain/renew + # the ingest.gebos.online cert that the layer4 TLS handler serves. + networking.firewall.allowedTCPPorts = [ 80 443 8883 ]; } diff --git a/nix/modules/gebos-caddy.nix b/nix/modules/gebos-caddy.nix index 36db762..e8ed7a7 100644 --- a/nix/modules/gebos-caddy.nix +++ b/nix/modules/gebos-caddy.nix @@ -3,7 +3,24 @@ let cfg = config.services.gebos.caddy; - siteBlock = host: site: + tcpProxySites = lib.filterAttrs (_: s: s ? tcpProxy) cfg.sites; + httpSites = lib.filterAttrs (_: s: !(s ? tcpProxy)) cfg.sites; + needsLayer4 = tcpProxySites != { }; + + # External MQTTS port the senders connect to (mqtt_port = 8883, SSL). + tlsListenPort = 8883; + + # Caddy built with the layer4 module (caddy-l4) — needed to TLS-terminate raw + # MQTT and proxy it to HiveMQ, which Caddy's HTTP-only core can't do. Only + # built when a host actually has a tcpProxy site; app-host's static/upstream + # sites run on stock Caddy so they don't pay for the custom build. + caddyWithL4 = pkgs.caddy.withPlugins { + plugins = [ "github.com/mholt/caddy-l4@v0.1.1" ]; + hash = "sha256-O6GuC2q1mA/Fa0utb2Yg7ZE73iq13oVYhJI1IVyOvog="; + }; + + # HTTP-style sites: static file server or reverse proxy. + httpSiteBlock = host: site: if site ? staticRoot then '' ${host} { root * ${site.staticRoot} @@ -18,14 +35,38 @@ let encode zstd gzip } '' - else if site ? tcpProxy then '' - ${host}:8883 { - # TLS termination for MQTT — Caddy's `layer4` app would be ideal here; - # for now, document that the upstream HiveMQ port is ${site.tcpProxy}. - # TODO: switch to the caddy-l4 module or terminate TLS in HiveMQ directly. - } - '' else throw "site `${host}` needs one of: staticRoot, upstream, tcpProxy"; + + # A tcpProxy site needs TWO Caddyfile pieces: + # 1. a layer4 listener (in the global options block) that terminates TLS on + # :8883 and proxies cleartext to the upstream (HiveMQ on 127.0.0.1:1883); + # 2. a normal HTTPS site block for the host, purely so Caddy's automatic + # HTTPS OBTAINS the ACME cert that the layer4 `tls` handler then serves by + # SNI — caddy-l4's `tls` handler only terminates, it never provisions + # certs. This is why ports 80/443 must stay open for the ACME challenge. + layer4Listener = _host: site: '' + :${toString tlsListenPort} { + route { + tls + proxy ${site.tcpProxy} + } + } +''; + + certSiteBlock = host: _site: '' + ${host} { + respond "gebos MQTT ingest — connect MQTTS on :${toString tlsListenPort}" 200 + } + ''; + + # Caddyfile global options block must come first and exist once; we fold every + # layer4 listener into a single `layer4 { ... }` inside it. + globalOptions = lib.optionalString needsLayer4 '' + { + layer4 { + ${lib.concatStrings (lib.mapAttrsToList layer4Listener tcpProxySites)} } + } + ''; in { options.services.gebos.caddy = { @@ -39,9 +80,13 @@ in config = lib.mkIf cfg.enable { services.caddy = { enable = true; + package = if needsLayer4 then caddyWithL4 else pkgs.caddy; email = "ops@gebos.online"; # TODO: confirm ACME contact - extraConfig = lib.concatStringsSep "\n" - (lib.mapAttrsToList siteBlock cfg.sites); + extraConfig = lib.concatStringsSep "\n" ( + [ globalOptions ] + ++ lib.mapAttrsToList httpSiteBlock httpSites + ++ lib.mapAttrsToList certSiteBlock tcpProxySites + ); }; }; } diff --git a/nix/modules/gebos-secrets.nix b/nix/modules/gebos-secrets.nix index 753587e..6afa806 100644 --- a/nix/modules/gebos-secrets.nix +++ b/nix/modules/gebos-secrets.nix @@ -17,12 +17,30 @@ let ]; ingesterKeys = [ "ingester_postgres_password" ]; postgresAdminKeys = [ "postgres_admin_password" ]; - hivemqKeys = [ "hivemq_ingester_password" "hivemq_admin_password" ]; + hivemqKeys = [ "hivemq_ingester_password" "hivemq_admin_password" "hivemq_device_password" ]; # HiveMQ file-RBAC credentials.xml. Only the passwords are secret; the role/ - # topic policy is structural. `ingester` may subscribe to the whole telemetry - # tree (t//d//); `admin` is an operational superuser. - hivemqCredentialsXml = ingesterPassword: adminPassword: '' + # topic policy is structural. Devices publish under `acrios//` + # (set by the sender firmware's mqtt_topic_base), so that is the namespace the + # ingester consumes and the device role is scoped to. + # + # Roles: + # ingest — the Go ingester; SUBSCRIBE the whole device tree. + # device — physical senders; full access under acrios/ (publish telemetry + # + the LWT/status and any downlink topics the firmware uses). + # superuser — operational break-glass. + # + # Option A (current): one shared `device` user (`bender`) for the whole fleet. + # Tenant isolation is enforced downstream by the ingester's IMSI→tenant + # registry lookup, NOT at the broker — any device could publish under any IMSI. + # + # TODO(Option B / per-device auth): give each sender its own RBAC user with + # username == IMSI and scope its role to `acrios/${{username}}/#`, so the + # broker enforces that a device can only publish under its own IMSI (real + # anti-spoofing + per-device revocation). Requires setting mqtt_user = + # on each device and a per-device password provisioning flow (generate → sops + # → push to device). file-RBAC hot-reloads, so added users need no restart. + hivemqCredentialsXml = ingesterPassword: adminPassword: devicePassword: '' @@ -36,17 +54,30 @@ let ${adminPassword} superuser + + bender + ${devicePassword} + device + ingest - t/# + acrios/# SUBSCRIBE + + device + + + acrios/# + + superuser @@ -112,7 +143,7 @@ in if hasSecrets && cfg.hivemq then config.sops.templates."hivemq-credentials".path else "${pkgs.writeText "hivemq-credentials-dev.xml" - (hivemqCredentialsXml "dev-ingester" "dev-admin")}"; + (hivemqCredentialsXml "dev-ingester" "dev-admin" "dev-device")}"; }; envFile = lib.mkOption { @@ -157,7 +188,8 @@ in templates."hivemq-credentials" = lib.mkIf cfg.hivemq { content = hivemqCredentialsXml config.sops.placeholder.hivemq_ingester_password - config.sops.placeholder.hivemq_admin_password; + config.sops.placeholder.hivemq_admin_password + config.sops.placeholder.hivemq_device_password; mode = "0400"; owner = "gebos-hivemq"; };