hivemq setup

.sops.yaml

@@ -20,7 +20,7 @@ keys:
 creation_rules:
   # Supabase compose stack → app-host only.
   - path_regex: nix/secrets/secrets\.yaml$
-    encrypted_regex: ^(supabase_|ingester_|postgres_admin_)
+    encrypted_regex: ^(supabase_|ingester_|postgres_admin_|hivemq_)
     key_groups:
       - age:
           - *dev_lars

README.md

@@ -61,6 +61,26 @@ There are two distinct phases. **Initial provisioning** turns a blank box into a
 NixOS host (`nixos-anywhere`, run once per machine). **Updates** push new
 closures to a host that already runs NixOS (`deploy-rs`, run on every change).
 
+### SSH key setup (do this first)
+
+Both phases authenticate over SSH with `~/.ssh/larsnolden`, which is
+passphrase-protected. Load it into an `ssh-agent` once so the deploy tools can
+reuse it without prompting:
+
+```fish
+eval (ssh-agent -c)        # bash/zsh: eval "$(ssh-agent -s)"
+ssh-add ~/.ssh/larsnolden  # enter the passphrase once
+ssh-add -l                 # confirm the key is loaded
+```
+
+This is **required** for `deploy-rs`, not just a convenience: with
+`magicRollback = true` (see `nix/deploy.nix`) activation opens two concurrent SSH
+connections — the activation command and a rollback waiter. Without an agent,
+both race to read the passphrase from the terminal, one loses, and the deploy
+fails with `Permission denied (publickey,keyboard-interactive)` even though
+manual SSH and the copy step work. The agent serves the key to every connection,
+so no prompt is needed.
+
 ### Initial provisioning (`nixos-anywhere`)
 
 `deploy-rs` only *updates* a machine that already runs NixOS — it copies a
@@ -109,6 +129,9 @@ Provision `db-host` first if you intend to deploy updates immediately afterward
 once, copied to each host, activated with auto-rollback. Order: `db-host` →
 `app-host` → `mqtt-ingest`.
 
+Running it by hand needs the key loaded into an `ssh-agent` first — see
+[SSH key setup](#ssh-key-setup-do-this-first) above.
+
 ```
 nix run github:serokell/deploy-rs -- .#db-host       # one host
 nix run github:serokell/deploy-rs -- .                # all hosts

ingester/default.nix

@@ -8,6 +8,10 @@ buildGoModule {
   # Set once `go mod tidy` has produced a real go.sum.
   vendorHash = null;
 
+  # The main package lives in cmd/gebos-ingester/, so `go install` names the
+  # binary after that directory — matching mainProgram and the systemd unit.
+  subPackages = [ "cmd/gebos-ingester" ];
+
   meta = {
     description = "MQTT → Postgres telemetry ingester for Gebos";
     mainProgram = "gebos-ingester";

nix/dev/process-compose.nix

@@ -68,7 +68,7 @@ in
     };
 
     ingester = {
-      command = "${pkgs.go}/bin/go run ./ingester";
+      command = "${pkgs.go}/bin/go run ./ingester/cmd/gebos-ingester";
       environment = ingesterEnv;
       depends_on = {
         "db".condition = "process_healthy";

nix/hosts/mqtt-ingest.nix

@@ -4,6 +4,8 @@
   networking.hostName = "mqtt-ingest";
 
   services.gebos.secrets.ingester = true;
+  # Broker authn/authz credentials (file-RBAC: ingester + admin MQTT users).
+  services.gebos.secrets.hivemq = true;
 
   services.gebos.hivemq = {
     enable = true;

nix/modules/gebos-hivemq.nix

@@ -2,31 +2,274 @@
 
 let
   cfg = config.services.gebos.hivemq;
+
+  # HiveMQ CE — fetched from the upstream GitHub release and unpacked into the
+  # Nix store. The store tree is read-only; the runtime data/log/extension
+  # folders are redirected to the writable state dir (see ExecStart below).
+  hivemq-ce = pkgs.stdenvNoCC.mkDerivation rec {
+    pname = "hivemq-ce";
+    version = "2026.5";
+
+    src = pkgs.fetchurl {
+      url = "https://github.com/hivemq/hivemq-community-edition/releases/download/${version}/hivemq-ce-${version}.zip";
+      hash = "sha256-/QBeU17KC6/7CF4RsVEohdBs4AeG+WFwHHZEmBb33sU=";
+    };
+
+    nativeBuildInputs = [ pkgs.unzip ];
+    dontConfigure = true;
+    dontBuild = true;
+
+    installPhase = ''
+      runHook preInstall
+      mkdir -p "$out"
+      cp -r . "$out/"
+      runHook postInstall
+    '';
+
+    meta = with lib; {
+      description = "HiveMQ Community Edition — open-source MQTT 3.x / 5 broker";
+      homepage = "https://github.com/hivemq/hivemq-community-edition";
+      license = licenses.asl20;
+      platforms = platforms.linux;
+    };
+  };
+
+  # HiveMQ file-RBAC extension — adds username/password authentication and
+  # topic-level authorization, replacing the insecure hivemq-allow-all that
+  # ships in the broker package. The zip unpacks to a single top-level
+  # `hivemq-file-rbac-extension/` folder which we keep verbatim in $out.
+  hivemq-file-rbac = pkgs.stdenvNoCC.mkDerivation rec {
+    pname = "hivemq-file-rbac-extension";
+    version = "4.6.15";
+
+    src = pkgs.fetchurl {
+      url = "https://github.com/hivemq/hivemq-file-rbac-extension/releases/download/${version}/hivemq-file-rbac-extension-${version}.zip";
+      hash = "sha256-R6PM+qAvEKMHdX3FTR0CfR8yarJrc2WeFAEu4eXH3Ps=";
+    };
+
+    nativeBuildInputs = [ pkgs.unzip ];
+    dontConfigure = true;
+    dontBuild = true;
+
+    # stdenv unpacks the single top-level archive dir and cds into it, so $PWD
+    # already holds the extension's files (jar, conf/, hivemq-extension.xml).
+    installPhase = ''
+      runHook preInstall
+      mkdir -p "$out/hivemq-file-rbac-extension"
+      cp -r . "$out/hivemq-file-rbac-extension/"
+      runHook postInstall
+    '';
+
+    meta = with lib; {
+      description = "HiveMQ file-based role-based access control extension";
+      homepage = "https://github.com/hivemq/hivemq-file-rbac-extension";
+      license = licenses.asl20;
+      platforms = platforms.linux;
+    };
+  };
+
+  stateDir = "/var/lib/gebos-hivemq";
+
+  # Non-secret RBAC extension config (reload interval + password format). The
+  # users/roles live in the secret credentials.xml, symlinked in at runtime.
+  rbacConfigXml = pkgs.writeText "hivemq-file-rbac-config.xml" ''
+    <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+    <extension-configuration>
+        <credentials-reload-interval>${toString cfg.rbac.reloadInterval}</credentials-reload-interval>
+        <password-type>${cfg.rbac.passwordType}</password-type>
+    </extension-configuration>
+  '';
+
+  # config.xml rendered from the module options. The local TCP listener is what
+  # the ingester (and, eventually, the Caddy TLS frontend) connects to.
+  configXml = pkgs.writeText "hivemq-config.xml" ''
+    <?xml version="1.0"?>
+    <hivemq xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:noNamespaceSchemaLocation="config.xsd">
+        <listeners>
+            <tcp-listener>
+                <port>${toString cfg.port}</port>
+                <bind-address>${cfg.bindAddress}</bind-address>
+            </tcp-listener>
+        </listeners>
+        <persistence>
+            <mode>${if cfg.persistentSessions then "file" else "in-memory"}</mode>
+        </persistence>
+        <anonymous-usage-statistics>
+            <enabled>false</enabled>
+        </anonymous-usage-statistics>
+    </hivemq>
+  '';
+
+  # Read-only conf folder: our generated config.xml plus the upstream
+  # logback.xml / config.xsd so HiveMQ logs and validates as shipped.
+  confDir = pkgs.runCommand "hivemq-conf" { } ''
+    mkdir -p "$out"
+    cp ${configXml} "$out/config.xml"
+    cp ${hivemq-ce}/conf/logback.xml "$out/logback.xml"
+    cp ${hivemq-ce}/conf/config.xsd "$out/config.xsd"
+  '';
+
+  # run.sh's JVM flags, reproduced so we can invoke java directly and point
+  # HiveMQ at the writable state dir via -Dhivemq.*.folder.
+  javaOpts = lib.concatStringsSep " " [
+    "-Djava.net.preferIPv4Stack=true"
+    "--add-opens java.base/java.lang=ALL-UNNAMED"
+    "--add-opens java.base/java.nio=ALL-UNNAMED"
+    "--add-opens java.base/sun.nio.ch=ALL-UNNAMED"
+    "--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED"
+    "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED"
+    "-Djava.security.egd=file:/dev/./urandom"
+    "-Duser.language=en -Duser.region=US"
+    "-XX:+CrashOnOutOfMemoryError"
+    "-XX:+HeapDumpOnOutOfMemoryError"
+    "-XX:HeapDumpPath=${stateDir}/log/heap-dump.hprof"
+    "-XX:ErrorFile=${stateDir}/log/hs_err_pid%p.log"
+  ];
 in
 {
   options.services.gebos.hivemq = {
     enable = lib.mkEnableOption "HiveMQ CE single-node broker";
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = hivemq-ce;
+      defaultText = lib.literalExpression "pkgs.hivemq-ce (vendored)";
+      description = "HiveMQ CE package to run.";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 1883;
+      description = "Plain MQTT TCP listener port (what the ingester connects to).";
+    };
+
+    bindAddress = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1";
+      description = ''
+        Address the MQTT listener binds to. Defaults to loopback: external
+        clients reach the broker through the Caddy TLS frontend, not directly.
+      '';
+    };
+
     persistentSessions = lib.mkOption {
       type = lib.types.bool;
       default = true;
+      description = ''
+        Whether client sessions and queued messages survive a restart
+        (file persistence) or are kept only in memory.
+      '';
     };
+
     tlsPort = lib.mkOption {
       type = lib.types.port;
       default = 8883;
+      description = ''
+        Externally advertised MQTTS port. TLS is currently terminated by the
+        Caddy frontend (see gebos-caddy), not by HiveMQ directly, so this value
+        is informational until a keystore is wired into the broker.
+      '';
+    };
+
+    rbac = {
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = hivemq-file-rbac;
+        defaultText = lib.literalExpression "pkgs.hivemq-file-rbac (vendored)";
+        description = "HiveMQ file-RBAC authn/authz extension package.";
+      };
+
+      passwordType = lib.mkOption {
+        type = lib.types.enum [ "HASHED" "PLAIN" ];
+        default = "PLAIN";
+        description = ''
+          Whether passwords in the credentials file are PBKDF2 HASHED or PLAIN
+          text. PLAIN is acceptable here because the credentials file is a
+          sops-encrypted secret at rest and rendered 0400/gebos-hivemq into a
+          tmpfs at runtime; switch to HASHED for defence-in-depth.
+        '';
+      };
+
+      reloadInterval = lib.mkOption {
+        type = lib.types.ints.positive;
+        default = 60;
+        description = "Seconds between credentials.xml reloads by the extension.";
+      };
+
+      credentialsFile = lib.mkOption {
+        type = lib.types.path;
+        default = config.services.gebos.secrets.hivemqCredentialsFile;
+        defaultText = lib.literalExpression "config.services.gebos.secrets.hivemqCredentialsFile";
+        description = ''
+          Path to the file-RBAC credentials.xml (users, roles, topic
+          permissions). Defaults to the sops-rendered secret; symlinked into
+          the extension's conf/ dir so it never lands in the writable state dir.
+        '';
+      };
     };
   };
 
   config = lib.mkIf cfg.enable {
-    # TODO: package HiveMQ CE (download tarball + JRE wrapper) and wire as a
+    users.users.gebos-hivemq = {
-    # systemd unit. Until then this module only declares options so dependent
+      isSystemUser = true;
-    # hosts can be evaluated.
+      group = "gebos-hivemq";
+      description = "Gebos HiveMQ CE service user";
+    };
+    users.groups.gebos-hivemq = { };
+
     systemd.services.gebos-hivemq = {
       description = "HiveMQ CE";
+      after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
+
+      # HiveMQ wants the extensions folder writable (it manages enable/disable
+      # markers there), so the extension lives under the state dir. We reconcile
+      # it on every start: (re)install the vendored file-RBAC extension when its
+      # store path changes, refresh its non-secret config, link the secret
+      # credentials from the sops tmpfs path, and remove the insecure allow-all
+      # extension that earlier deploys seeded from the broker package.
+      preStart = ''
+        set -eu
+        mkdir -p ${stateDir}/{data,log,extensions}
+
+        extDir=${stateDir}/extensions/hivemq-file-rbac-extension
+        marker=${stateDir}/extensions/.rbac-version
+
+        if [ "$(cat "$marker" 2>/dev/null || true)" != "${cfg.rbac.package}" ]; then
+          rm -rf "$extDir"
+          cp -r --no-preserve=mode ${cfg.rbac.package}/hivemq-file-rbac-extension "$extDir"
+          echo "${cfg.rbac.package}" > "$marker"
+        fi
+
+        cp --no-preserve=mode ${rbacConfigXml} "$extDir/conf/config.xml"
+        ln -sf ${cfg.rbac.credentialsFile} "$extDir/conf/credentials.xml"
+
+        rm -rf ${stateDir}/extensions/hivemq-allow-all-extension
+      '';
+
       serviceConfig = {
         Type = "simple";
-        ExecStart = "${pkgs.coreutils}/bin/true";  # TODO: real HiveMQ command
+        User = "gebos-hivemq";
+        Group = "gebos-hivemq";
+        StateDirectory = "gebos-hivemq";
+        WorkingDirectory = stateDir;
+        ExecStart = ''
+          ${pkgs.jre_headless}/bin/java \
+            -Dhivemq.home=${cfg.package} \
+            -Dhivemq.config.folder=${confDir} \
+            -Dhivemq.log.folder=${stateDir}/log \
+            -Dhivemq.data.folder=${stateDir}/data \
+            -Dhivemq.extensions.folder=${stateDir}/extensions \
+            ${javaOpts} \
+            -jar ${cfg.package}/bin/hivemq.jar
+        '';
         Restart = "on-failure";
+        RestartSec = "5s";
+        # HiveMQ exits 143 on SIGTERM shutdown — treat that as success.
+        SuccessExitStatus = "143";
+        KillSignal = "SIGTERM";
+        LimitNOFILE = 65536;
       };
     };
   };

nix/modules/gebos-secrets.nix

@@ -17,6 +17,45 @@ let
   ];
   ingesterKeys = [ "ingester_postgres_password" ];
   postgresAdminKeys = [ "postgres_admin_password" ];
+  hivemqKeys = [ "hivemq_ingester_password" "hivemq_admin_password" ];
+
+  # HiveMQ file-RBAC credentials.xml. Only the passwords are secret; the role/
+  # topic policy is structural. `ingester` may subscribe to the whole telemetry
+  # tree (t/<tenant>/d/<device>/<metric>); `admin` is an operational superuser.
+  hivemqCredentialsXml = ingesterPassword: adminPassword: ''
+    <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+    <file-rbac>
+        <users>
+            <user>
+                <name>ingester</name>
+                <password>${ingesterPassword}</password>
+                <roles><id>ingest</id></roles>
+            </user>
+            <user>
+                <name>admin</name>
+                <password>${adminPassword}</password>
+                <roles><id>superuser</id></roles>
+            </user>
+        </users>
+        <roles>
+            <role>
+                <id>ingest</id>
+                <permissions>
+                    <permission>
+                        <topic>t/#</topic>
+                        <activity>SUBSCRIBE</activity>
+                    </permission>
+                </permissions>
+            </role>
+            <role>
+                <id>superuser</id>
+                <permissions>
+                    <permission><topic>#</topic></permission>
+                </permissions>
+            </role>
+        </roles>
+    </file-rbac>
+  '';
 
   # Render an env file body from a list of (key, envVarName) pairs.
   envBody = pairs: lib.concatStringsSep "\n" (map
@@ -41,7 +80,8 @@ let
   enabledKeys =
        lib.optionals cfg.supabase      supabaseKeys
     ++ lib.optionals cfg.ingester      ingesterKeys
-    ++ lib.optionals cfg.postgresAdmin postgresAdminKeys;
+    ++ lib.optionals cfg.postgresAdmin postgresAdminKeys
+    ++ lib.optionals cfg.hivemq        hivemqKeys;
 in
 {
   options.services.gebos.secrets = {
@@ -55,6 +95,26 @@ in
     postgresAdmin = lib.mkEnableOption ''
       Postgres bootstrap admin password applied via a one-shot ALTER ROLE unit.'';
 
+    hivemq = lib.mkEnableOption ''
+      HiveMQ file-RBAC credentials.xml (ingester + admin MQTT users), rendered
+      0400/gebos-hivemq for the broker's file-RBAC extension.'';
+
+    hivemqCredentialsFile = lib.mkOption {
+      type = lib.types.path;
+      readOnly = true;
+      description = ''
+        Path of the rendered HiveMQ file-RBAC credentials.xml consumed by the
+        gebos-hivemq broker. Resolves to the sops-templated secret when sops
+        material is present and the hivemq toggle is on, else to a baked-in
+        dev-defaults file (so first-boot eval and local VMs work).
+      '';
+      default =
+        if hasSecrets && cfg.hivemq
+        then config.sops.templates."hivemq-credentials".path
+        else "${pkgs.writeText "hivemq-credentials-dev.xml"
+          (hivemqCredentialsXml "dev-ingester" "dev-admin")}";
+    };
+
     envFile = lib.mkOption {
       type = lib.types.path;
       readOnly = true;
@@ -90,6 +150,17 @@ in
         mode = "0400";
         owner = "root";
       };
+
+      # Rendered as a standalone file (not env) — the file-RBAC extension reads
+      # credentials.xml directly. Owned by the broker user so preStart can link
+      # it into the extension's conf/ dir.
+      templates."hivemq-credentials" = lib.mkIf cfg.hivemq {
+        content = hivemqCredentialsXml
+          config.sops.placeholder.hivemq_ingester_password
+          config.sops.placeholder.hivemq_admin_password;
+        mode = "0400";
+        owner = "gebos-hivemq";
+      };
     };
   };
 }

nix/secrets/secrets.yaml.example

@@ -27,3 +27,9 @@ ingester_postgres_password: "replace-me-openssl-rand-hex-32"
 
 # Postgres bootstrap (consumed on db-host by a one-shot ALTER ROLE unit)
 postgres_admin_password: "replace-me-openssl-rand-hex-32"
+
+# HiveMQ file-RBAC broker users (consumed on mqtt-ingest by gebos-hivemq).
+# PLAIN passwords (password-type defaults to PLAIN; the whole file is sops-
+# encrypted). The ingester will authenticate as `ingester`; `admin` is for ops.
+hivemq_ingester_password: "replace-me-openssl-rand-hex-32"
+hivemq_admin_password: "replace-me-openssl-rand-hex-32"