Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Lars Nolden
|
58155dc737 | ||
|
Lars Nolden
|
08d9fc594e | ||
|
Lars Nolden
|
5e2454f622 | ||
|
Lars Nolden
|
16fd12173f | ||
|
Lars Nolden
|
e1b3017fb7 | ||
|
Lars Nolden
|
d5a0be9e40 | ||
|
Lars Nolden
|
649dd737c7 | ||
|
Lars Nolden
|
bf8b25f374 | ||
|
Lars Nolden
|
70c2ada986 | ||
|
Lars Nolden
|
4f9c6086bd | ||
|
Lars Nolden
|
c526cf671f | ||
|
Lars Nolden
|
30b325393a | ||
|
Lars Nolden
|
7579d4dcf5 | ||
|
Lars Nolden
|
9a6a901dfe | ||
|
Lars Nolden
|
b10d22bd57 | ||
|
Lars Nolden
|
19027e3b48 | ||
|
Lars Nolden
|
08ae2764e0 | ||
 lars
|
1cf710fe5b |
@@ -6,7 +6,7 @@ on:
|
||||
|
||||
jobs:
|
||||
flake-check:
|
||||
runs-on: nix
|
||||
runs-on: nixos
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: nix flake check
|
||||
|
||||
@@ -5,15 +5,22 @@ on:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: nix
|
||||
runs-on: nixos
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup deploy SSH key
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
install -m 600 /dev/stdin ~/.ssh/id_ed25519 <<< "${{ secrets.DEPLOY_SSH_KEY }}"
|
||||
ssh-keyscan -H db-host.gebos.internal app-host.gebos.internal ingest.ge-bos.de >> ~/.ssh/known_hosts 2>/dev/null || true
|
||||
# Dedicated key file so we never overwrite the runner user's own key.
|
||||
install -m 600 /dev/stdin ~/.ssh/gebos_deploy <<< "${{ secrets.DEPLOY_SSH_KEY }}"
|
||||
ssh-keyscan -H db.gebos.online app.gebos.online ingest.gebos.online >> ~/.ssh/known_hosts 2>/dev/null || true
|
||||
# The gitea-runner is a systemd DynamicUser whose passwd home is `/`,
|
||||
# so ssh expands `~` to `/.ssh/...` rather than $HOME. deploy-rs passes
|
||||
# ssh opts with no shell, so it can't expand $HOME itself. Build the
|
||||
# absolute paths here (bash expands $HOME) and hand them to deploy-rs
|
||||
# via --ssh-opts; they append to nix/deploy.nix's path-free sshOpts.
|
||||
echo "DEPLOY_SSH_OPTS=-i $HOME/.ssh/gebos_deploy -o UserKnownHostsFile=$HOME/.ssh/known_hosts" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build all closures
|
||||
run: nix flake check --no-build
|
||||
@@ -22,10 +29,10 @@ jobs:
|
||||
# so it must land first. App-host's Supabase compose stack expects the
|
||||
# schemas to exist. mqtt-ingest only needs the network paths to db-host.
|
||||
- name: Deploy db-host
|
||||
run: nix run github:serokell/deploy-rs -- .#db-host --skip-checks
|
||||
run: nix run github:serokell/deploy-rs -- --ssh-opts "$DEPLOY_SSH_OPTS" .#db-host --skip-checks
|
||||
|
||||
- name: Deploy app-host
|
||||
run: nix run github:serokell/deploy-rs -- .#app-host --skip-checks
|
||||
run: nix run github:serokell/deploy-rs -- --ssh-opts "$DEPLOY_SSH_OPTS" .#app-host --skip-checks
|
||||
|
||||
- name: Deploy mqtt-ingest
|
||||
run: nix run github:serokell/deploy-rs -- .#mqtt-ingest --skip-checks
|
||||
run: nix run github:serokell/deploy-rs -- --ssh-opts "$DEPLOY_SSH_OPTS" .#mqtt-ingest --skip-checks
|
||||
|
||||
@@ -14,3 +14,4 @@ secrets.env
|
||||
# age private keys — never commit (public keys live in .sops.yaml)
|
||||
*.age
|
||||
keys.txt
|
||||
data
|
||||
+5
-5
@@ -10,17 +10,17 @@
|
||||
|
||||
keys:
|
||||
# ---- developers (laptops, hardware keys) ----
|
||||
- &dev_lars age1TODO_lars_personal_age_pubkey_replace_me_before_first_real_secret
|
||||
- &dev_lars age1cx8ul285kjkzmnhw6skdstnzrxnnme4xkflknzn7yhv52fgxqevqkd66cn
|
||||
|
||||
# ---- hosts (derived from each host's ssh_host_ed25519_key.pub via ssh-to-age) ----
|
||||
- &host_db age1TODO_db_host_age_pubkey_replace_me
|
||||
- &host_app age1TODO_app_host_age_pubkey_replace_me
|
||||
- &host_ingest age1TODO_mqtt_ingest_host_age_pubkey_replace_me
|
||||
- &host_db age1wpl3vz60tlt880p5fmfedr8c59dm4kf0czsacv0w5my706twuf5q9p0x43
|
||||
- &host_app age1g57pznep69nlyv3sltz6k0sml47v68m03gh68jkqwcq4jdx68vtsvnefq0
|
||||
- &host_ingest age190gu75rf3ra89mhk27xe3tv87tad087altqhugjlhkerqwe2jfqsnu738d
|
||||
|
||||
creation_rules:
|
||||
# Supabase compose stack → app-host only.
|
||||
- path_regex: nix/secrets/secrets\.yaml$
|
||||
encrypted_regex: ^(supabase_|ingester_|postgres_admin_)
|
||||
encrypted_regex: ^(supabase_|ingester_|postgres_admin_|emqx_)
|
||||
key_groups:
|
||||
- age:
|
||||
- *dev_lars
|
||||
|
||||
@@ -9,15 +9,96 @@ and CI/CD pipelines for the project. Everything is one `flake.nix`.
|
||||
|
||||
Three hosts (see issue #21 for the full design discussion):
|
||||
|
||||
| Host | Role | Public hostname |
|
||||
| ------------- | ----------------------------------------- | ---------------------- |
|
||||
| `mqtt-ingest` | HiveMQ CE + Go MQTT→Postgres ingester | `ingest.ge-bos.de` |
|
||||
| `db-host` | Postgres 17 + TimescaleDB (telemetry+auth)| internal only |
|
||||
| `app-host` | Caddy + Kong + Supabase (compose) + SPA | `app.ge-bos.de`, `api.ge-bos.de` |
|
||||
| Host | Role | Public hostname | Private IP |
|
||||
| ------------- | ----------------------------------------- | ---------------------- | ---------- |
|
||||
| `mqtt-ingest` | EMQX broker + Go MQTT→Postgres ingester | `ingest.gebos.online` | `10.0.0.4` |
|
||||
| `db-host` | Postgres 17 + TimescaleDB (telemetry+auth)| `db.gebos.online` (SSH only) | `10.0.0.2` |
|
||||
| `app-host` | Caddy + Kong + Supabase (compose) + SPA | `app.gebos.online`, `api.gebos.online` | `10.0.0.3` |
|
||||
|
||||
Public REST surface is PostgREST + SQL `/rpc/` functions, fronted by Kong, TLS-terminated by Caddy.
|
||||
Supabase Studio is bound to `127.0.0.1` on `app-host` — reach it with `ssh -L 3000:127.0.0.1:3000 app-host`.
|
||||
|
||||
### Networking
|
||||
|
||||
There is no private DNS. Each host has a public hostname (used for `deploy-rs`
|
||||
SSH access from the Gitea runner, which is **not** on the private network, plus
|
||||
TLS ingress where applicable), and a static `10.0.0.0/8` IP used for all
|
||||
host-to-host traffic:
|
||||
|
||||
- `db.gebos.online` is for SSH/deploy only — Postgres is never exposed publicly.
|
||||
- `app-host` and `mqtt-ingest` reach Postgres at `10.0.0.2:5432` over the private
|
||||
network.
|
||||
- `db-host` only accepts Postgres (port 5432) from `10.0.0.0/8` (firewall rule
|
||||
in `nix/hosts/db-host.nix`).
|
||||
|
||||
## MQTT ingest path
|
||||
|
||||
How a sender device's telemetry reaches Postgres, and the reasoning behind each
|
||||
choice. The device firmware is the fixed end of this contract, so the rest of
|
||||
the stack is built to match it rather than the other way around.
|
||||
|
||||
```
|
||||
device ──MQTTS:8883──▶ EMQX (native TLS) ──MQTT:1883 loopback──▶ ingester ──▶ Postgres
|
||||
acrios/<IMSI>/<metric> authn + ACL IMSI→tenant
|
||||
```
|
||||
|
||||
### Broker: EMQX in the official container, deny-by-default auth
|
||||
|
||||
EMQX runs as the official `emqx/emqx` image via `oci-containers` with host
|
||||
networking (`nix/modules/gebos-emqx.nix`). Authentication uses the built-in
|
||||
database, seeded from a `users.csv` rendered by sops-nix
|
||||
(`services.gebos.secrets.emqx`) and bootstrapped on first start; the bootstrap
|
||||
only inserts users that don't exist yet, so a password rotation means deleting
|
||||
the user (dashboard or `emqx ctl`) and restarting to re-import. Authorization
|
||||
is a static `acl.conf` with `no_match = deny`. Passwords are plaintext in the
|
||||
CSV because the file is already an encrypted secret at rest and `0400` at
|
||||
runtime.
|
||||
|
||||
There are three broker users: `ingester` (subscribes the device tree), `admin`
|
||||
(break-glass, `is_superuser` — bypasses the ACL), and `bender` (the shared
|
||||
device account, below).
|
||||
|
||||
### Topic scheme: the device's `acrios/<IMSI>/…` is the source of truth
|
||||
|
||||
Senders publish to `acrios/<IMSI>/<metric>` — `mqtt_topic_base` plus the SIM's
|
||||
IMSI plus the metric name. We adopted that namespace verbatim rather than
|
||||
reshaping it into the `t/<tenant>/d/<device>/…` form the schema originally
|
||||
imagined, because the firmware can't emit our `tenant_id`/`device_id` **UUIDs**
|
||||
— it only knows its IMSI. So the IMSI is the natural device key, and the
|
||||
ingester (subscribed to `acrios/#`) will resolve `IMSI → (tenant_id,
|
||||
device_id)` via a **device registry** before inserting into `public.telemetry`.
|
||||
That registry table and the ingester's topic parsing are still TODO — the
|
||||
ingester is currently a stub.
|
||||
|
||||
### Device auth: shared user now (Option A), per-device later (Option B)
|
||||
|
||||
The sender logs in with a single shared account (`bender`), authorized to
|
||||
publish/subscribe under `acrios/#`. Tenant isolation is therefore enforced
|
||||
**downstream** by the ingester's IMSI registry, not at the broker — any device
|
||||
could publish under any IMSI. That's an accepted trade-off for a small trusted
|
||||
fleet, and it gets data flowing without per-device provisioning.
|
||||
|
||||
The production answer (**Option B**, a TODO in `nix/modules/gebos-emqx.nix`)
|
||||
is one broker user per device with `username == IMSI`, scoped to
|
||||
`acrios/${{username}}/#` so the broker itself prevents a device from spoofing
|
||||
another's IMSI. It's not wired up because it needs a firmware change
|
||||
(`mqtt_user = <IMSI>`) and a per-device password provisioning flow. Note the
|
||||
firmware's client-id (`acrcv-<IMSI>`) carries a prefix the topic doesn't, so
|
||||
per-device scoping must key on username, not `${{clientid}}`.
|
||||
|
||||
### TLS: EMQX terminates natively, certs via ACME HTTP-01
|
||||
|
||||
EMQX terminates TLS itself on `:8883`; the plain MQTT listener stays on
|
||||
loopback for the co-located ingester. Certificates come from `security.acme`
|
||||
using the **HTTP-01** challenge: lego's built-in standalone server answers on
|
||||
`:80` (nothing else listens there — no Caddy on this host), so no DNS API
|
||||
secrets are needed. `mqtt-ingest` opens **80 + 8883** only. The cert's
|
||||
`postRun` hook installs `fullchain.pem`/`privkey.pem` at fixed paths in the
|
||||
broker state dir, and EMQX re-reads the PEMs from disk (~every 120 s), so
|
||||
renewals hot-reload without a restart or dropped connections. The key is RSA
|
||||
(`keyType = "rsa2048"`) because embedded sender TLS stacks often can't do
|
||||
ECDSA. Caddy remains only on app-host (stock build, HTTP sites).
|
||||
|
||||
## Repo layout
|
||||
|
||||
```
|
||||
@@ -44,10 +125,81 @@ Supabase compose, Kong, Caddy, ingester, Vite dev server). NixOS required.
|
||||
|
||||
## Deployment
|
||||
|
||||
There are two distinct phases. **Initial provisioning** turns a blank box into a
|
||||
NixOS host (`nixos-anywhere`, run once per machine). **Updates** push new
|
||||
closures to a host that already runs NixOS (`deploy-rs`, run on every change).
|
||||
|
||||
### SSH key setup (do this first)
|
||||
|
||||
Both phases authenticate over SSH with `~/.ssh/larsnolden`, which is
|
||||
passphrase-protected. Load it into an `ssh-agent` once so the deploy tools can
|
||||
reuse it without prompting:
|
||||
|
||||
```fish
|
||||
eval (ssh-agent -c) # bash/zsh: eval "$(ssh-agent -s)"
|
||||
ssh-add ~/.ssh/larsnolden # enter the passphrase once
|
||||
ssh-add -l # confirm the key is loaded
|
||||
```
|
||||
|
||||
This is **required** for `deploy-rs`, not just a convenience: with
|
||||
`magicRollback = true` (see `nix/deploy.nix`) activation opens two concurrent SSH
|
||||
connections — the activation command and a rollback waiter. Without an agent,
|
||||
both race to read the passphrase from the terminal, one loses, and the deploy
|
||||
fails with `Permission denied (publickey,keyboard-interactive)` even though
|
||||
manual SSH and the copy step work. The agent serves the key to every connection,
|
||||
so no prompt is needed.
|
||||
|
||||
### Initial provisioning (`nixos-anywhere`)
|
||||
|
||||
`deploy-rs` only *updates* a machine that already runs NixOS — it copies a
|
||||
prebuilt closure and activates it. A fresh box (e.g. a stock Debian image with
|
||||
only a `root` user) has no Nix store and no NixOS generation to switch to, so
|
||||
`deploy-rs` fails with `nix-store: command not found`. Use
|
||||
[`nixos-anywhere`](https://github.com/nix-community/nixos-anywhere) to install
|
||||
NixOS over SSH first; after that, `deploy-rs` takes over for all subsequent
|
||||
deploys.
|
||||
|
||||
`nixos-anywhere` SSHes in as `root`, kexecs into an in-memory NixOS installer,
|
||||
partitions and formats the disk per the host's [`disko`](https://github.com/nix-community/disko)
|
||||
config, installs `nixosConfigurations.<host>`, and reboots into NixOS. **This
|
||||
wipes the target disk.**
|
||||
|
||||
Prerequisites, per host, before running it:
|
||||
|
||||
1. **A real disk layout.** Hosts currently import the fictional
|
||||
`nix/hosts/placeholder-hardware.nix` (it only exists so `nix flake check`
|
||||
evaluates). Replace that import with a `disko` config describing the actual
|
||||
disk device (`/dev/sda` vs `/dev/vda`/nvme) and firmware (UEFI vs legacy
|
||||
BIOS). `disko` replaces the hand-generated `hardware-configuration.nix`.
|
||||
2. **Root SSH access** to the box. The `deploy` user and its authorized keys are
|
||||
created by `nix/hosts/common.nix` during the install, so deploy-rs access
|
||||
works automatically once NixOS is up.
|
||||
3. **Host secrets key** present so sops-nix can decrypt at first boot — see
|
||||
[`nix/secrets/README.md`](nix/secrets/README.md). Otherwise services that
|
||||
read `/run/secrets/*` (e.g. the ingester) fail to start after reboot.
|
||||
|
||||
Then, from the repo root:
|
||||
|
||||
```
|
||||
# installs NixOS onto the target, wiping its disk
|
||||
nix run github:nix-community/nixos-anywhere -- \
|
||||
--flake .#mqtt-ingest root@ingest.gebos.online
|
||||
```
|
||||
|
||||
Repeat with `.#db-host root@db.gebos.online` and `.#app-host root@app.gebos.online`.
|
||||
Provision `db-host` first if you intend to deploy updates immediately afterward
|
||||
(see the ordering note below). Once a host has rebooted into NixOS, never run
|
||||
`nixos-anywhere` against it again — use `deploy-rs`.
|
||||
|
||||
### Updates (`deploy-rs`)
|
||||
|
||||
`deploy-rs` from a Gitea Actions runner on push to `main`. Closures are built
|
||||
once, copied to each host, activated with auto-rollback. Order: `db-host` →
|
||||
`app-host` → `mqtt-ingest`.
|
||||
|
||||
Running it by hand needs the key loaded into an `ssh-agent` first — see
|
||||
[SSH key setup](#ssh-key-setup-do-this-first) above.
|
||||
|
||||
```
|
||||
nix run github:serokell/deploy-rs -- .#db-host # one host
|
||||
nix run github:serokell/deploy-rs -- . # all hosts
|
||||
|
||||
Generated
+47
-5
@@ -20,6 +20,26 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"disko": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781152676,
|
||||
"narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
@@ -87,16 +107,16 @@
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1779560665,
|
||||
"narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=",
|
||||
"lastModified": 1782233679,
|
||||
"narHash": "sha256-QyuGP5+QOtmXpy4i2X4DhBVBaySBdDKQEhqKcphcp34=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786",
|
||||
"rev": "667d5cf1c59585031d743c78b394b0a647537c35",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"ref": "nixos-26.05",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
@@ -119,10 +139,12 @@
|
||||
"root": {
|
||||
"inputs": {
|
||||
"deploy-rs": "deploy-rs",
|
||||
"disko": "disko",
|
||||
"flake-parts": "flake-parts",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
"process-compose-flake": "process-compose-flake",
|
||||
"services-flake": "services-flake"
|
||||
"services-flake": "services-flake",
|
||||
"sops-nix": "sops-nix"
|
||||
}
|
||||
},
|
||||
"services-flake": {
|
||||
@@ -140,6 +162,26 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"sops-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782165805,
|
||||
"narHash": "sha256-478kKQBvK6SYTOdN2h9jhKJv94nbXRbFMfuL1WshErg=",
|
||||
"owner": "Mic92",
|
||||
"repo": "sops-nix",
|
||||
"rev": "56b24064fdcaedca53553b1a6d607fd23b613a24",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "Mic92",
|
||||
"repo": "sops-nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
|
||||
@@ -2,11 +2,15 @@
|
||||
description = "Gebos — IoT telemetry stack on bare-metal NixOS";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05";
|
||||
flake-parts.url = "github:hercules-ci/flake-parts";
|
||||
deploy-rs.url = "github:serokell/deploy-rs";
|
||||
process-compose-flake.url = "github:Platonic-Systems/process-compose-flake";
|
||||
services-flake.url = "github:juspay/services-flake";
|
||||
disko = {
|
||||
url = "github:nix-community/disko";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
sops-nix = {
|
||||
url = "github:Mic92/sops-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
{ buildNpmPackage, lib }:
|
||||
{ buildNpmPackage }:
|
||||
|
||||
buildNpmPackage {
|
||||
pname = "gebos-frontend";
|
||||
version = "0.0.0";
|
||||
src = ./.;
|
||||
|
||||
# Filled in once package-lock.json exists.
|
||||
npmDepsHash = lib.fakeHash;
|
||||
npmDepsHash = "sha256-GeXgNbf94QRLNaZP8ma2vx19Vzm8jDWPeOTvE9exwm0=";
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
@@ -11,7 +11,7 @@ const DEV_SUPABASE_ANON_KEY =
|
||||
|
||||
const SUPABASE_URL =
|
||||
import.meta.env.VITE_SUPABASE_URL ??
|
||||
(import.meta.env.DEV ? DEV_SUPABASE_URL : "https://api.ge-bos.de");
|
||||
(import.meta.env.DEV ? DEV_SUPABASE_URL : "https://api.gebos.online");
|
||||
|
||||
const SUPABASE_ANON_KEY =
|
||||
import.meta.env.VITE_SUPABASE_ANON_KEY ??
|
||||
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
/// <reference types="vite/client" />
|
||||
@@ -8,6 +8,10 @@ buildGoModule {
|
||||
# Set once `go mod tidy` has produced a real go.sum.
|
||||
vendorHash = null;
|
||||
|
||||
# The main package lives in cmd/gebos-ingester/, so `go install` names the
|
||||
# binary after that directory — matching mainProgram and the systemd unit.
|
||||
subPackages = [ "cmd/gebos-ingester" ];
|
||||
|
||||
meta = {
|
||||
description = "MQTT → Postgres telemetry ingester for Gebos";
|
||||
mainProgram = "gebos-ingester";
|
||||
|
||||
+18
-4
@@ -12,14 +12,28 @@ let
|
||||
};
|
||||
in
|
||||
{
|
||||
sshOpts = [ "-o" "StrictHostKeyChecking=accept-new" ];
|
||||
# Path-free ssh options only. The key path and known_hosts are runner-specific
|
||||
# and injected by the deploy.yml via deploy-rs `--ssh-opts` (which appends to
|
||||
# this list for both `nix copy` and the activation SSH). They must be absolute:
|
||||
# the gitea-runner is a systemd DynamicUser whose passwd home is `/`, so ssh
|
||||
# expands `~` to `/.ssh/...` (NOT $HOME) and a tilde here finds no key. The
|
||||
# workflow builds the absolute paths from $HOME, where bash expands correctly.
|
||||
# `-F none` ignores the runner's ~/.ssh/config; IdentitiesOnly pins the key.
|
||||
sshOpts = [
|
||||
"-F" "none"
|
||||
"-o" "IdentitiesOnly=yes"
|
||||
"-o" "StrictHostKeyChecking=accept-new"
|
||||
];
|
||||
autoRollback = true;
|
||||
magicRollback = true;
|
||||
|
||||
nodes = {
|
||||
# Order matters: the Gitea Actions workflow invokes them in this sequence.
|
||||
db-host = mkNode "db-host" "db-host.gebos.internal"; # TODO: real address
|
||||
app-host = mkNode "app-host" "app-host.gebos.internal"; # TODO: real address
|
||||
mqtt-ingest = mkNode "mqtt-ingest" "ingest.ge-bos.de";
|
||||
# SSH/deploy targets use public hostnames because the Gitea runner is not on
|
||||
# the private network. Host-to-host traffic still uses the 10.0.0.0/8 IPs.
|
||||
# See the host address table in README.md.
|
||||
db-host = mkNode "db-host" "db.gebos.online";
|
||||
app-host = mkNode "app-host" "app.gebos.online";
|
||||
mqtt-ingest = mkNode "mqtt-ingest" "ingest.gebos.online";
|
||||
};
|
||||
}
|
||||
|
||||
@@ -38,6 +38,13 @@ let
|
||||
VITE_SUPABASE_URL = dev.supabaseUrl;
|
||||
VITE_SUPABASE_ANON_KEY = dev.anonKey;
|
||||
};
|
||||
|
||||
# services-flake ships no MQTT broker, so run mosquitto as a plain
|
||||
# process-compose process. Anonymous access on the dev listener only.
|
||||
mosquittoConf = pkgs.writeText "mosquitto-dev.conf" ''
|
||||
listener ${toString dev.mqttPort} ${dev.mqttHost}
|
||||
allow_anonymous true
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [ services-flake.processComposeModules.default ];
|
||||
@@ -48,16 +55,20 @@ in
|
||||
port = dev.pgPort;
|
||||
initialDatabases = [{ name = dev.pgDB; }];
|
||||
};
|
||||
|
||||
mosquitto."broker" = {
|
||||
enable = true;
|
||||
port = dev.mqttPort;
|
||||
};
|
||||
};
|
||||
|
||||
settings.processes = {
|
||||
broker = {
|
||||
command = "${pkgs.mosquitto}/bin/mosquitto -c ${mosquittoConf}";
|
||||
readiness_probe = {
|
||||
exec.command = "${pkgs.mosquitto}/bin/mosquitto_sub -h ${dev.mqttHost} -p ${toString dev.mqttPort} -t '$$SYS/#' -C 1 -W 2";
|
||||
initial_delay_seconds = 1;
|
||||
period_seconds = 2;
|
||||
};
|
||||
};
|
||||
|
||||
ingester = {
|
||||
command = "${pkgs.go}/bin/go run ./ingester";
|
||||
command = "${pkgs.go}/bin/go run ./ingester/cmd/gebos-ingester";
|
||||
environment = ingesterEnv;
|
||||
depends_on = {
|
||||
"db".condition = "process_healthy";
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
services.gebos.supabase = {
|
||||
enable = true;
|
||||
# Compose stack points at external db-host instead of the bundled `db` service.
|
||||
postgresHost = "db-host.gebos.internal"; # TODO: real internal hostname
|
||||
postgresHost = "10.0.0.2"; # db-host on the private network
|
||||
postgresPort = 5432;
|
||||
# Studio bound to loopback only — reach via `ssh -L 3000:127.0.0.1:3000 app-host`.
|
||||
studioBindAddress = "127.0.0.1";
|
||||
@@ -18,8 +18,8 @@
|
||||
enable = true;
|
||||
# Caddy terminates TLS, proxies to Kong on 127.0.0.1:8000.
|
||||
sites = {
|
||||
"api.ge-bos.de" = { upstream = "127.0.0.1:8000"; }; # → Kong → PostgREST/GoTrue
|
||||
"app.ge-bos.de" = { staticRoot = "${config.services.gebos.frontend.package}/share/frontend"; };
|
||||
"api.gebos.online" = { upstream = "127.0.0.1:8000"; }; # → Kong → PostgREST/GoTrue
|
||||
"app.gebos.online" = { staticRoot = "${config.services.gebos.frontend.package}/share/frontend"; };
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
+17
-2
@@ -1,7 +1,17 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
{
|
||||
system.stateVersion = "25.05";
|
||||
system.stateVersion = "26.05";
|
||||
|
||||
# Bootloader. The disko layout (hardware-configurations/disk-config.nix) lays
|
||||
# down both a BIOS-boot (EF02) partition and an ESP mounted at /boot, so grub
|
||||
# works whether the VM firmware is legacy BIOS or UEFI. disko fills in
|
||||
# boot.loader.grub.devices from the EF02 partition; we just enable grub here.
|
||||
boot.loader.grub = {
|
||||
enable = true;
|
||||
efiSupport = true;
|
||||
efiInstallAsRemovable = true;
|
||||
};
|
||||
|
||||
nix.settings = {
|
||||
experimental-features = [ "nix-command" "flakes" ];
|
||||
@@ -13,7 +23,8 @@
|
||||
isNormalUser = true;
|
||||
extraGroups = [ "wheel" ];
|
||||
openssh.authorizedKeys.keys = [
|
||||
# TODO: paste the Gitea runner's deploy ed25519 pubkey here
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJuB+MbW/YuXYGZ/lm6dp+aHJ1YqfaEkiqgj/Z6+cdd0 me@larsnolden.com" # developer machines key
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB9LFaFbro5jU9jG0ZNn66FO61gniKfi865jS6TnZ+SU deploy" # deployment nodes key
|
||||
];
|
||||
};
|
||||
security.sudo.wheelNeedsPassword = false;
|
||||
@@ -22,6 +33,10 @@
|
||||
enable = true;
|
||||
settings.PasswordAuthentication = false;
|
||||
};
|
||||
# TODO: restrict inbound SSH (port 22) to the Gitea deployment runner's IP
|
||||
# only, instead of leaving it open to the world. Likely via
|
||||
# services.openssh.openFirewall = false + a firewall.extraInputRules rule
|
||||
# accepting tcp dport 22 from the runner's source address.
|
||||
|
||||
networking.firewall.enable = true;
|
||||
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
{
|
||||
# Postgres data lives on the dedicated second disk (/dev/sdb), not the OS disk.
|
||||
# The shared hardware-configuration.nix + sda disk layout come from ./default.nix.
|
||||
imports = [ ./hardware-configurations/db-data-disk.nix ];
|
||||
|
||||
networking.hostName = "db-host";
|
||||
|
||||
services.gebos.secrets.postgresAdmin = true;
|
||||
|
||||
# Skeleton: real hardware-configuration.nix + boot/fs lives next to this file
|
||||
# once we have the actual box. Marked here so the structure is visible.
|
||||
# imports = [ ./db-host.hardware.nix ];
|
||||
|
||||
services.gebos.postgres = {
|
||||
enable = true;
|
||||
listenAddresses = [ "0.0.0.0" ];
|
||||
|
||||
@@ -10,11 +10,19 @@ let
|
||||
modules = [
|
||||
./common.nix
|
||||
inputs.sops-nix.nixosModules.sops
|
||||
inputs.disko.nixosModules.disko
|
||||
# All three boxes are identical qemu guests, so they share one
|
||||
# hardware-configuration.nix and one disko disk layout (single
|
||||
# /dev/sda → GPT → LVM → ext4 root). disko derives fileSystems +
|
||||
# bootloader devices from disk-config.nix. db-host additionally
|
||||
# imports the /dev/sdb data disk from ./db-host.nix.
|
||||
./hardware-configurations/hardware-configuration.nix
|
||||
./hardware-configurations/disk-config.nix
|
||||
modules.gebos-secrets
|
||||
modules.gebos-postgres
|
||||
modules.gebos-supabase
|
||||
modules.gebos-caddy
|
||||
modules.gebos-hivemq
|
||||
modules.gebos-emqx
|
||||
modules.gebos-ingester
|
||||
modules.gebos-frontend
|
||||
./${name}.nix
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
{ lib, ... }:
|
||||
|
||||
# Second disk for db-host: the Postgres data directory lives on /dev/sdb, kept
|
||||
# separate from the OS disk (/dev/sda). The whole disk is one GPT partition,
|
||||
# ext4, mounted at /var/lib/postgresql — so the cluster data dir
|
||||
# (/var/lib/postgresql/17, the upstream NixOS default) sits on this drive and
|
||||
# survives an OS-disk rebuild.
|
||||
{
|
||||
disko.devices.disk.disk2 = {
|
||||
device = lib.mkDefault "/dev/sdb";
|
||||
type = "disk";
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
data = {
|
||||
name = "data";
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "ext4";
|
||||
mountpoint = "/var/lib/postgresql";
|
||||
mountOptions = [ "defaults" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
disko.devices = {
|
||||
disk.disk1 = {
|
||||
device = lib.mkDefault "/dev/sda";
|
||||
type = "disk";
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
boot = {
|
||||
name = "boot";
|
||||
size = "1M";
|
||||
type = "EF02";
|
||||
};
|
||||
esp = {
|
||||
name = "ESP";
|
||||
size = "500M";
|
||||
type = "EF00";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "vfat";
|
||||
mountpoint = "/boot";
|
||||
};
|
||||
};
|
||||
root = {
|
||||
name = "root";
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "lvm_pv";
|
||||
vg = "pool";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
lvm_vg = {
|
||||
pool = {
|
||||
type = "lvm_vg";
|
||||
lvs = {
|
||||
root = {
|
||||
size = "100%FREE";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "ext4";
|
||||
mountpoint = "/";
|
||||
mountOptions = [
|
||||
"defaults"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports =
|
||||
[ (modulesPath + "/profiles/qemu-guest.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "virtio_pci" "virtio_scsi" "sd_mod" "sr_mod" ];
|
||||
boot.initrd.kernelModules = [ "dm-snapshot" ];
|
||||
boot.kernelModules = [ ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
}
|
||||
+12
-12
@@ -4,9 +4,15 @@
|
||||
networking.hostName = "mqtt-ingest";
|
||||
|
||||
services.gebos.secrets.ingester = true;
|
||||
# Broker authn users (users.csv).
|
||||
services.gebos.secrets.emqx = true;
|
||||
|
||||
services.gebos.hivemq = {
|
||||
# EMQX (official container image) terminates TLS on :8883 itself and serves
|
||||
# the ingester on loopback :1883. Certs come from security.acme via HTTP-01
|
||||
# (lego standalone on :80) — no Caddy on this host anymore.
|
||||
services.gebos.emqx = {
|
||||
enable = true;
|
||||
domain = "ingest.gebos.online";
|
||||
# Persistent sessions on disk — single-node, devices reconnect on restart.
|
||||
persistentSessions = true;
|
||||
tlsPort = 8883;
|
||||
@@ -14,18 +20,12 @@
|
||||
|
||||
services.gebos.ingester = {
|
||||
enable = true;
|
||||
# mqttBroker defaults to tcp://127.0.0.1:1883 (the local HiveMQ).
|
||||
postgresUrl = "postgres://gebos_ingest@db-host.gebos.internal:5432/postgres?sslmode=require";
|
||||
# mqttBroker defaults to tcp://127.0.0.1:1883 (the local EMQX).
|
||||
postgresUrl = "postgres://gebos_ingest@10.0.0.2:5432/postgres?sslmode=require";
|
||||
# GEBOS_POSTGRES_PASSWORD sourced from /run/secrets/gebos-env (sops-nix).
|
||||
};
|
||||
|
||||
# Caddy fronts TLS for `ingest.ge-bos.de` and forwards to HiveMQ.
|
||||
services.gebos.caddy = {
|
||||
enable = true;
|
||||
sites = {
|
||||
"ingest.ge-bos.de" = { tcpProxy = "127.0.0.1:8883"; };
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 8883 ];
|
||||
# 8883: MQTTS for senders. 80: ACME HTTP-01 challenge (lego's standalone
|
||||
# solver binds it during issuance/renewal; idle otherwise).
|
||||
networking.firewall.allowedTCPPorts = [ 80 8883 ];
|
||||
}
|
||||
|
||||
@@ -4,6 +4,6 @@
|
||||
gebos-supabase = import ./gebos-supabase.nix;
|
||||
gebos-caddy = import ./gebos-caddy.nix;
|
||||
gebos-postgres = import ./gebos-postgres.nix;
|
||||
gebos-hivemq = import ./gebos-hivemq.nix;
|
||||
gebos-emqx = import ./gebos-emqx.nix;
|
||||
gebos-frontend = import ./gebos-frontend.nix;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
let
|
||||
cfg = config.services.gebos.caddy;
|
||||
|
||||
# HTTP-style sites: static file server or reverse proxy.
|
||||
siteBlock = host: site:
|
||||
if site ? staticRoot then ''
|
||||
${host} {
|
||||
@@ -18,14 +19,7 @@ let
|
||||
encode zstd gzip
|
||||
}
|
||||
''
|
||||
else if site ? tcpProxy then ''
|
||||
${host}:8883 {
|
||||
# TLS termination for MQTT — Caddy's `layer4` app would be ideal here;
|
||||
# for now, document that the upstream HiveMQ port is ${site.tcpProxy}.
|
||||
# TODO: switch to the caddy-l4 module or terminate TLS in HiveMQ directly.
|
||||
}
|
||||
''
|
||||
else throw "site `${host}` needs one of: staticRoot, upstream, tcpProxy";
|
||||
else throw "site `${host}` needs one of: staticRoot, upstream";
|
||||
in
|
||||
{
|
||||
options.services.gebos.caddy = {
|
||||
@@ -39,9 +33,8 @@ in
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.caddy = {
|
||||
enable = true;
|
||||
email = "ops@ge-bos.de"; # TODO: confirm ACME contact
|
||||
extraConfig = lib.concatStringsSep "\n"
|
||||
(lib.mapAttrsToList siteBlock cfg.sites);
|
||||
email = "ops@gebos.online"; # TODO: confirm ACME contact
|
||||
extraConfig = lib.concatStringsSep "\n" (lib.mapAttrsToList siteBlock cfg.sites);
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.gebos.emqx;
|
||||
|
||||
stateDir = "/var/lib/gebos-emqx";
|
||||
|
||||
# UID/GID of the `emqx` user baked into the official image. Host-side files
|
||||
# the container must read or write (certs, bootstrap users, data dir) are
|
||||
# chowned to this numeric id. On these hosts uid 1000 is the `deploy` user,
|
||||
# which already has passwordless sudo, so this grants it nothing new.
|
||||
emqxUid = 1000;
|
||||
emqxGid = 1000;
|
||||
|
||||
# Full emqx.conf, replacing the image default. EMQX reads HOCON natively;
|
||||
# container paths below are bind mounts declared on the oci-container.
|
||||
emqxConf = pkgs.writeText "emqx.conf" ''
|
||||
node {
|
||||
name = "emqx@127.0.0.1"
|
||||
# Erlang distribution cookie. Not treated as a secret: the dist/epmd
|
||||
# ports are never opened in the firewall and the only local users are
|
||||
# root and deploy (wheel).
|
||||
cookie = "gebos-emqx"
|
||||
data_dir = "data"
|
||||
}
|
||||
|
||||
cluster {
|
||||
name = gebos
|
||||
# Single node, and required to be `singleton` for durable sessions on
|
||||
# the open-source builtin_local storage backend.
|
||||
discovery_strategy = singleton
|
||||
}
|
||||
|
||||
# Dashboard on loopback only (the container uses host networking) — reach
|
||||
# it via `ssh -L 18083:127.0.0.1:18083`. First login is admin/public with a
|
||||
# forced password change; the changed password persists in the data dir.
|
||||
dashboard {
|
||||
listeners.http.bind = "127.0.0.1:18083"
|
||||
}
|
||||
|
||||
# Plain MQTT for the co-located ingester only.
|
||||
listeners.tcp.default {
|
||||
bind = "${cfg.bindAddress}:${toString cfg.port}"
|
||||
}
|
||||
|
||||
# MQTTS for the senders. EMQX terminates TLS itself; certs are managed by
|
||||
# security.acme on the host and installed into ${stateDir}/certs (mounted
|
||||
# at /etc/emqx/certs). EMQX re-reads the PEM files from disk periodically
|
||||
# (~120 s), so renewals need no restart — paths stay fixed, contents swap.
|
||||
listeners.ssl.default {
|
||||
bind = "0.0.0.0:${toString cfg.tlsPort}"
|
||||
ssl_options {
|
||||
certfile = "/etc/emqx/certs/fullchain.pem"
|
||||
keyfile = "/etc/emqx/certs/privkey.pem"
|
||||
versions = ["tlsv1.2", "tlsv1.3"]
|
||||
}
|
||||
}
|
||||
|
||||
# The schema defines websocket listeners on 8083/8084 by default — unused.
|
||||
listeners.ws.default.enable = false
|
||||
listeners.wss.default.enable = false
|
||||
|
||||
# Sessions and queued messages survive a broker restart (QoS > 0 with
|
||||
# clean_start = false), mirroring HiveMQ's file persistence mode.
|
||||
durable_sessions.enable = ${lib.boolToString cfg.persistentSessions}
|
||||
|
||||
# Username/password auth against the built-in database, seeded from the
|
||||
# sops-rendered users.csv on first start (bootstrap only inserts users that
|
||||
# do not exist yet — to rotate a password, delete the user in the dashboard
|
||||
# or via `emqx ctl`, then restart to re-import).
|
||||
authentication = [
|
||||
{
|
||||
mechanism = password_based
|
||||
backend = built_in_database
|
||||
user_id_type = username
|
||||
password_hash_algorithm { name = plain, salt_position = disable }
|
||||
bootstrap_file = "/etc/emqx/auth/users.csv"
|
||||
bootstrap_type = plain
|
||||
}
|
||||
]
|
||||
|
||||
# Topic-level authorization from the static acl.conf; anything not
|
||||
# explicitly allowed is denied.
|
||||
authorization {
|
||||
no_match = deny
|
||||
sources = [
|
||||
{
|
||||
type = file
|
||||
enable = true
|
||||
path = "/etc/emqx/auth/acl.conf"
|
||||
}
|
||||
]
|
||||
}
|
||||
'';
|
||||
|
||||
# Topic policy (non-secret — usernames and topics only, no credentials).
|
||||
# Devices publish under `acrios/<IMSI>/<metric>` (the sender firmware's
|
||||
# mqtt_topic_base), so that is the namespace the ingester consumes and the
|
||||
# device user is scoped to. `admin` is is_superuser in users.csv and bypasses
|
||||
# this ACL entirely (break-glass).
|
||||
#
|
||||
# Option A (current): one shared device user (`bender`) for the whole fleet.
|
||||
# Tenant isolation is enforced downstream by the ingester's IMSI→tenant
|
||||
# registry lookup, NOT at the broker — any device could publish under any
|
||||
# IMSI. TODO(Option B): per-device users with username == IMSI and an ACL
|
||||
# rule scoped to `acrios/''${username}/#` for real anti-spoofing.
|
||||
aclConf = pkgs.writeText "emqx-acl.conf" ''
|
||||
%% ingester: consume the whole device tree.
|
||||
{allow, {username, "ingester"}, subscribe, ["acrios/#"]}.
|
||||
%% device fleet: publish telemetry + LWT/status, subscribe downlinks.
|
||||
{allow, {username, "bender"}, all, ["acrios/#"]}.
|
||||
{deny, all}.
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.services.gebos.emqx = {
|
||||
enable = lib.mkEnableOption "EMQX broker (official container image) with native TLS";
|
||||
|
||||
image = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "emqx/emqx:5.8.6";
|
||||
description = "Official EMQX image reference to run.";
|
||||
};
|
||||
|
||||
domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "ingest.gebos.online";
|
||||
description = "Public hostname the senders connect to; ACME cert is issued for it.";
|
||||
};
|
||||
|
||||
tlsPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8883;
|
||||
description = "MQTTS listener port (TLS terminated by EMQX itself).";
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 1883;
|
||||
description = "Plain MQTT TCP listener port (what the ingester connects to).";
|
||||
};
|
||||
|
||||
bindAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "127.0.0.1";
|
||||
description = ''
|
||||
Address the plain MQTT listener binds to. Defaults to loopback:
|
||||
external clients only ever reach the TLS listener.
|
||||
'';
|
||||
};
|
||||
|
||||
persistentSessions = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Whether client sessions and queued messages survive a restart
|
||||
(EMQX durable sessions) or are kept only in memory.
|
||||
'';
|
||||
};
|
||||
|
||||
acmeEmail = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "ops@gebos.online";
|
||||
description = "ACME account contact for the broker certificate.";
|
||||
};
|
||||
|
||||
bootstrapUsersFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = config.services.gebos.secrets.emqxBootstrapUsersFile;
|
||||
defaultText = lib.literalExpression "config.services.gebos.secrets.emqxBootstrapUsersFile";
|
||||
description = ''
|
||||
CSV (user_id,password,is_superuser) seeding EMQX's built-in auth
|
||||
database. Defaults to the sops-rendered secret; copied into the state
|
||||
dir on service start so the container user can read it.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
virtualisation.oci-containers = {
|
||||
backend = "docker";
|
||||
containers.emqx = {
|
||||
image = cfg.image;
|
||||
# Host networking: EMQX binds host ports directly, so loopback-only
|
||||
# listeners (plain MQTT, dashboard) really are loopback-only and the
|
||||
# TLS listener sees real client source addresses.
|
||||
extraOptions = [ "--network=host" ];
|
||||
volumes = [
|
||||
"${emqxConf}:/opt/emqx/etc/emqx.conf:ro"
|
||||
"${aclConf}:/etc/emqx/auth/acl.conf:ro"
|
||||
"${stateDir}/auth/users.csv:/etc/emqx/auth/users.csv:ro"
|
||||
"${stateDir}/certs:/etc/emqx/certs:ro"
|
||||
"${stateDir}/data:/opt/emqx/data"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# The oci-containers unit is docker-emqx.service; extend it to stage the
|
||||
# writable state the container mounts. The bootstrap users file is copied
|
||||
# (not symlinked) out of the sops tmpfs so it can be owned by the
|
||||
# container's emqx uid.
|
||||
systemd.services.docker-emqx = {
|
||||
# Don't start before the first cert issuance has been attempted. If ACME
|
||||
# fails (e.g. Porkbun creds not provisioned yet), EMQX crash-loops on the
|
||||
# missing certfile until the cert lands — Restart=always retries.
|
||||
after = [ "acme-finished-${cfg.domain}.target" ];
|
||||
wants = [ "acme-finished-${cfg.domain}.target" ];
|
||||
preStart = ''
|
||||
install -d -m 750 -o ${toString emqxUid} -g ${toString emqxGid} ${stateDir}/data
|
||||
install -d -m 755 ${stateDir}/auth ${stateDir}/certs
|
||||
install -m 400 -o ${toString emqxUid} -g ${toString emqxGid} \
|
||||
${cfg.bootstrapUsersFile} ${stateDir}/auth/users.csv
|
||||
'';
|
||||
};
|
||||
|
||||
# Cert issuance + renewal via HTTP-01: lego's built-in standalone server
|
||||
# answers the challenge on :80 (nothing else listens there — Caddy is gone
|
||||
# from this host), so no DNS API secrets are needed; the host just keeps
|
||||
# port 80 open alongside 8883. postRun is the deploy hook: install the
|
||||
# PEMs at the fixed paths EMQX watches, owned by the container user. EMQX
|
||||
# hot-reloads them; no restart or `emqx ctl` needed. RSA keys because
|
||||
# embedded sender TLS stacks often lack ECDSA.
|
||||
security.acme = {
|
||||
acceptTerms = true;
|
||||
certs.${cfg.domain} = {
|
||||
email = cfg.acmeEmail;
|
||||
listenHTTP = ":80";
|
||||
keyType = "rsa2048";
|
||||
postRun = ''
|
||||
install -D -m 644 -o ${toString emqxUid} -g ${toString emqxGid} \
|
||||
fullchain.pem ${stateDir}/certs/fullchain.pem
|
||||
install -D -m 600 -o ${toString emqxUid} -g ${toString emqxGid} \
|
||||
key.pem ${stateDir}/certs/privkey.pem
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.gebos.hivemq;
|
||||
in
|
||||
{
|
||||
options.services.gebos.hivemq = {
|
||||
enable = lib.mkEnableOption "HiveMQ CE single-node broker";
|
||||
persistentSessions = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
tlsPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8883;
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
# TODO: package HiveMQ CE (download tarball + JRE wrapper) and wire as a
|
||||
# systemd unit. Until then this module only declares options so dependent
|
||||
# hosts can be evaluated.
|
||||
systemd.services.gebos-hivemq = {
|
||||
description = "HiveMQ CE";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${pkgs.coreutils}/bin/true"; # TODO: real HiveMQ command
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -13,12 +13,20 @@ in
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
# TimescaleDB ships under the non-OSI "Timescale License" (TSL), which
|
||||
# nixpkgs flags as unfree. Allow just this package rather than opening the
|
||||
# whole config to unfree.
|
||||
nixpkgs.config.allowUnfreePredicate = pkg:
|
||||
lib.elem (lib.getName pkg) [ "timescaledb" ];
|
||||
|
||||
services.postgresql = {
|
||||
enable = true;
|
||||
package = pkgs.postgresql_17;
|
||||
enableTCPIP = true;
|
||||
# listen_addresses is managed explicitly below. The upstream postgresql
|
||||
# module also defines it (via its own default / enableTCPIP), so mkForce
|
||||
# makes this module the single source of truth and avoids the collision.
|
||||
settings = {
|
||||
listen_addresses = lib.concatStringsSep "," cfg.listenAddresses;
|
||||
listen_addresses = lib.mkForce (lib.concatStringsSep "," cfg.listenAddresses);
|
||||
shared_preload_libraries = "timescaledb,pg_stat_statements";
|
||||
};
|
||||
extensions = ps: with ps; [
|
||||
|
||||
@@ -17,6 +17,31 @@ let
|
||||
];
|
||||
ingesterKeys = [ "ingester_postgres_password" ];
|
||||
postgresAdminKeys = [ "postgres_admin_password" ];
|
||||
emqxKeys = [
|
||||
"emqx_ingester_password"
|
||||
"emqx_admin_password"
|
||||
"emqx_device_password"
|
||||
];
|
||||
|
||||
# EMQX authn bootstrap CSV (bootstrap_type = plain). Seeds the broker's
|
||||
# built-in auth database on first start. Broker users:
|
||||
# ingester — the Go ingester; SUBSCRIBE the whole device tree (see the ACL
|
||||
# in gebos-emqx.nix).
|
||||
# admin — operational break-glass; is_superuser bypasses the ACL.
|
||||
# bender — shared device user for the whole fleet (Option A). Tenant
|
||||
# isolation is enforced downstream by the ingester's IMSI→tenant
|
||||
# registry, NOT at the broker. TODO(Option B): per-device users
|
||||
# (username == IMSI) — see the ACL comment in gebos-emqx.nix.
|
||||
#
|
||||
# Plaintext passwords are acceptable here because the file is sops-encrypted
|
||||
# at rest and rendered 0400 into a tmpfs at runtime; passwords must not
|
||||
# contain commas (CSV).
|
||||
emqxUsersCsv = ingesterPassword: adminPassword: devicePassword: ''
|
||||
user_id,password,is_superuser
|
||||
ingester,${ingesterPassword},false
|
||||
admin,${adminPassword},true
|
||||
bender,${devicePassword},false
|
||||
'';
|
||||
|
||||
# Render an env file body from a list of (key, envVarName) pairs.
|
||||
envBody = pairs: lib.concatStringsSep "\n" (map
|
||||
@@ -41,7 +66,8 @@ let
|
||||
enabledKeys =
|
||||
lib.optionals cfg.supabase supabaseKeys
|
||||
++ lib.optionals cfg.ingester ingesterKeys
|
||||
++ lib.optionals cfg.postgresAdmin postgresAdminKeys;
|
||||
++ lib.optionals cfg.postgresAdmin postgresAdminKeys
|
||||
++ lib.optionals cfg.emqx emqxKeys;
|
||||
in
|
||||
{
|
||||
options.services.gebos.secrets = {
|
||||
@@ -55,6 +81,26 @@ in
|
||||
postgresAdmin = lib.mkEnableOption ''
|
||||
Postgres bootstrap admin password applied via a one-shot ALTER ROLE unit.'';
|
||||
|
||||
emqx = lib.mkEnableOption ''
|
||||
EMQX broker secrets: the authn bootstrap users.csv (ingester + admin +
|
||||
device MQTT users).'';
|
||||
|
||||
emqxBootstrapUsersFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
Path of the rendered EMQX authn bootstrap users.csv consumed by the
|
||||
gebos-emqx broker. Resolves to the sops-templated secret when sops
|
||||
material is present and the emqx toggle is on, else to a baked-in
|
||||
dev-defaults file (so first-boot eval and local VMs work).
|
||||
'';
|
||||
default =
|
||||
if hasSecrets && cfg.emqx
|
||||
then config.sops.templates."emqx-users.csv".path
|
||||
else "${pkgs.writeText "emqx-users-dev.csv"
|
||||
(emqxUsersCsv "dev-ingester" "dev-admin" "dev-device")}";
|
||||
};
|
||||
|
||||
envFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
readOnly = true;
|
||||
@@ -90,6 +136,17 @@ in
|
||||
mode = "0400";
|
||||
owner = "root";
|
||||
};
|
||||
|
||||
# Rendered as a standalone file — the gebos-emqx preStart copies it into
|
||||
# the broker state dir, chowned to the container's emqx uid.
|
||||
templates."emqx-users.csv" = lib.mkIf cfg.emqx {
|
||||
content = emqxUsersCsv
|
||||
config.sops.placeholder.emqx_ingester_password
|
||||
config.sops.placeholder.emqx_admin_password
|
||||
config.sops.placeholder.emqx_device_password;
|
||||
mode = "0400";
|
||||
owner = "root";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# Template for nix/secrets/secrets.yaml — the sops-encrypted store.
|
||||
#
|
||||
# To bootstrap:
|
||||
# 1. cp nix/secrets/secrets.yaml.example nix/secrets/secrets.yaml
|
||||
# 2. Replace the placeholder values below with real ones.
|
||||
# (See nix/secrets/README.md for how to generate JWT_SECRET / ANON_KEY /
|
||||
# SERVICE_ROLE_KEY etc.)
|
||||
# 3. sops -e -i nix/secrets/secrets.yaml
|
||||
# (requires sops + age + the recipients in .sops.yaml to be real keys)
|
||||
#
|
||||
# Once encrypted, edit in place with: sops nix/secrets/secrets.yaml
|
||||
#
|
||||
# Local dev does NOT need this file at all — process-compose.nix bakes in
|
||||
# dev defaults and the NixOS modules fall back to a writeText env file when
|
||||
# nix/secrets/secrets.yaml is absent. This file is only consumed at activation
|
||||
# time on real hosts.
|
||||
# Supabase compose stack (consumed on app-host by gebos-supabase.service)
|
||||
supabase_postgres_password: ENC[AES256_GCM,data:OrjkfiumbB3uMD0tuDkgzaiLDgnt20wYOP9tBAyVwVUXQJxLiDypJoUb4wTY,iv:b4/Lep0Jzb6UtAhyOCpcetXgppul+V5O43zokMtpM2M=,tag:3fvi5xpverfSG+xc/4/HHg==,type:str]
|
||||
supabase_jwt_secret: ENC[AES256_GCM,data:csu0Rj/DM07DCtYFieIe6eEca+eUPR6KGfQBqa6cqRe7bX4=,iv:OKYFa7Y1bbEAKh1ueM7K8JDLldrTJL/r83AGTtMx/IU=,tag:y6OGVofm4cUFPc0BBmkY+A==,type:str]
|
||||
supabase_anon_key: ENC[AES256_GCM,data:ReE4g5pNUHqi9M9ejnR88XUjwgaibcgZRM5GCMl+Ds7pzKUY442nqEtc2xVO2vQ=,iv:+lp/a/JshLZfdkxm4uQrgOKxBFmIduLtu3aUAhWrKlE=,tag:ft9swD2SeFJ/jMyL07kmGQ==,type:str]
|
||||
supabase_service_role_key: ENC[AES256_GCM,data:QZQbvJ6rkyoHIpxIvjVDFwxeZ4oIb7Bt2h2vgtW8/sp6swSxtW/oAeVfLkdeduir+2BC5mFAhA==,iv:/F5F3MRC5Yebn68g45F1sj4d18+60JZupneHa3ipExs=,tag:fuzDuSj2npxKQxP2Pz+cjQ==,type:str]
|
||||
supabase_dashboard_password: ENC[AES256_GCM,data:6KHJCPJ66dW9n/uzb3mpGIYqGZABHeEew1H5kx3I8kqgABqXcnHi6w==,iv:87NBfRoNOwtEKgGC0CacY/0xyVE5DQJpuN0X+7j8dfQ=,tag:TcMmIS2GgUlBXlH0elC7Rg==,type:str]
|
||||
# Ingester (consumed on mqtt-ingest by gebos-ingester.service)
|
||||
ingester_postgres_password: ENC[AES256_GCM,data:6jy1G6zG4r0z3c9B6s7jPUtvn/Nr/J2HQN23ch6AGFigiVbefY3hCw==,iv:X/PWQjTW/++MVGsi+hcxbkOJZQMOUH0a4mRH5CbOw4A=,tag:TBKFIdbXZ2YzDLbjdnsyxg==,type:str]
|
||||
# Postgres bootstrap (consumed on db-host by a one-shot ALTER ROLE unit)
|
||||
postgres_admin_password: ENC[AES256_GCM,data:j5XHA2ZepUq6rJMmIrKr88xjNLiMsJ+ng7CeQxNO3fVwydQhMW0hl0KB,iv:ZXilU07B3aJRQeIM0GVypFcGe4K7y5BsEOa2cIYzAOo=,tag:CEdzYR7NElODPYmTWdVccQ==,type:str]
|
||||
# EMQX broker users (consumed on mqtt-ingest by gebos-emqx as bootstrap users.csv).
|
||||
# PLAIN passwords (password-type defaults to PLAIN; the whole file is sops-
|
||||
# encrypted). The ingester will authenticate as `ingester`; `admin` is for ops.
|
||||
emqx_ingester_password: ENC[AES256_GCM,data:pV1FIY5uNeZ2zkjB9dv3C5DP2AmeyaV55lqgizQoKGKq0w3PU1gV+xev9g==,iv:E+anGu88+fhBQRLp55tYghx5SvHVniolXaj6kE/3FrQ=,tag:iJpH+0zAowKq+t/xXR+NjA==,type:str]
|
||||
emqx_admin_password: ENC[AES256_GCM,data:lrOUYSCkYFkXlelAoMUp9WFxo7COJWYkDSBVEr2RT0B2lfMnF3MzqhqJ,iv:uZi+oCw6esNuweIAFTGn3A6Ix5ctwn1Oj4GBchalXmA=,tag:84QigtqxRr1zddMtCsFu9w==,type:str]
|
||||
# Shared device login (Option A) — must match mqtt_user/mqtt_password on the
|
||||
# senders. Per-device users (Option B) are a TODO in gebos-secrets.nix.
|
||||
emqx_device_password: ENC[AES256_GCM,data:pFp3RxJoDw==,iv:1FQzAHSr4VIyuuq2qgFIcuzOq487W4sKV1x8Opoa0jM=,tag:zwn7dOmZArUOtlZBv5G63w==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzaVAwall2bTE0Y09ZeUJn
|
||||
SFR2cHBDaFZtRmpodGV0ZHJsUzVuWGdNb213ClFFZ04xV0pTVzR5eFh0NjNsYWpZ
|
||||
SU9IUzBoZWJaNjZmUW1Tc3NER2hpRDAKLS0tIG9BVnAzYW1jL1VZOHRLeTJvQzh0
|
||||
Y3dtSnp5eGNnWnN0SWVYa0laNExtUWsKhiZOiy1otoGW+Y5HOvFz1nc549LTyvvl
|
||||
hP0McG7/p0Awh9bQNzhvgQO88M4S0wIrDJ3P0Gu5/HKzKZXQ8EhSOA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1cx8ul285kjkzmnhw6skdstnzrxnnme4xkflknzn7yhv52fgxqevqkd66cn
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzRHpzVWtpQnFrWFBNV1c5
|
||||
UitrRVdQWGUyb0Z6WHVlMFhibjRhWXhFMmdzCnpyMW1QVUErMjBBUHJvMnN6RWJP
|
||||
b0JselpJQkoxNDFtdU5TYlRBRmpUOGMKLS0tIDdQNGY3TUxhZGtkdzVZSlIwRXZK
|
||||
NGJqWmJoSkFXMTVKOGNWMGlUN0o1czQKw3/naLb0zt5yHeITSJp9GKEOJfW5bSlL
|
||||
CtQLx6Nn1WFOTcuoxgaf/xtUCBiSYSRg7pM9KwX/Pfu9gpvWjScUZA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1wpl3vz60tlt880p5fmfedr8c59dm4kf0czsacv0w5my706twuf5q9p0x43
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHL3hhWG5WL2N5THRaUFBz
|
||||
SEo0SE0vQUZIZ05ad0ZUN3lrU3hHSTRISm1zCmVEdWxRSU8zSE1vODNwbVVub2VZ
|
||||
eWZqU3FiOG9mNHpBZjBQeEdhYkF6SmcKLS0tIEh1WkJIOXdDRmNMMHR2VE5jdXpU
|
||||
WHRqTUFJQTJQTVFBSzBqVlVWY2ROU28KToql5OybiwITIS9GHieUGfgRILlhmUyM
|
||||
abkD57fWwMzh1h5UNdUt/VMQEuYX9pK/QhLOrWNgZumk7JesbKRL3A==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1g57pznep69nlyv3sltz6k0sml47v68m03gh68jkqwcq4jdx68vtsvnefq0
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqUTN5NENOcm1WRnltbzBE
|
||||
RFgwY3l5cXVYeVptZG9UenBqc25GbCs2NFFFCkNvSytuOVUzRkV2R0doMFpvNndu
|
||||
a1NyYmVVRE9LRmpwdis4Sk5lcnNoMkEKLS0tIFhTZjFhZVhFZ2FIKzg4dzJ1a25y
|
||||
cW84SytPUGZmMkVxdmFhMmQ2dy9XZlEK/S1drmtv8te8m2Zgpkm2l+RdOTTbhKCs
|
||||
ziyUG3WkxBzihV7WpVyfKOu8XMJfLUsgiusjCeCsmkdq+UN+VBlZ/g==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age190gu75rf3ra89mhk27xe3tv87tad087altqhugjlhkerqwe2jfqsnu738d
|
||||
encrypted_regex: ^(supabase_|ingester_|postgres_admin_|emqx_|porkbun_)
|
||||
lastmodified: "2026-07-02T13:11:42Z"
|
||||
mac: ENC[AES256_GCM,data:eJRwD6eGsmca+Gv3TP7JLSoCXlH34boilkxxEqRkbz8vNsCpEANZufet4Oes8vpNubU8yx509nrGo3SoC+MihvRaw/hfWqYOlXhlB0j6RsQuuG3LRQFVLJ1m1Q4zw5PYEcB/JHc4ORSIDrMb6nlemBc1dsDItIqyxlOqFnesUec=,iv:EBmK7XZ9ZDQT68FiItivu3QglhX0LfvMgw6n6qUhXUM=,tag:IqvQ4CVOrnWpQ3cE/mC/2w==,type:str]
|
||||
version: 3.13.1
|
||||
@@ -27,3 +27,13 @@ ingester_postgres_password: "replace-me-openssl-rand-hex-32"
|
||||
|
||||
# Postgres bootstrap (consumed on db-host by a one-shot ALTER ROLE unit)
|
||||
postgres_admin_password: "replace-me-openssl-rand-hex-32"
|
||||
|
||||
# EMQX broker users (consumed on mqtt-ingest by gebos-emqx). Plaintext
|
||||
# passwords in a bootstrap users.csv (the whole file is sops-encrypted at
|
||||
# rest); no commas allowed. The ingester will authenticate as `ingester`;
|
||||
# `admin` is the operational break-glass superuser.
|
||||
emqx_ingester_password: "replace-me-openssl-rand-hex-32"
|
||||
emqx_admin_password: "replace-me-openssl-rand-hex-32"
|
||||
# Shared device login (Option A) — must match mqtt_user/mqtt_password on the
|
||||
# senders. Per-device users (Option B) are a TODO in gebos-emqx.nix.
|
||||
emqx_device_password: "replace-me-to-match-the-sender-mqtt_password"
|
||||
|
||||
@@ -16,7 +16,7 @@ services:
|
||||
image: kong:2.8.1
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:8000:8000/tcp" # Caddy proxies api.ge-bos.de here
|
||||
- "127.0.0.1:8000:8000/tcp" # Caddy proxies api.gebos.online here
|
||||
environment:
|
||||
KONG_DATABASE: "off"
|
||||
KONG_DECLARATIVE_CONFIG: /home/kong/kong.yml
|
||||
@@ -29,10 +29,10 @@ services:
|
||||
environment:
|
||||
GOTRUE_DB_DRIVER: postgres
|
||||
GOTRUE_DB_DATABASE_URL: postgres://supabase_auth_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
|
||||
GOTRUE_SITE_URL: https://app.ge-bos.de
|
||||
GOTRUE_SITE_URL: https://app.gebos.online
|
||||
GOTRUE_JWT_SECRET: ${JWT_SECRET}
|
||||
GOTRUE_JWT_EXP: "3600"
|
||||
API_EXTERNAL_URL: https://api.ge-bos.de
|
||||
API_EXTERNAL_URL: https://api.gebos.online
|
||||
|
||||
rest:
|
||||
image: postgrest/postgrest:v12.0.2
|
||||
@@ -52,7 +52,7 @@ services:
|
||||
STUDIO_PG_META_URL: http://meta:8080
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
DEFAULT_ORGANIZATION_NAME: Gebos
|
||||
SUPABASE_URL: https://api.ge-bos.de
|
||||
SUPABASE_URL: https://api.gebos.online
|
||||
DASHBOARD_USERNAME: gebos
|
||||
DASHBOARD_PASSWORD: ${DASHBOARD_PASSWORD}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user