Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,48 @@
|
||||
# Prometheus — Boilerplate
|
||||
|
||||
Scrapes `/metrics` from the three boilerplate apps every 15s.
|
||||
|
||||
## Files
|
||||
|
||||
- `prometheus.yml` — scrape jobs (api / worker / web)
|
||||
- `docker-compose.yml` — Coolify-friendly compose stack
|
||||
- `secrets/metrics_token` — bearer token (chmod 600, do NOT commit)
|
||||
|
||||
## First-time deploy on Hetzner staging
|
||||
|
||||
```sh
|
||||
# 1. Create the secret file with the same value as METRICS_TOKEN in Coolify
|
||||
mkdir -p secrets
|
||||
openssl rand -hex 32 > secrets/metrics_token
|
||||
chmod 600 secrets/metrics_token
|
||||
|
||||
# 2. Create the shared docker network (also used by Grafana)
|
||||
docker network create boilerplate-observability
|
||||
|
||||
# 3. Start the stack
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Then expose `prometheus.boiler.saas-ctrl.com` via Coolify's proxy with
|
||||
**basic-auth or Cloudflare-Access** in front — Prometheus has no
|
||||
built-in auth and ships internal state.
|
||||
|
||||
## Verifying scrape targets
|
||||
|
||||
```sh
|
||||
curl -s -H "Authorization: Bearer $(cat secrets/metrics_token)" \
|
||||
https://api.boiler.saas-ctrl.com/metrics | head -20
|
||||
```
|
||||
|
||||
The Prometheus UI at `https://prometheus.boiler.saas-ctrl.com/targets`
|
||||
should list all three jobs as **UP**.
|
||||
|
||||
## Cardinality budget
|
||||
|
||||
- `ai_tokens_consumed_total` — capped at ~100 tenant buckets + 1 `other`
|
||||
- `subscription_active` — capped by the plan-id table (single digits today)
|
||||
- All other metrics have bounded label sets (HTTP routes, queue names)
|
||||
|
||||
If a panel suddenly shows millions of series, the most likely culprit
|
||||
is a new histogram with an unbounded label — search the codebase for
|
||||
`labels({` and verify.
|
||||
@@ -0,0 +1,45 @@
|
||||
# Prometheus stack for the boilerplate.
|
||||
#
|
||||
# Deploy as a Coolify "Docker Compose" service or with vanilla
|
||||
# `docker compose up -d` on the Hetzner observability box. The bind mount
|
||||
# expects the prometheus.yml + secrets file to live alongside this file.
|
||||
#
|
||||
# Required files alongside this compose:
|
||||
# * prometheus.yml — committed
|
||||
# * secrets/metrics_token — NOT committed; operator provisions
|
||||
#
|
||||
# Storage: a named volume keeps tsdb data so a `compose down && up` won't
|
||||
# wipe history. Retention is set to 30d which is plenty for an 8-week
|
||||
# staging cycle on 32 GB RAM.
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.6.0
|
||||
container_name: boilerplate-prometheus
|
||||
restart: unless-stopped
|
||||
user: "65534:65534"
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=30d
|
||||
- --web.console.libraries=/usr/share/prometheus/console_libraries
|
||||
- --web.console.templates=/usr/share/prometheus/consoles
|
||||
- --web.enable-lifecycle
|
||||
- --web.external-url=https://prometheus.boiler.saas-ctrl.com
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./secrets:/etc/prometheus/secrets:ro
|
||||
- prometheus-data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks:
|
||||
- observability
|
||||
|
||||
networks:
|
||||
observability:
|
||||
name: boilerplate-observability
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
name: boilerplate-prometheus-data
|
||||
@@ -0,0 +1,81 @@
|
||||
# Prometheus scrape configuration for the boilerplate.
|
||||
#
|
||||
# Scrapes the three application processes' /metrics endpoints with
|
||||
# bearer-token authentication. METRICS_TOKEN is injected via the
|
||||
# `metrics_token` file (Docker Secret) or the PROM_METRICS_TOKEN env var
|
||||
# at compose-up time.
|
||||
#
|
||||
# Targets:
|
||||
# * api — apps/api admin metrics surface (port 3201 default)
|
||||
# * worker — apps/worker admin server (port 9101 default; 9100 is health-probe)
|
||||
# * web — apps/web SSR resource route on /api/metrics
|
||||
#
|
||||
# Operator deploys this on the Hetzner staging server alongside Sentry.
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
scrape_timeout: 10s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
cluster: boilerplate
|
||||
environment: staging
|
||||
|
||||
scrape_configs:
|
||||
- job_name: boilerplate-api
|
||||
metrics_path: /metrics
|
||||
scheme: https
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials_file: /etc/prometheus/secrets/metrics_token
|
||||
static_configs:
|
||||
- targets: ["api.boiler.saas-ctrl.com"]
|
||||
labels:
|
||||
service: api
|
||||
|
||||
- job_name: boilerplate-worker
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials_file: /etc/prometheus/secrets/metrics_token
|
||||
static_configs:
|
||||
# Worker admin server runs inside the private network on
|
||||
# WORKER_ADMIN_PORT (9101 default for the general queue group;
|
||||
# 9102/9103/9104 for mail/import/webhook groups per #45).
|
||||
#
|
||||
# OPERATOR WIRING REQUIRED — pick one:
|
||||
# (a) Attach Prometheus to the same Docker network as the worker
|
||||
# and use the worker container's stable alias (in Coolify v4
|
||||
# the UUID-based generated name changes on every redeploy
|
||||
# so you must either add a docker-compose override that
|
||||
# sets `networks.<net>.aliases: [boilerplate-worker]` or
|
||||
# use Coolify's "Internal-only" service feature).
|
||||
# (b) Expose the worker admin server through Traefik on a
|
||||
# dedicated hostname (e.g. worker-admin.<domain>) with a
|
||||
# Traefik IPAllowList middleware restricted to the
|
||||
# Prometheus host, plus the standard Bearer-token auth.
|
||||
#
|
||||
# Until (a) or (b) is wired the worker job stays DOWN; api+web
|
||||
# cover the HTTP-side metrics, the per-queue queue_jobs_*
|
||||
# counters from the worker registry are not yet visible.
|
||||
- targets: ["worker.boilerplate.internal:9101"]
|
||||
labels:
|
||||
service: worker
|
||||
|
||||
- job_name: boilerplate-web
|
||||
metrics_path: /api/metrics
|
||||
scheme: https
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials_file: /etc/prometheus/secrets/metrics_token
|
||||
static_configs:
|
||||
# /api/metrics is admin-only — the resource route is mounted on
|
||||
# the admin host. Scraping the tenant-facing host returns 404 by
|
||||
# design. Use admin.<domain> for staging; in prod set to
|
||||
# admin.<prod-domain>.
|
||||
- targets: ["admin.boiler.saas-ctrl.com"]
|
||||
labels:
|
||||
service: web
|
||||
|
||||
# When promoting to production, also scrape the production targets behind
|
||||
# their own bearer-token. Add scrape_configs entries with environment=production.
|
||||
@@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
Reference in new issue
Block a user