OnePlatform Monitoring and Alerting Guide¶

healthcheck:
  test: ["CMD-SHELL", "wget -qO- http://localhost:3000/healthz || exit 1"]
  interval: 10s
  timeout: 5s
  retries: 5
  start_period: 20s

sinks:
  loki:
    type: loki
    inputs:
      - parse_docker_json
    endpoint: http://loki:3100
    encoding:
      codec: json
    # Use the service field merged from the inner JSON log for routing.
    # Falls back to the container_id if the inner log is not structured JSON.
    labels:
      service: "{{ service }}"
      level:   "{{ level }}"
      env:     "production"
    # Loki requires logs to arrive in chronological order per stream.
    # out_of_order_action: rewrite rewrites the timestamp rather than dropping.
    out_of_order_action: rewrite

sinks:
  elasticsearch:
    type: elasticsearch
    inputs:
      - parse_docker_json
    endpoints:
      - http://elasticsearch:9200
    index: "oneplatform-logs-%Y.%m.%d"
    encoding:
      codec: json
    auth:
      strategy: basic
      user: elastic
      password: "${ELASTIC_PASSWORD}"
    # Bulk ingest for throughput; flush every 1s or 500 events
    batch:
      max_events: 500
      timeout_secs: 1

transforms:
  # Only forward warn and error to Elasticsearch (reduces cost)
  filter_errors_only:
    type: filter
    inputs:
      - parse_docker_json
    condition:
      type: vrl
      source: |
        .level == "warn" || .level == "error"

sinks:
  elasticsearch:
    inputs:
      - filter_errors_only
    # ... rest of sink config

logging:
  driver: "json-file"
  options:
    max-size: "50m"   # rotate when file reaches 50 MB
    max-file: "5"     # keep at most 5 rotated files per container

global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:

  # Caddy exposes a Prometheus metrics endpoint at /metrics (enabled via the
  # metrics global option in the Caddyfile). This is internal-only.
  - job_name: caddy
    static_configs:
      - targets: ['caddy:2019']
    metrics_path: /metrics

  # MinIO built-in Prometheus endpoint
  - job_name: minio
    static_configs:
      - targets: ['minio:9000']
    metrics_path: /minio/v2/metrics/cluster
    scheme: http

  # redis_exporter — scrapes Redis INFO and exposes Prometheus metrics
  - job_name: redis
    static_configs:
      - targets: ['redis-exporter:9121']

  # pgbouncer_exporter — scrapes PgBouncer SHOW STATS/POOLS/CLIENTS
  - job_name: pgbouncer
    static_configs:
      - targets: ['pgbouncer-exporter:9127']

services:
  prometheus:
    image: prom/prometheus:v2.52.0
    container_name: prometheus
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'
    ports:
      - "127.0.0.1:9090:9090"
    networks:
      - oneplatform-internal
    restart: unless-stopped

  redis-exporter:
    image: oliver006/redis_exporter:v1.61.0-alpine
    container_name: redis-exporter
    environment:
      # op_admin has full access — needed for INFO, MONITOR, and key-count commands
      REDIS_ADDR: redis://redis:6379
      REDIS_USER: op_admin
      REDIS_PASSWORD_FILE: /data/init/redis_password_admin.txt
    volumes:
      - init-data:/data/init:ro
    networks:
      - oneplatform-internal
    depends_on:
      redis:
        condition: service_healthy
    restart: unless-stopped

  pgbouncer-exporter:
    image: prometheuscommunity/pgbouncer-exporter:v0.7.0
    container_name: pgbouncer-exporter
    environment:
      DATA_SOURCE_NAME: "postgresql://pgbouncer_stats:${PGBOUNCER_STATS_PASSWORD}@pgbouncer:5433/pgbouncer?sslmode=disable"
    networks:
      - oneplatform-internal
    depends_on:
      pgbouncer:
        condition: service_healthy
    restart: unless-stopped

  grafana:
    image: grafana/grafana:10.4.3
    container_name: grafana
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD:-changeme}"
      GF_USERS_ALLOW_SIGN_UP: "false"
    volumes:
      - grafana-data:/var/lib/grafana
    ports:
      - "127.0.0.1:3001:3000"
    networks:
      - oneplatform-internal
    depends_on:
      - prometheus
    restart: unless-stopped

volumes:
  prometheus-data:
  grafana-data:

services:
  bull-board:
    image: deadly0/bull-board:latest
    container_name: bull-board
    environment:
      REDIS_HOST: redis
      REDIS_PORT: "6379"
      REDIS_PASSWORD: "${OP_REDIS_PASSWORD_ADMIN}"
      REDIS_USER: op_admin
      # List the queue names to display
      QUEUE_PREFIX: "bull"
    ports:
      - "127.0.0.1:3002:3000"
    networks:
      - oneplatform-internal
    depends_on:
      redis:
        condition: service_healthy
    restart: unless-stopped

groups:
  - name: oneplatform.service_health
    interval: 30s
    rules:

      - alert: ServiceHealthCheckFailing
        # A Docker HEALTHCHECK failure means Docker has been unable to get a 200
        # from /healthz for retries × interval (50s). The container is in
        # "unhealthy" state and will be replaced by restart: unless-stopped.
        expr: |
          time() - container_last_seen{name=~".*-service"} > 120
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Service {{ $labels.name }} is not responding"
          description: "Container {{ $labels.name }} has not been seen for > 2 minutes."

      - alert: ContainerRestarting
        expr: |
          increase(container_start_time_seconds{name=~".*-service"}[5m]) > 2
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} is restarting"
          description: "Container {{ $labels.name }} has restarted more than twice in 5 minutes."

  - name: oneplatform.redis
    rules:

      - alert: RedisMemoryHigh
        # At 90% of the 256 MB limit, auth token eviction becomes likely.
        expr: |
          redis_memory_used_bytes / redis_config_maxmemory * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Redis memory usage is above 90%"
          description: "Redis is using {{ $value | humanize }}% of its 256 MB limit. Auth token eviction may occur."

      - alert: RedisMemoryCritical
        expr: |
          redis_memory_used_bytes / redis_config_maxmemory * 100 > 98
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Redis memory usage is critical (>98%)"
          description: "Redis will begin evicting keys immediately. Increase maxmemory or reduce key TTLs."

      - alert: RedisEvictingKeys
        expr: |
          increase(redis_evicted_keys_total[5m]) > 0
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: "Redis is evicting keys"
          description: "{{ $value }} keys evicted in the last 5 minutes. Check if auth:* keys are affected."

      - alert: RedisDown
        expr: |
          redis_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Redis is unreachable"
          description: "redis_exporter cannot reach Redis. All services will begin failing."

  - name: oneplatform.pgbouncer
    rules:

      - alert: PgBouncerPoolExhausted
        # cl_waiting > 0 means clients are queueing — latency will spike.
        expr: |
          pgbouncer_pools_client_waiting_connections > 0
        for: 30s
        labels:
          severity: warning
        annotations:
          summary: "PgBouncer pool exhausted for {{ $labels.database }}"
          description: "{{ $value }} clients are waiting for a connection in pool {{ $labels.database }}."

      - alert: PgBouncerHighWaitTime
        # maxwait > 5s means a client has been waiting 5 seconds for a connection.
        expr: |
          pgbouncer_pools_client_maxwait_seconds > 5
        for: 10s
        labels:
          severity: critical
        annotations:
          summary: "PgBouncer client waiting > 5s in pool {{ $labels.database }}"
          description: "Max client wait is {{ $value }}s. Increase pool_size for {{ $labels.database }} or scale horizontally."

      - alert: PgBouncerDown
        expr: |
          pgbouncer_up == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "PgBouncer is unreachable"
          description: "All database connections will fail. Services cannot process requests."

  - name: oneplatform.queues
    rules:

      - alert: QueueDepthGrowing
        # Waiting jobs growing for 10 minutes indicates workers can't keep up
        # or workers have crashed.
        expr: |
          increase(redis_key_size{key=~"bull:.*:wait"}[10m]) > 100
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "BullMQ queue {{ $labels.key }} depth growing"
          description: "Queue {{ $labels.key }} has grown by {{ $value }} jobs in 10 minutes."

      - alert: DLQNonEmpty
        # Any job reaching the DLQ means automatic retries are exhausted.
        # Manual intervention is required.
        expr: |
          redis_key_size{key=~"bull:.*:dlq:failed"} > 0
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: "Dead-letter queue {{ $labels.key }} has {{ $value }} jobs"
          description: "Jobs in DLQ require manual inspection. Check failed job details via Bull Board."

  - name: oneplatform.minio
    rules:

      - alert: MinIODiskSpaceLow
        expr: |
          minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes * 100 < 20
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "MinIO free disk space below 20%"
          description: "MinIO has {{ $value | humanize }}% free space. Ingestion uploads may fail."

      - alert: MinIODiskSpaceCritical
        expr: |
          minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes * 100 < 5
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "MinIO free disk space below 5%"
          description: "MinIO disk is nearly full. Uploads will fail immediately."

      - alert: MinIOErrorRateHigh
        expr: |
          rate(minio_s3_requests_errors_total[5m]) / rate(minio_s3_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "MinIO error rate exceeds 5%"
          description: "MinIO is returning errors on {{ $value | humanizePercentage }} of S3 requests."

rule_files:
  - /etc/prometheus/alerts.yml

global:
  resolve_timeout: 5m

route:
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: default

  routes:
    - match:
        severity: critical
      receiver: pagerduty
      continue: true

receivers:
  - name: default
    slack_configs:
      - api_url: "${SLACK_WEBHOOK_URL}"
        channel: '#oneplatform-alerts'
        text: |
          {{ range .Alerts }}
          *{{ .Annotations.summary }}*
          {{ .Annotations.description }}
          Severity: {{ .Labels.severity }}
          {{ end }}

  - name: pagerduty
    pagerduty_configs:
      - routing_key: "${PAGERDUTY_ROUTING_KEY}"
        description: "{{ .CommonAnnotations.summary }}"

apiVersion: 1
providers:
  - name: OnePlatform
    type: file
    disableDeletion: false
    updateIntervalSeconds: 30
    options:
      path: /var/lib/grafana/dashboards

services:
  grafana:
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro

apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false

services:
  jaeger:
    image: jaegertracing/all-in-one:1.57
    container_name: jaeger
    environment:
      COLLECTOR_OTLP_ENABLED: "true"
    ports:
      - "127.0.0.1:16686:16686"   # Jaeger UI
      - "127.0.0.1:4318:4318"     # OTLP/HTTP receiver
    networks:
      - oneplatform-internal
    restart: unless-stopped

  # Enable OTEL export on the logging service
  logging-service:
    environment:
      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
      OTEL_SERVICE_NAME: logging-service
      OTEL_RESOURCE_ATTRIBUTES: deployment.environment=production

services:
  zipkin:
    image: openzipkin/zipkin:3.3
    container_name: zipkin
    ports:
      - "127.0.0.1:9411:9411"
    networks:
      - oneplatform-internal
    restart: unless-stopped