D9.5 Alertmanager 🔔

Τι κάνει το Alertmanager

Το Alertmanager παίρνει alerts από το Prometheus και τα δρομολογεί σε email, Slack, PagerDuty κ.ά. — με deduplication, grouping και silencing.

Prometheus ──► Alertmanager ──► Email
                    │        ──► Slack
                    │        ──► PagerDuty
                    │
                    ├── Grouping   (1 notification αντί 100)
                    ├── Dedup      (ίδιο alert → 1 φορά)
                    ├── Silencing  (maintenance windows)
                    └── Inhibition (αν host down → suppresses app alerts)
  

alertmanager.yml config

# templates/alertmanager.yml.j2
---
global:
  smtp_smarthost:   "{{ smtp_host }}:587"
  smtp_from:        "alertmanager@{{ app_domain }}"
  smtp_auth_username: "{{ smtp_user }}"
  smtp_auth_password: "{{ vault_smtp_pass }}"
  smtp_require_tls: true
  resolve_timeout: 5m

route:
  group_by:       [alertname, instance]
  group_wait:     30s
  group_interval: 5m
  repeat_interval: 4h
  receiver:       default

  routes:
    # ── Critical → Slack + Email ─────────────
    - match:
        severity: critical
      receiver: critical
      group_wait: 10s
      repeat_interval: 1h

    # ── Warning → Email μόνο ─────────────────
    - match:
        severity: warning
      receiver: warning
      repeat_interval: 4h

receivers:

  - name: default
    email_configs:
      - to: "{{ admin_email }}"
        send_resolved: true

  - name: critical
    slack_configs:
      - api_url:    "{{ vault_slack_webhook }}"
        channel:    "#alerts-critical"
        title:      "🚨 {{ '{{' }} .GroupLabels.alertname {{ '}}' }}"
        text: |
          {{ '{{' }} range .Alerts {{ '}}' }}
          *Instance:* {{ '{{' }} .Labels.instance {{ '}}' }}
          *Description:* {{ '{{' }} .Annotations.description {{ '}}' }}
          {{ '{{' }} end {{ '}}' }}
        send_resolved: true
    email_configs:
      - to:          "{{ admin_email }}"
        send_resolved: true

  - name: warning
    email_configs:
      - to:          "{{ admin_email }}"
        send_resolved: true

inhibit_rules:
  # Αν host down → suppresses άλλα alerts από αυτόν
  - source_match:
      alertname: HostDown
    target_match_re:
      alertname: ".*"
    equal: [instance]
  

Deployment Alertmanager

tasks:

  - name: Alertmanager config directory
    ansible.builtin.file:
      path:  /etc/alertmanager
      state: directory
      owner: "65534"
      mode:  '0700'

  - name: Alertmanager config
    ansible.builtin.template:
      src:   templates/alertmanager.yml.j2
      dest:  /etc/alertmanager/alertmanager.yml
      owner: "65534"
      mode:  '0600'    # ← 600: SMTP password μέσα!
    notify: Reload Alertmanager
    no_log: true

  - name: Alertmanager data volume
    community.docker.docker_volume:
      name:  alertmanager_data
      state: present

  - name: "🔔 Alertmanager container"
    community.docker.docker_container:
      name:            alertmanager
      image:           "prom/alertmanager:v0.26.0"
      state:           started
      restart_policy:  unless-stopped
      user:            "65534:65534"
      read_only:       true
      memory:          "128m"
      cap_drop:        [ALL]
      security_opts:   ["no-new-privileges:true"]
      tmpfs:
        /tmp: "size=32m"
      ports:
        - "127.0.0.1:9093:9093"
      volumes:
        - "/etc/alertmanager:/etc/alertmanager:ro"
        - "alertmanager_data:/alertmanager:rw"
      command:
        - "--config.file=/etc/alertmanager/alertmanager.yml"
        - "--storage.path=/alertmanager"
        - "--web.listen-address=:9093"
      networks:
        - name: monitoring_net
      labels:
        monitoring: alertmanager
        managed-by: ansible

handlers:

  - name: Reload Alertmanager
    ansible.builtin.uri:
      url:    "http://localhost:9093/-/reload"
      method: POST
  

Silence — Maintenance windows

tasks:

  # ── Δημιουργία maintenance silence ────────
  - name: "🔇 Maintenance silence ({{ duration }})"
    ansible.builtin.uri:
      url:    "http://localhost:9093/api/v2/silences"
      method: POST
      body_format: json
      body:
        matchers:
          - name:    instance
            value:   "{{ inventory_hostname }}"
            isRegex: false
        startsAt: "{{ ansible_date_time.iso8601 }}"
        endsAt:   "{{ (ansible_date_time.epoch | int + (duration | default('2h') | regex_replace('h','') | int * 3600)) | strftime('%Y-%m-%dT%H:%M:%SZ') }}"
        comment:  "Ansible maintenance: {{ maintenance_reason | default('Planned maintenance') }}"
        createdBy: ansible
    when: create_silence | default(false)
  

Σύνοψη D9.5

Alertmanager
│
├── Routing:
│   ├── critical → Slack + Email (fast)
│   └── warning  → Email only (slower)
│
├── Features:
│   ├── Grouping   → 1 alert αντί πολλών
│   ├── Dedup      → ίδιο alert → 1 φορά
│   ├── Silencing  → maintenance windows
│   └── Inhibition → host down → suppress app alerts
│
└── Config: template με Ansible Vault secrets
    mode: 0600 (SMTP password!)
  