D9.3 Prometheus με Ansible 🔥

# templates/prometheus.yml.j2
---
global:
  scrape_interval:     15s
  evaluation_interval: 15s
  external_labels:
    environment: "{{ app_env | default('production') }}"
    datacenter:  "{{ datacenter | default('dc1') }}"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - "alertmanager:9093"

rule_files:
  - "/etc/prometheus/rules/*.yml"

scrape_configs:

  # ── Prometheus itself ────────────────────
  - job_name: prometheus
    static_configs:
      - targets: ["localhost:9090"]

  # ── Node Exporters (όλοι οι managed hosts) ──
  - job_name: node_exporter
    static_configs:
      - targets:
{% for host in groups['all_managed'] %}
          - "{{ hostvars[host]['ansible_host'] | default(host) }}:9100"
{% endfor %}
    relabel_configs:
      - source_labels: [__address__]
        regex:         "([^:]+):.*"
        target_label:  instance
        replacement:   "$1"

  # ── cAdvisor (container metrics) ─────────
  - job_name: cadvisor
    static_configs:
      - targets:
{% for host in groups['all_managed'] %}
          - "{{ hostvars[host]['ansible_host'] | default(host) }}:8080"
{% endfor %}

tasks:

  # ── Config directory ──────────────────────
  - name: Prometheus directories
    ansible.builtin.file:
      path:  "{{ item }}"
      state: directory
      owner: "65534"    # ← nobody
      group: "65534"
      mode:  '0755'
    loop:
      - /etc/prometheus
      - /etc/prometheus/rules

  # ── Config από template ───────────────────
  - name: Prometheus config
    ansible.builtin.template:
      src:   templates/prometheus.yml.j2
      dest:  /etc/prometheus/prometheus.yml
      owner: "65534"
      group: "65534"
      mode:  '0644'
    notify: Reload Prometheus

  # ── Alert rules ───────────────────────────
  - name: Alert rules
    ansible.builtin.copy:
      src:   files/prometheus/rules/
      dest:  /etc/prometheus/rules/
      owner: "65534"
      mode:  '0644'
    notify: Reload Prometheus

  # ── Volume για data ───────────────────────
  - name: Prometheus data volume
    community.docker.docker_volume:
      name:  prometheus_data
      state: present
      labels:
        managed-by: ansible
        retention:  "30d"

  # ── Container ─────────────────────────────
  - name: "🔥 Prometheus container"
    community.docker.docker_container:
      name:            prometheus
      image:           "prom/prometheus:v2.49.0"
      state:           started
      restart_policy:  unless-stopped
      user:            "65534:65534"
      read_only:       true
      memory:          "1g"
      cap_drop:        [ALL]
      security_opts:   ["no-new-privileges:true"]
      tmpfs:
        /tmp: "size=64m"
      ports:
        - "127.0.0.1:9090:9090"    # ← localhost ΜΟΝΟ
      volumes:
        - "/etc/prometheus:/etc/prometheus:ro"
        - "prometheus_data:/prometheus:rw"
      command:
        - "--config.file=/etc/prometheus/prometheus.yml"
        - "--storage.tsdb.path=/prometheus"
        - "--storage.tsdb.retention.time=30d"
        - "--web.enable-lifecycle"          # ← reload χωρίς restart
        - "--web.enable-admin-api"
      networks:
        - name: monitoring_net
      labels:
        monitoring: self
        managed-by: ansible

handlers:

  - name: Reload Prometheus
    ansible.builtin.uri:
      url:    "http://localhost:9090/-/reload"
      method: POST
    ignore_errors: true

# files/prometheus/rules/alerts.yml
---
groups:
  - name: host_alerts
    rules:

      - alert: HighCPU
        expr:  >
          100 - (avg by(instance)
          (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for:   5m
        labels:
          severity: warning
        annotations:
          summary:     "High CPU on {{ $labels.instance }}"
          description: "CPU > 80% for 5 minutes: {{ $value | printf \"%.1f\" }}%"

      - alert: LowDisk
        expr:  >
          (node_filesystem_avail_bytes{mountpoint="/"} /
          node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
        for:   5m
        labels:
          severity: critical
        annotations:
          summary:     "Low disk on {{ $labels.instance }}"
          description: "Disk < 15% free: {{ $value | printf \"%.1f\" }}%"

      - alert: ContainerDown
        expr:  >
          absent(container_last_seen{name=~".+"}) or
          time() - container_last_seen{name=~".+"} > 60
        for:   1m
        labels:
          severity: critical
        annotations:
          summary: "Container down: {{ $labels.name }}"

      - alert: HighMemory
        expr:  >
          (container_memory_usage_bytes /
          container_spec_memory_limit_bytes) * 100 > 85
        for:   5m
        labels:
          severity: warning
        annotations:
          summary: "High memory in container {{ $labels.name }}"