Configuration management — Ansible, Packer আর immutable infrastructure

[web]
web1 ansible_host=abcl-an-web1
web2 ansible_host=abcl-an-web2

[web:vars]
ansible_user=deploy
ansible_ssh_private_key_file=~/.ssh/abcl_demo

$ ansible-inventory --graph
@all:
  |--@ungrouped:
  |--@web:
  |  |--web1
  |  |--web2

$ ansible web -m ansible.builtin.ping
web1 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
web2 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}

$ ansible web -m ansible.builtin.command -a "cat /etc/alpine-release"
web2 | CHANGED | rc=0 >>
3.20.10
web1 | CHANGED | rc=0 >>
3.20.10

---
- name: Configure the web tier
  hosts: web
  become: true
  tasks:
    - name: Install nginx
      ansible.builtin.command:
        cmd: apk add --no-cache nginx
        creates: /usr/sbin/nginx

    - name: Web root exists
      ansible.builtin.file:
        path: /var/www/bazar
        state: directory
        mode: "0755"

    - name: Home page from template
      ansible.builtin.template:
        src: index.html.j2
        dest: /var/www/bazar/index.html
        mode: "0644"

    - name: Nginx config from template
      ansible.builtin.template:
        src: nginx.conf.j2
        dest: /etc/nginx/nginx.conf
        mode: "0644"
        validate: nginx -t -c %s
      notify: Reload nginx

    - name: Is nginx running?
      ansible.builtin.command: pgrep nginx
      register: nginx_proc
      changed_when: false
      failed_when: false
      check_mode: false

    - name: Start nginx if it is not running
      ansible.builtin.command: nginx
      when: nginx_proc.rc != 0

  handlers:
    - name: Reload nginx
      ansible.builtin.command: nginx -s reload

---
- name: Prepare app servers
  hosts: web
  become: true
  gather_facts: false
  tasks:
    - name: App user exists
      ansible.builtin.user:
        name: bazar
        shell: /sbin/nologin

    - name: App folder exists
      ansible.builtin.file:
        path: /opt/bazar
        state: directory
        owner: bazar
        mode: "0755"

    - name: Limit is in app.conf (idempotent)
      ansible.builtin.lineinfile:
        path: /opt/bazar/app.conf
        line: max_conn=100
        create: true
        mode: "0644"

    - name: Limit is appended to naive.conf (naive)
      ansible.builtin.shell: echo "max_conn=100" >> /opt/bazar/naive.conf

# First run
TASK [App user exists] *********************************************************
changed: [web2]
changed: [web1]
…
PLAY RECAP *********************************************************************
web1 : ok=4    changed=4    unreachable=0    failed=0    skipped=0    rescued=0 …
web2 : ok=4    changed=4    unreachable=0    failed=0    skipped=0    rescued=0 …

# Second run
TASK [Limit is in app.conf (idempotent)] ***************************************
ok: [web2]
ok: [web1]

TASK [Limit is appended to naive.conf (naive)] *********************************
changed: [web1]
changed: [web2]

PLAY RECAP *********************************************************************
web1 : ok=4    changed=1    unreachable=0    failed=0    skipped=0    rescued=0 …
web2 : ok=4    changed=1    unreachable=0    failed=0    skipped=0    rescued=0 …

# After the third run
$ ansible web1 -m ansible.builtin.command -a "wc -l /opt/bazar/app.conf /opt/bazar/naive.conf"
web1 | CHANGED | rc=0 >>
        1 /opt/bazar/app.conf
        3 /opt/bazar/naive.conf
        4 total

// One host, four tasks, repeated runs. Try RUNS = 6 or EDIT_BEFORE_RUN = 2.
const RUNS = 4;
const EDIT_BEFORE_RUN = 4;   // someone deletes the line by hand; 0 = nobody
const host = { users: [], dirs: [], appConf: [], naiveConf: [] };

const tasks = [
  ["user", h => h.users.includes("bazar") ? "ok"
                : (h.users.push("bazar"), "changed")],
  ["file", h => h.dirs.includes("/opt/bazar") ? "ok"
                : (h.dirs.push("/opt/bazar"), "changed")],
  ["lineinfile", h => h.appConf.includes("max_conn=100") ? "ok"
                : (h.appConf.push("max_conn=100"), "changed")],
  ["shell", h => (h.naiveConf.push("max_conn=100"), "changed")],
];

for (let run = 1; run <= RUNS; run++) {
  if (run === EDIT_BEFORE_RUN) host.appConf = [];
  const results = tasks.map(([name, act]) => name + "=" + act(host));
  const changed = results.filter(r => r.endsWith("=changed")).length;
  console.log("run " + run + ": ok=" + tasks.length + " changed=" + changed
    + " | app.conf lines=" + host.appConf.length
    + " naive.conf lines=" + host.naiveConf.length);
  console.log("   " + results.join("  "));
}

# group_vars/web.yml  (every host in group web)
listen_port: 8080
app_title: Bazar

# host_vars/web2.yml  (only web2, wins over the group)
app_title: Bazar (canary)

# templates/index.html.j2
<h1>{{ app_title }}</h1>
<p>{{ ansible_facts['hostname'] }} on {{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_version'] }}</p>

# templates/nginx.conf.j2
# Managed by Ansible. Manual edits will be overwritten.
worker_processes 1;
events { worker_connections 256; }
http {
    server {
        listen {{ listen_port }};
        root /var/www/bazar;
    }
}

$ ansible web -m ansible.builtin.setup -a "filter=ansible_distribution*"
web1 | SUCCESS => {
    "ansible_facts": {
        "ansible_distribution": "Alpine",
        …
        "ansible_distribution_version": "3.20.10"
    },
    "changed": false
}
…
$ ansible web -m ansible.builtin.command -a "wget -qO- http://127.0.0.1:8082/"
web1 | CHANGED | rc=0 >>
<h1>Bazar</h1>
<p>web1 on Alpine 3.20.10</p>
web2 | CHANGED | rc=0 >>
<h1>Bazar (canary)</h1>
<p>web2 on Alpine 3.20.10</p>

$ find roles -type f | sort
roles/web_site/defaults/main.yml
roles/web_site/handlers/main.yml
roles/web_site/tasks/main.yml
roles/web_site/templates/index.html.j2
roles/web_site/templates/nginx.conf.j2

$ cat site-role.yml
---
- name: Configure the web tier
  hosts: web
  become: true
  roles:
    - web_site

$ ansible-playbook site-role.yml
…
TASK [web_site : Nginx config from template] ***********************************
ok: [web1]
ok: [web2]
…
PLAY RECAP *********************************************************************
web1 : ok=6    changed=0    unreachable=0    failed=0    skipped=1    rescued=0 …
web2 : ok=6    changed=0    unreachable=0    failed=0    skipped=1    rescued=0 …

$ ansible-playbook site.yml --check --diff --limit web1
…
TASK [Nginx config from template] **********************************************
--- before: /etc/nginx/nginx.conf
+++ after: /root/.ansible/tmp/…/nginx.conf.j2
@@ -3,7 +3,7 @@
 events { worker_connections 256; }
 http {
     server {
-        listen 8080;
+        listen 8081;
         root /var/www/bazar;
     }
 }

changed: [web1]
…
RUNNING HANDLER [Reload nginx] *************************************************
skipping: [web1]

PLAY RECAP *********************************************************************
web1 : ok=6    changed=1    unreachable=0    failed=0    skipped=2    rescued=0 …

$ ansible web1 -m ansible.builtin.command -a "grep listen /etc/nginx/nginx.conf"
web1 | CHANGED | rc=0 >>
        listen 8080;

# The naive playbook in check mode: shell is not predicted, only skipped
TASK [Limit is appended to naive.conf (naive)] *********************************
skipping: [web2]
skipping: [web1]

---
- name: Roll the web tier one host at a time
  hosts: web
  become: true
  serial: 1
  max_fail_percentage: 0
  roles:
    - web_site
  post_tasks:
    - name: Health check on the new port
      ansible.builtin.uri:
        url: "http://127.0.0.1:{{ listen_port }}/"
      register: health
      until: health is succeeded
      retries: 5
      delay: 1

$ ansible-playbook rolling.yml -e listen_port=8082
PLAY [Roll the web tier one host at a time] ************************************
…
RUNNING HANDLER [web_site : Reload nginx] **************************************
changed: [web1]

TASK [Health check on the new port] ********************************************
ok: [web1]

PLAY [Roll the web tier one host at a time] ************************************
…
TASK [Health check on the new port] ********************************************
ok: [web2]

PLAY RECAP *********************************************************************
web1 : ok=8    changed=2    unreachable=0    failed=0    skipped=1    rescued=0 …
web2 : ok=8    changed=2    unreachable=0    failed=0    skipped=1    rescued=0 …

# Now a bad value
$ ansible-playbook rolling.yml -e listen_port=notaport
…
TASK [web_site : Nginx config from template] ***********************************
[ERROR]: Task failed: Module failed: failed to validate
…
fatal: [web1]: FAILED! => {… "msg": "failed to validate", "stderr": "nginx: [emerg]
host not found in \"notaport\" of the \"listen\" directive in …

NO MORE HOSTS LEFT *************************************************************

NO MORE HOSTS LEFT *************************************************************

PLAY RECAP *********************************************************************
web1 : ok=4    changed=0    unreachable=0    failed=1    skipped=0    rescued=0 …

$ ansible web -m ansible.builtin.command -a "grep listen /etc/nginx/nginx.conf"
web2 | CHANGED | rc=0 >>
        listen 8082;
web1 | CHANGED | rc=0 >>
        listen 8082;

// Rolling with serial and max_fail_percentage, using Ansible's two stop rules.
function mulberry32(a) {
  return () => { a |= 0; a = a + 0x6D2B79F5 | 0;
    let t = Math.imul(a ^ a >>> 15, 1 | a);
    t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
    return ((t ^ t >>> 14) >>> 0) / 4294967296; };
}
function roll(label, n, serial, maxFail, bad) {
  const size = typeof serial === "string"
    ? Math.max(1, Math.floor(parseInt(serial) / 100 * n)) : serial;
  let updated = 0, failed = 0, batch = 0, stopped = false;
  for (let first = 1; first <= n && !stopped; first += size) {
    const hosts = [];
    for (let h = first; h < first + size && h <= n; h++) hosts.push(h);
    const f = hosts.filter(h => bad.includes(h)).length;
    const rate = 100 * f / hosts.length;
    batch++; failed += f;
    stopped = f === hosts.length || (maxFail !== null && rate > maxFail);
    if (!stopped) updated += hosts.length - f;
  }
  const left = n - updated - failed;
  console.log(label + ": batches=" + batch + " updated=" + updated + " failed="
    + failed + " not finished=" + left + (stopped ? "  STOPPED" : "  done"));
}
roll("A serial 3, max 30, h2 bad ", 10, 3, 30, [2]);
roll("B serial 3, max 40, h2 h5  ", 10, 3, 40, [2, 5]);
roll("C serial 4, max 50, h1 h2  ", 10, 4, 50, [1, 2]);
roll("D serial 4, max 49, h1 h2  ", 10, 4, 49, [1, 2]);
roll("E serial 3, none, h1 h2 h3 ", 10, 3, null, [1, 2, 3]);
roll("F serial 25%, max 0, none  ", 10, "25%", 0, []);
const rnd = mulberry32(56), FAIL_RATE = 0.04, bad = [];
for (let h = 1; h <= 100; h++) if (rnd() < FAIL_RATE) bad.push(h);
console.log("seeded bad hosts: " + bad.join(" "));
roll("G 100 hosts, serial 10, max 10", 100, 10, 10, bad);

$ ansible-vault encrypt secrets.yml --vault-password-file ~/.vault_pass
$ head -n 3 secrets.yml
$ANSIBLE_VAULT;1.1;AES256
33346133643538323036373939643165346566393531653537336534393739333633636662623938
3336376164366230626237636161653362323030653461610a373561303365646162316138396531

$ ansible-playbook vault_demo.yml
[ERROR]: Error reading `vars_files` file 'secrets.yml': Attempting to decrypt but no vault secrets found.
…
$ ansible-playbook vault_demo.yml --vault-password-file ~/.vault_pass
TASK [Write the env file] ******************************************************
changed: [web2]
changed: [web1]
…
$ ansible web1 -b -m ansible.builtin.command -a 'stat -c "%a %U %n" /opt/bazar/db.env'
web1 | CHANGED | rc=0 >>
600 bazar /opt/bazar/db.env

# A hand edit on web2: one managed file, one unmanaged file
$ ansible web2 -b -m ansible.builtin.shell -a "sed -i 's/listen 8081/listen 8099/' /etc/nginx/nginx.conf; nginx -s reload; echo hotfix > /etc/hotfix-note"
web2 | CHANGED | rc=0 >>

$ ansible-playbook site.yml --diff
…
TASK [Nginx config from template] **********************************************
ok: [web1]
--- before: /etc/nginx/nginx.conf
+++ after: /root/.ansible/tmp/…/nginx.conf.j2
@@ -3,7 +3,7 @@
 events { worker_connections 256; }
 http {
     server {
-        listen 8099;
+        listen 8081;
         root /var/www/bazar;
     }
 }

changed: [web2]
…
RUNNING HANDLER [Reload nginx] *************************************************
changed: [web2]

PLAY RECAP *********************************************************************
web1 : ok=6    changed=0    unreachable=0    failed=0    skipped=1    rescued=0 …
web2 : ok=7    changed=2    unreachable=0    failed=0    skipped=1    rescued=0 …

$ ansible web -b -m ansible.builtin.shell -a "grep listen /etc/nginx/nginx.conf; ls /etc/hotfix-note"
…
web1 | FAILED | rc=1 >>
        listen 8081;ls: /etc/hotfix-note: No such file or directory…
web2 | CHANGED | rc=0 >>
        listen 8081;
/etc/hotfix-note

// Drift in a 20-server fleet over 90 simulated days. All rates are assumptions.
function mulberry32(a) {
  return () => { a |= 0; a = a + 0x6D2B79F5 | 0;
    let t = Math.imul(a ^ a >>> 15, 1 | a);
    t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
    return ((t ^ t >>> 14) >>> 0) / 4294967296; };
}
const SERVERS = 20, DAYS = 90, HAND_EDIT = 0.03;
const MANAGED = 0.4, CM_EVERY = 7, LIFETIME = 14;

function simulate(mode, seed) {
  const rnd = mulberry32(seed);
  let edits = Array.from({ length: SERVERS }, () => []);
  let nextId = 1;
  for (let day = 1; day <= DAYS; day++) {
    edits.forEach((list, s) => {
      if (rnd() < HAND_EDIT) list.push({ id: nextId++, managed: rnd() < MANAGED });
      if (mode === "immutable" && (day + s) % LIFETIME === 0) edits[s] = [];
    });
    if (mode === "ansible" && day % CM_EVERY === 0)
      edits = edits.map(list => list.filter(e => !e.managed));
    if (day % 30 === 0) {
      const drifted = edits.filter(l => l.length > 0).length;
      const total = edits.reduce((sum, l) => sum + l.length, 0);
      const shapes = new Set(edits.map(l => l.map(e => e.id).join(","))).size;
      console.log(mode.padEnd(10) + "day " + String(day).padStart(2)
        + ": drifted " + String(drifted).padStart(2) + "/" + SERVERS
        + ", edits alive " + String(total).padStart(2)
        + ", different servers " + shapes);
    }
  }
}
for (const mode of ["by hand", "ansible", "immutable"]) simulate(mode, 56);

packer {
  required_plugins {
    amazon = {
      source  = "github.com/hashicorp/amazon"
      version = "~> 1.3"
    }
    ansible = {
      source  = "github.com/hashicorp/ansible"
      version = "~> 1.1"
    }
  }
}

variable "image_version" {
  type = string
}

source "amazon-ebs" "web" {
  region        = "ap-southeast-1"
  instance_type = "t3.small"
  ssh_username  = "ubuntu"
  ami_name      = "bazar-web-${var.image_version}"

  source_ami_filter {
    filters = {
      name                = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"
      virtualization-type = "hvm"
    }
    owners      = ["123456789012"]   # sample id: use the publisher you trust
    most_recent = true
  }
}

build {
  sources = ["source.amazon-ebs.web"]

  provisioner "ansible" {
    playbook_file = "./site-role.yml"
    groups        = ["web"]   # the play says hosts: web
  }
}