Docker Compose — একাধিক service, environment আর local stack

docker network create shop-front
docker network create shop-back
docker volume create shop-redis-data
docker build -t shop-api ./app
docker build -t shop-worker ./app

docker run -d --name redis --network shop-back \
  -v shop-redis-data:/data -e REDIS_PASSWORD=... redis:7.4-alpine ...
# now wait by hand until redis answers, then:
docker run -d --name api --network shop-back \
  -e REDIS_HOST=redis -e REDIS_PASSWORD=... shop-api
docker network connect shop-front api
docker run -d --name worker --network shop-back \
  -e REDIS_HOST=redis -e REDIS_PASSWORD=... shop-worker \
  python -u worker.py
# wait again, then:
docker run -d --name proxy --network shop-front \
  -p 127.0.0.1:8080:80 -v "$PWD/proxy/nginx.conf:/etc/nginx/conf.d/default.conf:ro" \
  nginx:1.27-alpine

# compose.yaml: the fictional bookshop "Boighor"
name: abcl-dc

services:
  proxy:
    image: nginx:1.27-alpine
    ports:
      - "127.0.0.1:${PROXY_PORT:-18401}:80"
    volumes:
      - ./proxy/nginx.conf:/etc/nginx/conf.d/default.conf:ro
    networks: [front]
    depends_on:
      api:
        condition: service_healthy
    restart: unless-stopped

  api:
    build: ./app
    environment:
      REDIS_HOST: redis
      REDIS_PASSWORD: ${REDIS_PASSWORD:?put REDIS_PASSWORD in .env}
      LOG_LEVEL: ${LOG_LEVEL:-info}
    networks: [front, back]
    depends_on:
      redis:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8000/healthz"]
      interval: 2s
      timeout: 2s
      retries: 5

  worker:
    build: ./app
    command: ["python", "-u", "worker.py"]
    environment:
      REDIS_HOST: redis
      REDIS_PASSWORD: ${REDIS_PASSWORD:?put REDIS_PASSWORD in .env}
    networks: [back]
    depends_on:
      redis:
        condition: service_healthy

  redis:
    image: redis:7.4-alpine
    user: redis
    # sleep 6 stands in for the slow start of a real database
    command: ["sh", "-c", "sleep 6 && exec redis-server --appendonly yes --requirepass \"$$REDIS_PASSWORD\""]
    environment:
      REDIS_PASSWORD: ${REDIS_PASSWORD:?put REDIS_PASSWORD in .env}
    volumes:
      - redis-data:/data
    networks: [back]
    healthcheck:
      test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG"]
      interval: 2s
      timeout: 2s
      retries: 5
      start_period: 10s

  probe:
    image: alpine:3.20
    profiles: [test]
    command: ["wget", "-q", "-O", "-", "http://proxy/books"]
    networks: [front]

volumes:
  redis-data:

networks:
  front:
  back:

FROM python:3.12-alpine
WORKDIR /app
COPY *.py .
ENV LOG_LEVEL=image-default
USER nobody
CMD ["python", "-u", "api.py"]

$ docker compose --progress plain up -d --build
 Image abcl-dc-worker Built
 Image abcl-dc-api Built
 Network abcl-dc_front Created
 Network abcl-dc_back Created
 Volume abcl-dc_redis-data Created
 Container abcl-dc-redis-1 Created
 Container abcl-dc-worker-1 Created
 Container abcl-dc-api-1 Created
 Container abcl-dc-proxy-1 Created
 Container abcl-dc-redis-1 Started
 Container abcl-dc-redis-1 Waiting
 Container abcl-dc-redis-1 Healthy
 Container abcl-dc-worker-1 Started
 Container abcl-dc-api-1 Started
 Container abcl-dc-api-1 Waiting
 Container abcl-dc-api-1 Healthy
 Container abcl-dc-proxy-1 Started

$ docker compose ps
NAME               IMAGE               COMMAND                  SERVICE   …  STATUS                  PORTS
abcl-dc-api-1      abcl-dc-api         "python -u api.py"       api       …  Up 3 seconds (healthy)
abcl-dc-proxy-1    nginx:1.27-alpine   "/docker-entrypoint.…"   proxy     …  Up Less than a second   127.0.0.1:18401->80/tcp
abcl-dc-redis-1    redis:7.4-alpine    "docker-entrypoint.s…"   redis     …  Up … (healthy)          6379/tcp
abcl-dc-worker-1   abcl-dc-worker      "python -u worker.py"    worker    …  Up 3 seconds

$ curl -s http://127.0.0.1:18401/books
{"served_by": "462c93b441d9", "orders_done": 0}

$ curl -s -X POST http://127.0.0.1:18401/orders
{"queued": "order-1"}
…(two more orders)…

$ docker compose logs --no-color worker
worker-1  | worker started, waiting for orders
worker-1  | packed order-1 total_done=1
worker-1  | packed order-2 total_done=2
worker-1  | packed order-3 total_done=3

$ docker compose exec api sh -c 'id -un; printenv LOG_LEVEL'
nobody
info

$ docker compose exec proxy sh -c \
    'getent hosts redis || echo "redis: name not found"'
redis: name not found

$ docker compose exec proxy sh -c \
    'getent hosts worker || echo "worker: name not found"'
worker: name not found

$ docker compose exec worker sh -c \
    'getent hosts proxy || echo "proxy: not found"'
proxy: not found

// Two containers can talk only if they share at least one network
const stack = {
  proxy:  { nets: ["front"],         host: "127.0.0.1:18401->80" },
  api:    { nets: ["front", "back"], host: null },
  worker: { nets: ["back"],          host: null },
  redis:  { nets: ["back"],          host: null },
};

function shared(a, b) {
  return stack[a].nets.filter(n => stack[b].nets.includes(n));
}

const names = Object.keys(stack);
console.log("from/to".padEnd(11) + names.map(n => n.padEnd(8)).join(""));
for (const a of names) {
  const cells = names.map(b => (a === b ? "-" : shared(a, b).length ? "yes" : "no"));
  console.log(a.padEnd(11) + cells.map(c => c.padEnd(8)).join(""));
}

console.log("\nshared network for each allowed pair:");
names.forEach((a, i) => names.slice(i + 1).forEach(b => {
  const s = shared(a, b);
  if (s.length) console.log(" ", a, "<->", b, "via", s.join(", "));
}));

console.log("\nfrom the host machine:");
for (const n of names) {
  console.log(" ", n.padEnd(7), stack[n].host ? "open at " + stack[n].host : "no published port");
}

# api depends on cache, cache on db, db on api
$ docker compose config -q
dependency cycle detected: api -> cache -> db -> api
$ echo $?
1

// Which containers may start, wave by wave, from the depends_on graph
const stacks = {
  bookshop: { proxy: ["api"], api: ["redis"], worker: ["redis"], redis: [] },
  cycle:    { api: ["cache"], cache: ["db"], db: ["api"] },
};

function findCycle(g, left) {
  const path = [];
  let s = [...left].sort()[0];
  while (!path.includes(s)) {
    path.push(s);
    s = g[s].find(d => left.has(d));
  }
  return [...path.slice(path.indexOf(s)), s].join(" -> ");
}

function resolve(g) {
  const left = new Set(Object.keys(g)), done = new Set(), waves = [];
  while (left.size) {
    const ready = [...left].filter(s => g[s].every(d => done.has(d))).sort();
    if (!ready.length) return { error: "dependency cycle detected: " + findCycle(g, left) };
    ready.forEach(s => { left.delete(s); done.add(s); });
    waves.push(ready);
  }
  return { waves };
}

for (const [name, g] of Object.entries(stacks)) {
  const r = resolve(g);
  console.log("== " + name);
  if (r.error) { console.log(r.error); continue; }
  r.waves.forEach((w, i) => console.log("start wave " + (i + 1) + ":", w.join(", ")));
  console.log("stop order: " + r.waves.slice().reverse().map(w => w.join(", ")).join(" | "));
}

$ docker inspect abcl-dc-redis-1 \
    --format '{{.State.Health.Status}} streak={{.State.Health.FailingStreak}}'
unhealthy streak=5

$ docker compose ps -a
NAME               STATUS
abcl-dc-api-1      Created
abcl-dc-proxy-1    Created
abcl-dc-redis-1    Up 19 seconds (unhealthy)
abcl-dc-worker-1   Created

$ cat .env.example        # committed: names only, no real values
REDIS_PASSWORD=
PROXY_PORT=18401

$ cat .gitignore
.env

$ mv .env .env.off; docker compose config -q; echo "exit=$?"
error while interpolating services.api.environment.REDIS_PASSWORD: required variable REDIS_PASSWORD is missing a value: put REDIS_PASSWORD in .env
exit=1

# .env contains ONLY_IN_DOTENV=hello, compose.yaml never mentions it
$ docker compose exec api sh -c \
    'printenv ONLY_IN_DOTENV || echo "ONLY_IN_DOTENV is not set"'
ONLY_IN_DOTENV is not set

// Step 1 fills ${VAR} in the file. Step 2 builds the container's environment.
const image   = { LVL: "image", FROM_ENVIRONMENT: "image", ONLY_FILE: "image", ONLY_IMAGE: "image" };
const envFile = { LVL: "env_file", FROM_ENVIRONMENT: "env_file", ONLY_FILE: "env_file" };
const envAttr = { FROM_ENVIRONMENT: "${LVL:-compose-default}" };

const cases = {
  "1 .env only":   { shell: {}, dotenv: { LVL: "dotenv" }, runE: {} },
  "2 shell LVL":   { shell: { LVL: "shell" }, dotenv: { LVL: "dotenv" }, runE: {} },
  "3 no .env":     { shell: {}, dotenv: {}, runE: {} },
  "4 run -e":      { shell: {}, dotenv: { LVL: "dotenv" },
                     runE: { LVL: "cli", FROM_ENVIRONMENT: "cli" } },
  "5 --env-file":  { shell: {}, dotenv: { LVL: "alt" }, runE: {} }, // replaces .env
};

function interpolate(text, c) {                   // shell beats .env beats :-default
  return text.replace(/\$\{(\w+):-([^}]*)\}/, (_, name, dflt) =>
    c.shell[name] ?? c.dotenv[name] ?? dflt);
}

function containerEnv(c) {                        // later layers win
  const fromAttr = {};
  for (const k in envAttr) fromAttr[k] = interpolate(envAttr[k], c);
  return { ...image, ...envFile, ...fromAttr, ...c.runE };
}

for (const [name, c] of Object.entries(cases)) {
  const e = containerEnv(c);
  console.log(name.padEnd(13), "LVL=" + e.LVL.padEnd(9),
    "FROM_ENVIRONMENT=" + e.FROM_ENVIRONMENT.padEnd(16), "ONLY_IMAGE=" + e.ONLY_IMAGE);
}

$ docker compose config --no-path-resolution api
…
    environment:
      …
      REDIS_PASSWORD: demo-only-not-a-real-secret
…

# compose.yaml
services:
  a:
    image: alpine:3.20
    secrets:
      - redis_password
secrets:
  redis_password:
    file: ./redis_password.txt

# inside the container (real run)
$ ls /run/secrets
redis_password
$ printenv | grep -c demo-only
0

$ docker compose config --services
redis
worker
api
proxy

$ docker compose --profile test config --services
redis
worker
api
probe
proxy

$ docker compose run --rm probe
{"served_by": "462c93b441d9", "orders_done": 3}

$ docker compose config --no-path-resolution api
name: abcl-dc
services:
  api:
    build:
      context: ./app
      dockerfile: Dockerfile
    command:
      - python
      - -u
      - api.py
    depends_on:
      redis:
        condition: service_healthy
        required: true
    environment:
      LOG_LEVEL: debug
      REDIS_HOST: redis
      REDIS_PASSWORD: demo-only-not-a-real-secret
…
    ports:
      - mode: ingress
        host_ip: 127.0.0.1
        target: 8000
        published: "18402"
        protocol: tcp
    volumes:
      - type: bind
        source: ./app
        target: /app
        read_only: true
…

$ docker compose -f compose.yaml -f compose.ci.yaml config … api
…
      LOG_LEVEL: warn
…

upstream api_pool {
    zone api_pool 64k;        # one round-robin counter for all nginx workers
    server api:8000;          # name is resolved when nginx loads this file
}
server {
    listen 80;
    location / {
        proxy_pass http://api_pool;
        proxy_connect_timeout 2s;
    }
}

$ docker compose --progress plain up -d --scale api=3
 Container abcl-dc-api-3 Created
 Container abcl-dc-api-2 Created
 …
 Container abcl-dc-api-2 Healthy
 Container abcl-dc-api-3 Healthy

$ for i in 1 2 3 4 5 6; do curl -s http://127.0.0.1:18401/books; done
{"served_by": "462c93b441d9", "orders_done": 3}
{"served_by": "462c93b441d9", "orders_done": 3}
…(all six the same)…

$ docker compose exec proxy nginx -s reload
…

$ for i in 1 2 3 4 5 6; do curl -s http://127.0.0.1:18401/books; done
{"served_by": "462c93b441d9", "orders_done": 3}
{"served_by": "653033f77df5", "orders_done": 3}
{"served_by": "4399686adfdf", "orders_done": 3}
{"served_by": "462c93b441d9", "orders_done": 3}
{"served_by": "653033f77df5", "orders_done": 3}
{"served_by": "4399686adfdf", "orders_done": 3}

$ docker stop abcl-dc-api-2
$ for i in 1 2 3 4 5 6; do
    curl -s -o /dev/null -w '%{http_code} %{time_total}\n' http://127.0.0.1:18401/books
  done
200 0.003040
200 0.003320
200 2.005331
200 0.003158
200 0.003002
200 0.003243