Docker image — layer, Dockerfile আর build cache

FROM python:3.12-alpine
WORKDIR /app
ENV GREETING=salam
COPY requirements.txt .
RUN python -m pip install --no-index -r requirements.txt
COPY app.py .
EXPOSE 8000
CMD ["python", "app.py"]

$ docker build --progress=plain -t abcl-di-app:1 .
…
#4 [internal] load build context
#4 transferring context: 508B done
#4 DONE 0.1s

#5 [1/5] FROM docker.io/library/python:3.12-alpine@sha256:e6dae589c1c0…
#5 DONE 0.2s

#6 [2/5] WORKDIR /app
#6 DONE 0.0s

#7 [3/5] COPY requirements.txt .
#7 DONE 0.1s

#8 [4/5] RUN python -m pip install --no-index -r requirements.txt
#8 1.640 WARNING: Running pip as the 'root' user can result in broken …
#8 DONE 1.7s

#9 [5/5] COPY app.py .
#9 DONE 0.1s

#10 exporting to image
#10 exporting layers 0.4s done
…
#10 naming to docker.io/library/abcl-di-app:1 done
#10 DONE 0.7s

$ docker run -d --name abcl-di-c1 --hostname demo abcl-di-app:1
$ curl -s -m 3 http://127.0.0.1:18371/ ; echo "curl exit=$?"
curl exit=7
$ docker rm -f abcl-di-c1

$ docker run -d --name abcl-di-c2 --hostname demo \
    -p 127.0.0.1:18371:8000 abcl-di-app:1
$ curl -s http://127.0.0.1:18371/
salam from the shop app
$ docker port abcl-di-c2
8000/tcp -> 127.0.0.1:18371
$ docker rm -f abcl-di-c2

$ docker run -d --name abcl-di-c2 --hostname demo -e GREETING=adab \
    -p 127.0.0.1:18371:8000 abcl-di-app:1
$ curl -s http://127.0.0.1:18371/
adab from the shop app

$ docker history abcl-di-app:1
IMAGE          CREATED BY                                      SIZE
86d7dd680608   CMD ["python" "app.py"]                         0B
<missing>      EXPOSE [8000/tcp]                               0B
<missing>      COPY app.py . # buildkit                        12.3kB
<missing>      RUN /bin/sh -c python -m pip install --no-in…   9.66MB
<missing>      COPY requirements.txt . # buildkit              12.3kB
<missing>      ENV GREETING=salam                              0B
<missing>      WORKDIR /app                                    8.19kB
<missing>      CMD ["python3"]                                 0B
<missing>      RUN /bin/sh -c set -eux;  for src in idle3 p…   16.4kB
<missing>      RUN /bin/sh -c set -eux;   apk add --no-cach…   44.1MB
…
<missing>      RUN /bin/sh -c set -eux;  apk add --no-cache…   2.81MB
…
<missing>      CMD ["/bin/sh"]                                 0B
<missing>      ADD alpine-minirootfs-3.24.3-x86_64.tar.gz /…   9.08MB

$ docker image inspect --format '{{json .RootFS.Layers}}' abcl-di-app:1
["sha256:75c137aedfb2…",   # base image starts here
 "sha256:7bd6d027fc87…",
 "sha256:a26f58fa1b9b…",
 "sha256:6cf2f486e746…",   # base image ends here
 "sha256:1534a38a1c10…",   # WORKDIR
 "sha256:9b6d29dfe79e…",   # COPY requirements.txt
 "sha256:18a7dd77bfe4…",   # RUN pip
 "sha256:e88f27928769…"]   # COPY app.py

$ docker image ls abcl-di-app
IMAGE           ID             DISK USAGE   CONTENT SIZE   EXTRA
abcl-di-app:1   86d7dd680608       87.2MB         21.4MB

$ docker image inspect --format '{{.Size}}' abcl-di-app:1 python:3.12-alpine
21440783
18062363

# image abcl-di-cmd:1
FROM alpine:3.20
CMD ["echo", "from CMD"]

# image abcl-di-ep:1
FROM alpine:3.20
ENTRYPOINT ["echo", "from ENTRYPOINT:"]
CMD ["default arg"]

# image abcl-di-shellep:1
FROM alpine:3.20
ENTRYPOINT echo fixed text
CMD ["ignored"]

$ docker run --rm abcl-di-cmd:1
from CMD
$ docker run --rm abcl-di-cmd:1 echo replaced
replaced
$ docker run --rm abcl-di-cmd:1 hello
docker: Error response from daemon: … exec: "hello": executable file not found in $PATH
…
exit=127

$ docker run --rm abcl-di-ep:1
from ENTRYPOINT: default arg
$ docker run --rm abcl-di-ep:1 hello
from ENTRYPOINT: hello
$ docker run --rm --entrypoint echo abcl-di-ep:1 replaced entrypoint
replaced entrypoint

$ docker run --rm abcl-di-shellep:1 hello
fixed text

# abcl-di-sigshell:1  CMD python worker.py; echo finished
# abcl-di-sigexec:1   CMD ["python", "worker.py"]

$ docker exec abcl-di-shellform cat /proc/1/cmdline | tr '\0' ' '
/bin/sh -c python worker.py; echo finished
$ docker exec abcl-di-execform cat /proc/1/cmdline | tr '\0' ' '
python worker.py

$ docker stop -t 5 abcl-di-shellform     # took about 5.3 s in this run
$ docker stop -t 5 abcl-di-execform      # took about 0.3 s in this run

$ docker inspect --format '{{.Name}} exit={{.State.ExitCode}}' \
    abcl-di-shellform abcl-di-execform
/abcl-di-shellform exit=137
/abcl-di-execform exit=0

$ docker logs abcl-di-shellform
worker started
$ docker logs abcl-di-execform
worker started
got SIGTERM, saving work

$ docker exec abcl-di-cow1 sh -c 'echo hi > /tmp/new.txt; rm /etc/motd
    echo more >> /etc/hostname.demo'
$ docker diff abcl-di-cow1
C /tmp
A /tmp/new.txt
C /etc
A /etc/hostname.demo
D /etc/motd
$ docker ps -s --filter name=abcl-di-cow1 --format '{{.Names}} {{.Size}}'
abcl-di-cow1 20.5kB (virtual 29.5MB)

$ docker exec abcl-di-cow1 sh -c 'echo one-line >> /big.log'
$ docker diff abcl-di-cow1
…
C /big.log
…
$ docker ps -s --filter name=abcl-di-cow1 --format '{{.Names}} {{.Size}}'
abcl-di-cow1 21MB (virtual 50.5MB)

$ docker exec abcl-di-cow2 sh -c 'wc -c /big.log; cat /tmp/new.txt'
20971520 /big.log
cat: can't open '/tmp/new.txt': No such file or directory
$ docker ps -s --filter name=abcl-di-cow2 --format '{{.Names}} {{.Size}}'
abcl-di-cow2 4.1kB (virtual 29.5MB)

import gzip, hashlib, io, tarfile

def sha(data):
    return "sha256:" + hashlib.sha256(data).hexdigest()

def layer(files, mtime=0):
    """A layer is a tar of the changed files. Same bytes, same digest."""
    buf = io.BytesIO()
    with tarfile.open(fileobj=buf, mode="w", format=tarfile.USTAR_FORMAT) as tar:
        for name in sorted(files):
            body = files[name].encode()
            info = tarfile.TarInfo(name)
            info.size, info.mtime = len(body), mtime
            tar.addfile(info, io.BytesIO(body))
    return buf.getvalue()

def show(label, digest):
    print(f"{label:<26}{digest[7:19]}")

app = {"app/app.py": "print('salam')\n"}
one = layer(app)
show("app layer", sha(one))
show("same files, built again", sha(layer(app)))
show("one letter changed", sha(layer({"app/app.py": "print('Salam')\n"})))
show("same text, new file time", sha(layer(app, mtime=1)))
show("gzip of app layer", sha(gzip.compress(one, mtime=0)))

base_a = sha(layer({"etc/os": "alpine 3.20\n"}))
base_b = sha(layer({"etc/os": "alpine 3.24\n"}))
diff = sha(one)                            # the same top layer on two bases
show("top layer on base A", sha(f"{base_a} {diff}".encode()))
show("top layer on base B", sha(f"{base_b} {diff}".encode()))

#10 exporting manifest sha256:d3149cd3ccce377d…  done
#10 exporting config sha256:741582a63fdc9b79…  done
#10 exporting attestation manifest sha256:39a7bf7fd2a9f404…  done
#10 exporting manifest list sha256:86d7dd680608501…  done

$ docker build -q --provenance=false -t abcl-di-np:1 .
sha256:741582a63fdc9b79e7a05c33eac26fa77ff74d6f6c5d46efc51240a33f933a9c
$ docker build -q --provenance=false -t abcl-di-np:2 .
sha256:741582a63fdc9b79e7a05c33eac26fa77ff74d6f6c5d46efc51240a33f933a9c

$ docker build -q --no-cache --provenance=false -t abcl-di-np:3 .
sha256:d1d876031ff9df639a87b28e23ba94ecc5bed0226a30a9119862835e2d4dd8b1

$ docker pull python:3.13-alpine3.20
3.13-alpine3.20: Pulling from library/python
cb733b4064f2: Pulling fs layer
74269e1557ed: Pulling fs layer
0648f32ed48b: Pulling fs layer
…
Digest: sha256:40a4559d3d6b2117b1fbe426f17d55b9100fa40609733a1d0c3f39e2151d4b33
Status: Downloaded newer image for python:3.13-alpine3.20

$ docker image inspect --format '{{json .RootFS.Layers}}' python:3.12-alpine3.20
["sha256:994456c4fd7b…", "sha256:742e38c37a3d…", "sha256:7b7e15815a1d…", "sha256:dbb969ba5eaf…"]
$ docker image inspect --format '{{json .RootFS.Layers}}' python:3.13-alpine3.20
["sha256:994456c4fd7b…", "sha256:e1c020a6a204…", "sha256:20f7784406b1…", "sha256:8cfa70d20f39…"]

$ docker build --no-cache --progress=plain -t abcl-di-ctx:big .
…
#6 [internal] load build context
#6 transferring context: 60.83MB 4.0s done
…
$ docker run --rm abcl-di-ctx:big ls -a /app
.   ..   .env.sample   .git-demo   Dockerfile   app.py   data   requirements.txt

$ cat .dockerignore
.git-demo
data
.env*
Dockerfile
.dockerignore

$ docker build --no-cache --progress=plain -t abcl-di-ctx:small .
…
#4 [internal] load build context
#4 transferring context: 63B done
…
$ docker run --rm abcl-di-ctx:small ls -a /app
.   ..   app.py   requirements.txt

$ docker image inspect --format '{{.Size}}' abcl-di-ctx:big abcl-di-ctx:small
78890251
18053670

FROM python:3.12-alpine
WORKDIR /app
COPY . .
RUN python -m pip install --no-index -r requirements.txt
EXPOSE 8000
CMD ["python", "app.py"]

$ docker build --progress=plain -t abcl-di-bad:5 .     # only app.py changed
…
#6 [2/4] WORKDIR /app
#6 CACHED
#7 [3/4] COPY . .
#7 DONE 0.0s
#8 [4/4] RUN python -m pip install --no-index -r requirements.txt
#8 DONE 1.7s

FROM python:3.12-alpine
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install --no-index -r requirements.txt
COPY app.py .
EXPOSE 8000
CMD ["python", "app.py"]

$ docker build --progress=plain -t abcl-di-good:6 .    # only app.py changed
…
#6 [2/5] WORKDIR /app
#6 CACHED
#7 [3/5] COPY requirements.txt .
#7 CACHED
#8 [4/5] RUN python -m pip install --no-index -r requirements.txt
#8 CACHED
#9 [5/5] COPY app.py .
#9 DONE 0.0s

$ docker build --progress=plain -t abcl-di-good:7 .    # requirements.txt changed
…
#6 [2/5] WORKDIR /app
#6 CACHED
#7 [3/5] COPY requirements.txt .
#7 DONE 0.0s
#8 [4/5] RUN python -m pip install --no-index -r requirements.txt
#8 DONE 1.5s
#9 [5/5] COPY app.py .
#9 DONE 0.1s

$ touch app.py requirements.txt                        # new time, same bytes
$ docker build --progress=plain -t abcl-di-good:5 .
…
#7 [3/5] COPY requirements.txt .
#7 CACHED
#8 [4/5] RUN python -m pip install --no-index -r requirements.txt
#8 CACHED
#9 [5/5] COPY app.py .
#9 CACHED

// Build-cache simulator. Seconds are assumed. Reorder a list, then press Run.
const STEP = {                 // name: [input files, assumed seconds]
  'FROM python':     [[], 0], 'WORKDIR /app': [[], 0],
  'COPY reqs':       [['requirements.txt'], 1],
  'COPY app.py':     [['app.py'], 1],
  'COPY . .':        [['requirements.txt', 'app.py', 'README.md'], 1],
  'RUN pip install': [[], 90],
  'CMD python':      [[], 0],
};
const bad  = ['FROM python', 'WORKDIR /app', 'COPY . .', 'RUN pip install', 'CMD python'];
const good = ['FROM python', 'WORKDIR /app', 'COPY reqs', 'RUN pip install', 'COPY app.py', 'CMD python'];
function keys(order, version) {            // each key contains the key above it
  let above = '';
  return order.map(name => above += ' > ' + name + ' '
    + STEP[name][0].map(f => f + '=' + (version[f] || 1)));
}
function rebuild(title, order, changed) {
  const cache = new Set(keys(order, {}));  // the first build fills the cache
  const version = {};
  changed.forEach(f => version[f] = 2);
  let total = 0, broken = false;
  console.log('== ' + title + ' | changed: ' + (changed.join(', ') || 'nothing'));
  keys(order, version).forEach((key, i) => {
    const hit = cache.has(key), secs = hit ? 0 : STEP[order[i]][1];
    total += secs;
    const why = hit ? '' : broken ? '(a step above changed)' : '(input changed)';
    console.log(order[i].padEnd(16), hit ? 'CACHED' : 'RUN ' + secs + 's', why);
    broken = broken || !hit;
  });
  console.log('total: ' + total + 's');
}
rebuild('bad order', bad, ['app.py']);
rebuild('good order', good, ['app.py']);
rebuild('good order', good, ['requirements.txt']);
rebuild('good order', good, ['README.md']);

# Dockerfile.trap
FROM alpine:3.20
RUN dd if=/dev/urandom of=/big.bin bs=1M count=30
RUN rm /big.bin

# Dockerfile.oneline
FROM alpine:3.20
RUN dd if=/dev/urandom of=/big.bin bs=1M count=30 && rm /big.bin

$ docker history abcl-di-trap:1
IMAGE          CREATED BY                                      SIZE
c13ce53a6a88   RUN /bin/sh -c rm /big.bin # buildkit           4.1kB
<missing>      RUN /bin/sh -c dd if=/dev/urandom of=/big.bi…   31.5MB
…
$ docker history abcl-di-oneline:1
IMAGE          CREATED BY                                      SIZE
20ccf9fa62ea   RUN /bin/sh -c dd if=/dev/urandom of=/big.bi…   4.1kB
…
$ docker image inspect --format '{{.Size}}' abcl-di-trap:1 abcl-di-oneline:1
35099407
3631926
$ docker run --rm abcl-di-trap:1 ls -la /big.bin
ls: /big.bin: No such file or directory

// Layer-size calculator. Sizes in MB, uncompressed. Edit a plan and press Run.
const BASE = 8.47;                         // alpine:3.20, from a real run
const plans = {                            // one { } = one RUN = one layer
  'dd, then rm in the next RUN': [{ add: { 'big.bin': 31.5 } }, { del: ['big.bin'] }],
  'dd && rm in the same RUN':    [{ add: { 'big.bin': 31.5 }, del: ['big.bin'] }],
  'chmod in a later RUN':        [{ add: { 'data.db': 21 } }, { touch: ['data.db'] }],
  'install + clean, 2 RUNs':     [{ add: { 'tools': 120, 'cache': 60 } }, { del: ['cache'] }],
  'install + clean, 1 RUN':      [{ add: { 'tools': 120, 'cache': 60 }, del: ['cache'] }],
};
for (const [name, layers] of Object.entries(plans)) {
  const seen = {};                         // what a container can see
  let stored = BASE;                       // what the image must carry
  for (const layer of layers) {
    const here = { ...layer.add };
    for (const f of layer.touch || []) here[f] = seen[f];   // whole file again
    for (const f of layer.del || []) { delete here[f]; delete seen[f]; }
    Object.assign(seen, here);
    stored += Object.values(here).reduce((a, b) => a + b, 0);
  }
  const visible = BASE + Object.values(seen).reduce((a, b) => a + b, 0);
  console.log(name.padEnd(28), 'stored', stored.toFixed(2).padStart(7),
    '| visible', visible.toFixed(2).padStart(7),
    '| wasted', (stored - visible).toFixed(2).padStart(7));
}