Terraform state — backend, lock, import আর drift

resource "kubernetes_config_map_v1" "app" {
  metadata {
    name      = "boighor-config"
    namespace = "abcl-ts"
  }
  data = {
    LOG_LEVEL = "info"
  }
}

resource "kubernetes_secret_v1" "db" {
  metadata {
    name      = "boighor-db"
    namespace = "abcl-ts"
  }
  data = {
    password = "fake-pass-1234"
  }
}

{
  "version": 4,
  "terraform_version": "1.17.1",
  "serial": 3,
  "lineage": "66e65cd8-cf8f-dc59-1527-03a976d8c15f",
  "outputs": {},
  "resources": [
    {
      "mode": "managed",
      "type": "kubernetes_secret_v1",
      "name": "db",
      "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]",
      "instances": [
        {
          "attributes": {
            "data": {
              "password": "fake-pass-1234"
            },
            "id": "abcl-ts/boighor-db",
            …
          },
          "sensitive_attributes": [ … ],
          …
        }
      ]
    },
    …
  ]
}

$ terraform state show kubernetes_secret_v1.db
# kubernetes_secret_v1.db:
resource "kubernetes_secret_v1" "db" {
    binary_data_wo                 = (write-only attribute)
    data                           = (sensitive value)
    data_wo                        = (write-only attribute)
    id                             = "abcl-ts/boighor-db"
    immutable                      = false
    type                           = "Opaque"
    …
}

# the config uses data_wo + data_wo_revision instead of data
$ grep -c 'fake-pass-5678' terraform.tfstate
0
$ grep -n '"data_wo\|"data"' terraform.tfstate
20:            "data": null,
21:            "data_wo": null,
22:            "data_wo_revision": 1,
51:                "value": "data"
$ kubectl get secret boighor-db2 -n abcl-ts \
    -o jsonpath='{.data.password}' | base64 -d
fake-pass-5678

terraform {
  backend "kubernetes" {
    secret_suffix  = "boighor"
    namespace      = "abcl-ts"
    config_path    = "~/.kube/config"
    config_context = "docker-desktop"
  }
}

$ terraform plan

Error: Backend initialization required, please run "terraform init"

Reason: Initial configuration of the requested backend "kubernetes"
…
$ terraform init -migrate-state -force-copy
Initializing the backend...

Successfully configured the backend "kubernetes"! Terraform will automatically
use this backend unless the backend configuration changes.
…
Terraform has been successfully initialized!

$ kubectl get secret,lease -n abcl-ts
NAME                             TYPE     DATA   AGE
secret/boighor-db                Opaque   1      119s
secret/tfstate-default-boighor   Opaque   1      0s

NAME                                                     HOLDER   AGE
lease.coordination.k8s.io/lock-tfstate-default-boighor            0s

$ kubectl get secret tfstate-default-boighor -n abcl-ts \
    -o jsonpath='{.metadata.annotations.encoding}'
gzip
$ kubectl get secret tfstate-default-boighor -n abcl-ts \
    -o jsonpath='{.data.tfstate}' | base64 -d | gzip -d \
    | grep -c 'fake-pass-1234'
1

const JOBS = [
  { who: "Rafi", start: 0, read: 2, write: 20, adds: "queue" },
  { who: "Mitu", start: 6, read: 8, write: 14, adds: "cache" },
];
function run(useLock, lockTimeout) {
  let state = { serial: 7, items: ["db"] };
  const real = ["db"], log = [];
  let holder = null;
  const live = JOBS.map((j) => ({ ...j, shift: 0, snap: null, done: false }));
  for (let t = 0; t <= 40; t++) {
    for (const j of live) {
      if (j.done || t < j.start) continue;
      if (useLock && holder !== j.who) {
        if (holder === null) { holder = j.who; j.shift = t - j.start; log.push(t + "s " + j.who + " takes the lock"); }
        else if (t - j.start >= lockTimeout) { j.done = true; log.push(t + "s " + j.who + " Error acquiring the state lock"); continue; }
        else continue;
      }
      const at = t - j.shift;
      if (at === j.read) { j.snap = { ...state, items: [...state.items] }; log.push(t + "s " + j.who + " reads serial " + j.snap.serial); }
      if (at === j.write) {
        real.push(j.adds);
        state = { serial: j.snap.serial + 1, items: [...j.snap.items, j.adds] };
        log.push(t + "s " + j.who + " creates " + j.adds + ", writes serial " + state.serial);
        j.done = true;
        if (holder === j.who) { holder = null; log.push(t + "s " + j.who + " frees the lock"); }
      }
    }
  }
  console.log(log.join("\n"));
  const orphans = real.filter((x) => !state.items.includes(x));
  console.log("state:", state.items.join(","), "| real:", real.join(","), "| orphans:", orphans.join(",") || "none");
}
console.log("--- no lock ---");            run(false, 0);
console.log("--- lock, no wait ---");      run(true, 0);
console.log("--- lock-timeout=60s ---");   run(true, 60);

$ terraform state list
kubernetes_config_map_v1.web_config
kubernetes_secret_v1.db

$ terraform state rm kubernetes_config_map_v1.web_config
Removed kubernetes_config_map_v1.web_config
Successfully removed 1 resource instance(s).

$ kubectl get cm boighor-config -n abcl-ts
NAME             DATA   AGE
boighor-config   1      23s

$ terraform apply -auto-approve
…
Plan: 1 to add, 0 to change, 0 to destroy.
kubernetes_config_map_v1.web_config: Creating...

Error: configmaps "boighor-config" already exists

moved {
  from = kubernetes_config_map_v1.app
  to   = kubernetes_config_map_v1.web_config
}

// Plan = compare configuration addresses with state addresses.
// Change `config` or `moved`, then run again.
const state = {
  "managed_db.main": { id: "db-0a1b", holdsData: true },
  "local_cfg.app": { id: "cfg-77", holdsData: false },
};
const config = ["managed_db.primary", "local_cfg.app"];   // main was renamed
const SCENARIOS = {
  "no moved block": [],
  "with moved block": [{ from: "managed_db.main", to: "managed_db.primary" }],
};
function plan(moved) {
  const st = structuredClone(state), lines = [];
  for (const m of moved) {
    if (st[m.from] && !st[m.to]) { st[m.to] = st[m.from]; delete st[m.from];
      lines.push("  ~ moved   " + m.from + " -> " + m.to + " (id " + st[m.to].id + " kept)"); }
    else if (st[m.from]) lines.push("  ! error   " + m.to + " already exists in state");
  }
  let add = 0, destroy = 0, lost = 0;
  for (const addr of config) if (!st[addr]) { add++; lines.push("  + create  " + addr + " (new id)"); }
  for (const addr of Object.keys(st)) if (!config.includes(addr)) {
    destroy++; if (st[addr].holdsData) lost++;
    lines.push("  - destroy " + addr + " (id " + st[addr].id + ")" + (st[addr].holdsData ? " DATA LOST" : ""));
  }
  lines.push("  Plan: " + add + " to add, " + destroy + " to destroy, " + lost + " data store(s) lost");
  return lines.join("\n");
}
for (const [name, moved] of Object.entries(SCENARIOS)) console.log(name + ":\n" + plan(moved));

# __generated__ by Terraform
# Please review these resources and move them into your main configuration files.

# __generated__ by Terraform from "abcl-ts/boighor-flags"
resource "kubernetes_config_map_v1" "flags" {
  binary_data = {}
  data = {
    MAX_CART     = "20"
    NEW_CHECKOUT = "off"
  }
  immutable = false
  metadata {
    annotations   = {}
    generate_name = null
    labels        = {}
    name          = "boighor-flags"
    namespace     = "abcl-ts"
  }
}

removed {
  from = kubernetes_config_map_v1.flags

  lifecycle {
    destroy = false
  }
}

$ terraform plan
…
Plan: 0 to add, 0 to change, 0 to destroy.

Warning: Some objects will no longer be managed by Terraform

If you apply this plan, Terraform will discard its tracking information for
the following objects, but it will not delete them:
 - kubernetes_config_map_v1.flags
…
$ terraform apply -auto-approve
…
Apply complete! Resources: 0 added, 0 changed, 0 destroyed.
$ kubectl get cm boighor-flags -n abcl-ts
NAME            DATA   AGE
boighor-flags   2      49s

$ kubectl patch configmap boighor-config -n abcl-ts --type merge \
    -p '{"data":{"LOG_LEVEL":"debug"}}'
configmap/boighor-config patched

$ terraform plan -refresh-only
…
Note: Objects have changed outside of Terraform

Terraform detected the following changes made outside of Terraform since the
last "terraform apply" which may have affected this plan:

  # kubernetes_config_map_v1.app has changed
  ~ resource "kubernetes_config_map_v1" "app" {
      ~ data        = {
          ~ "LOG_LEVEL" = "info" -> "debug"
        }
        id          = "abcl-ts/boighor-config"
        …
    }
…
This is a refresh-only plan, so Terraform will not take any actions to undo
these. If you were expecting these changes then you can apply this plan to
record the updated values in the Terraform state without changing any remote
objects.

$ terraform plan -detailed-exitcode
…
  ~ resource "kubernetes_config_map_v1" "app" {
      ~ data        = {
          ~ "LOG_LEVEL" = "debug" -> "info"
        }
…
Plan: 0 to add, 1 to change, 0 to destroy.
$ echo $?
2

$ terraform apply -refresh-only -auto-approve
…
Apply complete! Resources: 0 added, 0 changed, 0 destroyed.
$ terraform plan
…
Plan: 0 to add, 1 to change, 0 to destroy.

$ kubectl get lease -n abcl-ts
NAME                           HOLDER                                 AGE
lock-tfstate-default-boighor   fd708136-ffdd-8202-d9da-6ff710b0bb0f   87s
$ kubectl get job boighor-reindex -n abcl-ts
NAME              STATUS    COMPLETIONS   DURATION   AGE
boighor-reindex   Running   0/1           7s         7s

$ terraform plan
Error: Error acquiring the state lock
…
$ terraform force-unlock -force 11111111-2222-3333-4444-555555555555
Failed to unlock state: lock id "11111111-2222-3333-4444-555555555555" does not match existing lock
…
$ terraform force-unlock -force fd708136-ffdd-8202-d9da-6ff710b0bb0f
Terraform state has been successfully unlocked!
…
$ terraform plan
  # kubernetes_job_v1.reindex will be created
Plan: 1 to add, 0 to change, 0 to destroy.
$ terraform apply -auto-approve
…
Error: Failed to create Job! API error: jobs.batch "boighor-reindex" already exists

$ terraform import kubernetes_job_v1.reindex abcl-ts/boighor-reindex
…
Import successful!

$ terraform plan
  # kubernetes_config_map_v1.web_config will be created
  # kubernetes_secret_v1.db will be created
Plan: 2 to add, 0 to change, 0 to destroy.

$ terraform apply -auto-approve
…
Error: configmaps "boighor-config" already exists
Error: secrets "boighor-db" already exists

# recover.tf has one import block for each resource
$ terraform plan
…
Plan: 2 to import, 0 to add, 1 to change, 0 to destroy.
$ terraform apply -auto-approve
…
Apply complete! Resources: 2 imported, 0 added, 1 changed, 0 destroyed.
$ terraform plan
No changes. Your infrastructure matches the configuration.

$ terraform state push older.json
Failed to write state: cannot import state with serial 17 over newer state with serial 18
$ terraform state push other.json
Failed to write state: cannot import state with lineage "00000000-1111-2222-3333-444444444444" over unrelated state with lineage "66e65cd8-cf8f-dc59-1527-03a976d8c15f"

// Assumed numbers, not measurements. Change the stacks and run again.
const REFRESH_SEC = 0.5;     // one read call for one resource
const PARALLEL = 10;         // terraform's default -parallelism
const APPLY_MIN = 6;         // minutes the lock is held per apply
const stacks = [
  { name: "network", resources: 60, changesPerDay: 0.2 },
  { name: "database", resources: 40, changesPerDay: 0.5 },
  { name: "platform", resources: 200, changesPerDay: 2 },
  { name: "apps", resources: 700, changesPerDay: 12 },
];
const refreshSec = (n) => Math.ceil(n / PARALLEL) * REFRESH_SEC;
function report(name, n, changes, share) {
  const busyMin = changes * APPLY_MIN;
  console.log(name.padEnd(10), String(n).padStart(5), (refreshSec(n).toFixed(1) + "s").padStart(8),
    String(busyMin.toFixed(0)).padStart(6) + "m", (share * 100).toFixed(1).padStart(6) + "%");
}
const total = stacks.reduce((a, s) => a + s.resources, 0);
const allChanges = stacks.reduce((a, s) => a + s.changesPerDay, 0);
console.log("stack      items  refresh  locked  blast");
console.log("-- one state --");
report("all", total, allChanges, 1);
console.log("-- split by blast radius --");
for (const s of stacks) report(s.name, s.resources, s.changesPerDay, s.resources / total);
const weighted = stacks.reduce((a, s) => a + s.changesPerDay * s.resources, 0) / allChanges;
console.log("avg items at risk per apply:", total, "->", Math.round(weighted));
console.log("avg refresh per apply:", refreshSec(total).toFixed(1) + "s ->",
  (stacks.reduce((a, s) => a + s.changesPerDay * refreshSec(s.resources), 0) / allChanges).toFixed(1) + "s");

data "terraform_remote_state" "app" {
  backend = "kubernetes"
  config = {
    secret_suffix  = "boighor"
    namespace      = "abcl-ts"
    config_path    = "~/.kube/config"
    config_context = "docker-desktop"
  }
}

data "kubernetes_config_map_v1" "app" {
  metadata {
    name      = "boighor-config"
    namespace = "abcl-ts"
  }
}

output "from_remote_state" {
  value = data.terraform_remote_state.app.outputs.config_map_name
}

output "from_data_source" {
  value = data.kubernetes_config_map_v1.app.data["LOG_LEVEL"]
}

$ terraform apply -auto-approve
…
Apply complete! Resources: 0 added, 0 changed, 0 destroyed.

Outputs:

from_data_source = "info"
from_remote_state = "boighor-config"