Metric আর Prometheus — counter, gauge, histogram, RED আর USE

# shop.py: tiny API with hand-written Prometheus metrics (stdlib only)
import random, threading, time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

lock, rng = threading.Lock(), random.Random(42)
requests, latencies, users = {}, [], 0
in_flight = 0

def histogram(name, bounds):
    out = [f"# TYPE {name} histogram"]
    for le in bounds + ["+Inf"]:
        n = sum(1 for v in latencies if le == "+Inf" or v <= le)
        out.append(f'{name}_bucket{{le="{le}"}} {n}')
    return out + [f"{name}_sum {sum(latencies):.4f}", f"{name}_count {len(latencies)}"]

def summary(name):
    out, s = [f"# TYPE {name} summary"], sorted(latencies) or [0]
    for q in (0.5, 0.95, 0.99):
        out.append(f'{name}{{quantile="{q}"}} {s[int(q * (len(s) - 1))]:.4f}')
    return out + [f"{name}_sum {sum(latencies):.4f}", f"{name}_count {len(latencies)}"]

def render():
    out = ["# HELP shop_requests_total Orders handled.", "# TYPE shop_requests_total counter"]
    out += [f'shop_requests_total{{route="/order",status="{k}"}} {v}' for k, v in sorted(requests.items())]
    out += ["# HELP shop_in_flight Requests being served now.", "# TYPE shop_in_flight gauge"]
    out += [f"shop_in_flight {in_flight}"]
    out += histogram("shop_latency_fine_seconds", [.025, .05, .1, .15, .2, .3, .5, 1])
    out += histogram("shop_latency_coarse_seconds", [.1, .5, 1]) + summary("shop_latency_true_seconds")
    if users:  # the mistake: one series per user
        out += ["# TYPE shop_orders_by_user_total counter"]
        out += [f'shop_orders_by_user_total{{user_id="u{i}"}} 1' for i in range(users)]
    return "\n".join(out) + "\n"

class Handler(BaseHTTPRequestHandler):
    def log_message(self, *args): pass
    def reply(self, code, text, ctype="text/plain"):
        self.send_response(code); self.send_header("Content-Type", ctype); self.end_headers()
        self.wfile.write(text.encode())
    def do_GET(self):
        global in_flight, users
        if self.path == "/metrics":
            with lock: text = render()
            return self.reply(200, text, "text/plain; version=0.0.4; charset=utf-8")
        if self.path.startswith("/seed-users/"):
            users = int(self.path.rsplit("/", 1)[1]); return self.reply(200, "ok\n")
        with lock:
            in_flight += 1
            delay = rng.uniform(.02, .08) if rng.random() < .95 else rng.uniform(.11, .16)
            status = "500" if rng.random() < .02 else "200"
        start = time.perf_counter(); time.sleep(delay)
        with lock:
            in_flight -= 1; latencies.append(time.perf_counter() - start)
            requests[status] = requests.get(status, 0) + 1
        self.reply(int(status), "done\n")

ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever()

$ docker compose up -d
…
 Container abcl-mp-prom Started
 Container abcl-mp-app Started

$ curl -s 127.0.0.1:18632/metrics
# HELP shop_requests_total Orders handled.
# TYPE shop_requests_total counter
shop_requests_total{route="/order",status="200"} 694
shop_requests_total{route="/order",status="500"} 10
# HELP shop_in_flight Requests being served now.
# TYPE shop_in_flight gauge
shop_in_flight 2
# TYPE shop_latency_fine_seconds histogram
shop_latency_fine_seconds_bucket{le="0.025"} 56
shop_latency_fine_seconds_bucket{le="0.05"} 333
shop_latency_fine_seconds_bucket{le="0.1"} 664
…
shop_latency_fine_seconds_bucket{le="+Inf"} 704
shop_latency_fine_seconds_sum 38.5635
shop_latency_fine_seconds_count 704
…

# Turn raw latencies into Prometheus text format, by hand.
latencies = [0.031, 0.044, 0.052, 0.068, 0.073, 0.095, 0.121, 0.140, 0.260, 0.710]
bounds = [0.05, 0.1, 0.25, 0.5, float("inf")]

print("# TYPE shop_latency_seconds histogram")
for upper in bounds:
    count = sum(1 for v in latencies if v <= upper)
    le = "+Inf" if upper == float("inf") else upper
    print(f'shop_latency_seconds_bucket{{le="{le}"}} {count}')
print(f"shop_latency_seconds_sum {sum(latencies):.3f}")
print(f"shop_latency_seconds_count {len(latencies)}")

average = sum(latencies) / len(latencies)
print(f"\n# average = _sum / _count = {average:.4f} s")
ones = [sum(1 for v in latencies if v <= b) for b in bounds]
each = [b - a for a, b in zip([0] + ones, ones)]
print("# non-cumulative counts per bucket:", each)

$ curl -sG 127.0.0.1:18631/api/v1/query \
    --data-urlencode 'query=rate(shop_requests_total[1m])'
{status="200"}  13.948020022445089
{status="500"}  0.3028305751999572

$ … 'query=increase(shop_requests_total[1m])'
{status="200"}  836.8812013467053
{status="500"}  18.169834511997433

$ … 'query=sum(rate(shop_requests_total[1m]))'
{}  14.250850597645046

$ … 'query=rate(shop_requests_total{status="200"}[5s])'
(ফল নেই)

$ … 'query=rate(shop_in_flight)'
{"status":"error","errorType":"bad_data","error":"invalid parameter
\"query\": 1:6: parse error: expected type range vector in call to
function \"rate\", got instant vector"}

$ … 'query=shop_requests_total{status="200"}[1m]'
সময় (জানালা খোলার পর, s)   মান
 1.278    675
 6.277    744
11.278    814
16.277    884     <- restart এখানে
37.145    195
42.145    267
47.146    338
52.152    410
57.152    482

rate(...[1m])       12.367111715645917
increase(...[1m])   742.026702938755
delta(...[1m])      -207.25203135626586
resets(...[1m])     1

// rate() and increase() the Prometheus way, on samples from a real run.
// Each pair: [seconds after the window opened, counter value].
// The app restarted between 884 and 195. Try editing the numbers.
const RANGE = 60;
const samples = [[1.278, 675], [6.277, 744], [11.278, 814], [16.277, 884],
  [37.145, 195], [42.145, 267], [47.146, 338], [52.152, 410], [57.152, 482]];

function evaluate(s, range) {
  const first = s[0], last = s[s.length - 1];
  const naive = last[1] - first[1];
  let growth = naive;
  for (let i = 1; i < s.length; i++) {
    if (s[i][1] < s[i - 1][1]) {
      growth += s[i - 1][1];
      console.log(`reset: ${s[i - 1][1]} -> ${s[i][1]}, add back ${s[i - 1][1]}`);
    }
  }
  const covered = last[0] - first[0];
  const avgGap = covered / (s.length - 1);
  let toStart = first[0], toEnd = range - last[0];
  if (toStart >= avgGap * 1.1) toStart = avgGap / 2;
  if (toEnd >= avgGap * 1.1) toEnd = avgGap / 2;
  // A counter cannot go below zero, so do not extrapolate past that point.
  if (growth > 0) toStart = Math.min(toStart, covered * first[1] / growth);
  const factor = (covered + toStart + toEnd) / covered;
  return { naive, growth, factor, increase: growth * factor };
}

const r = evaluate(samples, RANGE);
console.log("last - first (wrong):", r.naive);
console.log("growth after reset fix:", r.growth);
console.log("extrapolation factor:", r.factor.toFixed(5));
console.log("increase[1m]:", r.increase.toFixed(3));
console.log("rate[1m]:", (r.increase / RANGE).toFixed(3), "per second");

$ … 'query=histogram_quantile(0.99, sum by (le) (shop_latency_coarse_seconds_bucket))'
{}  0.4262628571428567     # 3 bucket

$ … 'query=histogram_quantile(0.99, sum by (le) (shop_latency_fine_seconds_bucket))'
{}  0.16065853658536558    # 8 bucket

$ … 'query=shop_latency_true_seconds'
{quantile="0.5"}   0.051
{quantile="0.95"}  0.1149
{quantile="0.99"}  0.1516   # সব মাপ সাজিয়ে পাওয়া সত্য মান

$ curl -s 127.0.0.1:18632/metrics | grep -E 'shop_latency_fine'
…
shop_latency_fine_seconds_bucket{le="0.2"} 3258
…
shop_latency_fine_seconds_count 3258

// Histogram quantile by linear interpolation vs the true percentile.
const BOUNDS_LIST = [[0.1, 0.5, 1], [0.025, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5, 1]];
const Q = 0.99, N = 20000, TAIL_SHARE = 0.05, TAIL_FROM = 0.11, TAIL_TO = 0.16;

function mulberry32(a) {
  return () => { a |= 0; a = a + 0x6D2B79F5 | 0;
    let t = Math.imul(a ^ a >>> 15, 1 | a);
    t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
    return ((t ^ t >>> 14) >>> 0) / 4294967296; };
}
const rnd = mulberry32(42), values = [];
for (let i = 0; i < N; i++) {
  const [lo, hi] = rnd() < TAIL_SHARE ? [TAIL_FROM, TAIL_TO] : [0.02, 0.08];
  values.push(lo + (hi - lo) * rnd());
}
values.sort((a, b) => a - b);
const truth = values[Math.ceil(Q * N) - 1];

function quantile(bounds) {
  const rank = Q * N; let below = 0, lower = 0;
  for (const upper of bounds) {
    const count = values.filter(v => v <= upper).length;
    if (count >= rank) {
      const inBucket = count - below;
      return { lower, upper, inBucket, est: lower + (upper - lower) * (rank - below) / inBucket };
    }
    below = count; lower = upper;
  }
  return { lower, upper: Infinity, inBucket: N - below, est: lower };
}
console.log("true value:", (truth * 1000).toFixed(1), "ms");
for (const b of BOUNDS_LIST) {
  const r = quantile(b);
  console.log(`${b.length} buckets: (${r.lower}, ${r.upper}] holds ${r.inBucket}, estimate ${(r.est * 1000).toFixed(1)} ms, error ${((r.est / truth - 1) * 100).toFixed(1)}%`);
}

# Rate: requests per second
sum(rate(shop_requests_total[1m]))
{}  14.250850597645046

# Errors: share of requests that failed
sum(rate(shop_requests_total{status=~"5.."}[1m])) / sum(rate(shop_requests_total[1m]))
{}  0.021249999999999998

# Duration: p95 over the last minute
histogram_quantile(0.95, sum by (le) (rate(shop_latency_fine_seconds_bucket[1m])))
{}  0.11060606060606068

# CPU utilization (share of time not idle), per server
1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))

# CPU saturation: run queue compared with the core count
node_load1 / count without (cpu, mode) (node_cpu_seconds_total{mode="idle"})

# Disk utilization: share of time the device was busy
rate(node_disk_io_time_seconds_total[5m])

# Network errors per second
rate(node_network_receive_errs_total[5m])

$ docker exec abcl-mp-prom promtool check rules /etc/prometheus/rules.yml
Checking /etc/prometheus/rules.yml
  SUCCESS: 2 rules found

$ … 'query=job:shop_requests:rate1m'
{job="shop"}   14.2167119698066
$ … 'query=job:shop_errors:ratio1m'
{job="shop"}   0.017881705639614855

# /api/v1/rules, shortened
group shop_red: interval 5, evaluationTime 0.0003613

// Series count is the product of label value counts. Change the numbers.
const BASE = { route: 20, method: 3, status: 5, instance: 50 };
const BUCKETS = 10;               // histogram: buckets + +Inf + _sum + _count
const KB_PER_SERIES = 3;          // assumed; one small run measured about 2.3
const ADD = { user_id: 100000 };  // the label someone wants to add

const product = o => Object.values(o).reduce((a, b) => a * b, 1);
const show = (name, n) => {
  const gb = n * KB_PER_SERIES * 1000 / 1e9;
  console.log(name.padEnd(26), String(n).padStart(16), "series", gb.toFixed(2).padStart(12), "GB");
};
const seriesPerHistogram = BUCKETS + 3;
const counter = product(BASE);
show("counter", counter);
show("histogram", counter * seriesPerHistogram);
show("counter + user_id", counter * product(ADD));
show("histogram + user_id", counter * seriesPerHistogram * product(ADD));

// Worst case assumes every combination occurs. Real traffic fills a share.
const SEEN = 0.02;
show("hist + user_id, 2% seen", Math.round(counter * seriesPerHistogram * product(ADD) * SEEN));