cluster

Infrastructure files for Nordgedanken and Midnightthoughts.
git clone git://archive.git.mtrnord.blog/MTRNord/cluster.git
Log | Files | Refs | README

soak-with-memory-check.js (4132B)

      1 /**
      2  * Soak test with memory leak detection.
      3  *
      4  * Runs the same 15 rps steady-state load as soak.js. At setup() time, queries
      5  * Prometheus for a baseline of the connectivity-tester-stage container memory.
      6  * After the soak, teardown() re-queries and fails the `checks` threshold if
      7  * memory grew more than MEMORY_GROWTH_THRESHOLD (default 30%).
      8  *
      9  * Env vars:
     10  *   SOAK_RPS               — requests/sec (default: 15)
     11  *   SOAK_DURATION          — total test duration (default: 30m)
     12  *   PROMETHEUS_URL         — Prometheus HTTP API base URL
     13  *   MEMORY_GROWTH_THRESHOLD — fractional threshold, e.g. 0.30 = 30% (default: 0.30)
     14  */
     15 
     16 import http from 'k6/http';
     17 import { check } from 'k6';
     18 import { BASE_URL, SERVER_NAMES, randomItem } from './config.js';
     19 
     20 const SOAK_RPS                = parseInt(__ENV.SOAK_RPS || '15', 10);
     21 const SOAK_DURATION           = __ENV.SOAK_DURATION || '30m';
     22 const PROMETHEUS_URL          = __ENV.PROMETHEUS_URL || 'http://prometheus-operated.monitoring.svc.cluster.local:9090';
     23 const MEMORY_GROWTH_THRESHOLD = parseFloat(__ENV.MEMORY_GROWTH_THRESHOLD || '0.30');
     24 
     25 // cAdvisor metric — excludes page cache, reflects true resident memory
     26 const MEMORY_QUERY = 'avg(container_memory_working_set_bytes{namespace="matrix",pod=~"connectivity-tester-stage-.*",container="federation-tester-api"})';
     27 
     28 export const options = {
     29   scenarios: {
     30     soak: {
     31       executor: 'constant-arrival-rate',
     32       exec: 'federationOk',
     33       rate: SOAK_RPS,
     34       timeUnit: '1s',
     35       duration: SOAK_DURATION,
     36       // At 15 rps × ~1.5s avg = ~23 VUs needed; allocate headroom
     37       preAllocatedVUs: 40,
     38       maxVUs: 80,
     39     },
     40   },
     41   thresholds: {
     42     http_req_failed:   ['rate<0.05'],
     43     // If p95 climbs above 5s during a soak that was fine at 1.2s, something is leaking
     44     http_req_duration: ['p(95)<5000'],
     45     // Memory growth check — reported via teardown() check()
     46     checks: ['rate==1.0'],
     47   },
     48 };
     49 
     50 function queryMemoryBytes() {
     51   const url = `${PROMETHEUS_URL}/api/v1/query?query=${encodeURIComponent(MEMORY_QUERY)}`;
     52   const res = http.get(url, { timeout: '10s', tags: { name: 'prometheus_memory_query' } });
     53   if (res.status !== 200) {
     54     console.warn(`Prometheus query failed: HTTP ${res.status}`);
     55     return null;
     56   }
     57   try {
     58     const body = JSON.parse(res.body);
     59     if (body.status === 'success' && body.data.result.length > 0) {
     60       return parseFloat(body.data.result[0].value[1]);
     61     }
     62     console.warn('Prometheus returned no results for memory query');
     63   } catch (e) {
     64     console.warn(`Failed to parse Prometheus response: ${e}`);
     65   }
     66   return null;
     67 }
     68 
     69 export function setup() {
     70   const baseline = queryMemoryBytes();
     71   if (baseline !== null) {
     72     console.log(`Baseline memory: ${(baseline / 1024 / 1024).toFixed(1)} MiB`);
     73   } else {
     74     console.warn('Could not establish baseline memory; memory growth check will be skipped in teardown');
     75   }
     76   return { baselineMemory: baseline };
     77 }
     78 
     79 export function federationOk() {
     80   const server = randomItem(SERVER_NAMES);
     81   const res = http.get(
     82     `${BASE_URL}/api/federation/federation-ok?server_name=${encodeURIComponent(server)}`,
     83     { timeout: '15s' },
     84   );
     85   check(res, {
     86     'federation-ok: status 200':  (r) => r.status === 200,
     87     'federation-ok: GOOD or BAD': (r) =>
     88       r.body.trim() === 'GOOD' || r.body.trim() === 'BAD',
     89   });
     90 }
     91 
     92 export function teardown(data) {
     93   if (data.baselineMemory === null) {
     94     console.warn('Baseline memory was unavailable; skipping memory growth check');
     95     return;
     96   }
     97   const current = queryMemoryBytes();
     98   if (current === null) {
     99     console.warn('Post-soak memory unavailable; skipping memory growth check');
    100     return;
    101   }
    102 
    103   const growth     = (current - data.baselineMemory) / data.baselineMemory;
    104   const baseMiB    = (data.baselineMemory / 1024 / 1024).toFixed(1);
    105   const currentMiB = (current / 1024 / 1024).toFixed(1);
    106   console.log(`Memory: baseline=${baseMiB} MiB → current=${currentMiB} MiB (${(growth * 100).toFixed(1)}% growth)`);
    107 
    108   check(growth, {
    109     [`memory growth <= ${(MEMORY_GROWTH_THRESHOLD * 100).toFixed(0)}%`]: (g) => g <= MEMORY_GROWTH_THRESHOLD,
    110   });
    111 }