cluster

Infrastructure files for Nordgedanken and Midnightthoughts.
git clone git://archive.git.mtrnord.blog/MTRNord/cluster.git
Log | Files | Refs | README

commit df55da1c9a257f1d3b591a19b598d3177f53cb11
parent 5450f14d9386eb381234a6aaf2e7db159466c95d
Author: MTRNord <MTRNord@users.noreply.github.com>
Date:   Thu, 26 Feb 2026 11:38:06 +0100

try to deploy opencost

Signed-off-by: MTRNord <MTRNord@users.noreply.github.com>

Diffstat:
Mapps/talos_cluster/kustomization.yaml | 1+
Mapps/talos_cluster/monitoring-stack/dashboards/kustomization.yaml | 10++++++++++
Aapps/talos_cluster/monitoring-stack/dashboards/opencost-namespace.json | 1156++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Aapps/talos_cluster/monitoring-stack/dashboards/opencost-overview.json | 1230+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mapps/talos_cluster/namespaces/kustomization.yaml | 1+
Aapps/talos_cluster/namespaces/opencost.yaml | 8++++++++
Aapps/talos_cluster/opencost/hetzner-pricing.yaml | 25+++++++++++++++++++++++++
Aapps/talos_cluster/opencost/kustomization.yaml | 7+++++++
Aapps/talos_cluster/opencost/release.yaml | 69+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Aapps/talos_cluster/opencost/repository.yaml | 8++++++++
10 files changed, 2515 insertions(+), 0 deletions(-)

diff --git a/apps/talos_cluster/kustomization.yaml b/apps/talos_cluster/kustomization.yaml @@ -35,3 +35,4 @@ resources: - ./peertube - ./persephone - ./codeberg + - ./opencost diff --git a/apps/talos_cluster/monitoring-stack/dashboards/kustomization.yaml b/apps/talos_cluster/monitoring-stack/dashboards/kustomization.yaml @@ -60,3 +60,13 @@ configMapGenerator: grafana_dashboard: "1" annotations: grafana_folder: Envoy + - name: grafana-dashboards-opencost + namespace: monitoring + files: + - opencost-overview.json=opencost-overview.json + - opencost-namespace.json=opencost-namespace.json + options: + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: OpenCost diff --git a/apps/talos_cluster/monitoring-stack/dashboards/opencost-namespace.json b/apps/talos_cluster/monitoring-stack/dashboards/opencost-namespace.json @@ -0,0 +1,1155 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "description": "A detailed namespace-level cost analysis dashboard that breaks down infrastructure spending by pods, containers, and persistent volumes within a selected namespace. Use this dashboard to understand which workloads are driving costs within a namespace, track cost trends over time, and identify optimization opportunities at the pod and container level. This dashboard is ideal for application teams monitoring their own resource consumption and costs. The dashboards were generated using opencost-mixin. https://github.com/adinhodovic/opencost-mixin. Open issues and create feature requests in the repository.", + "editable": false, + "links": [ + { + "asDropdown": false, + "includeVars": false, + "keepTime": true, + "tags": [ + "opencost", + "opencost-mixin" + ], + "targetBlank": true, + "title": "OpenCost", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Current hourly cost rate for the selected namespace, including CPU, RAM, and PV costs. This provides real-time visibility into namespace spending and helps track the immediate impact of workload changes on costs.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) / (1024 * 1024 * 1024) * 1\n )\n)\n\n+\nsum(\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) * 1\n )\n)\n\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n ) or vector(0)\n) * 1\n\n", + "instant": false + } + ], + "title": "Hourly Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total daily cost for the selected namespace. The percentage change indicates cost variance compared to the previous period, helping application teams monitor their daily spending and detect unexpected cost increases.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 3, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) / (1024 * 1024 * 1024) * 24\n )\n)\n\n+\nsum(\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) * 24\n )\n)\n\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n ) or vector(0)\n) * 24\n\n", + "instant": false + } + ], + "title": "Daily Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly cost for the selected namespace based on current hourly rates. Application teams can use this to track their budget allocation and ensure they stay within their cost targets.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 4, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n)\n\n+\nsum(\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) * 730\n )\n)\n\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n ) or vector(0)\n) * 730\n\n", + "instant": false + } + ], + "title": "Monthly Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly CPU cost for the selected namespace. Compare this with RAM costs to understand your namespace compute profile and identify if CPU requests are appropriately sized for your workloads.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) * 730\n )\n)\n", + "instant": false + } + ], + "title": "Monthly CPU Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly RAM cost for the selected namespace. High memory costs may indicate opportunities to optimize container memory requests or identify memory-intensive workloads that could benefit from tuning.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 6, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n)\n", + "instant": false + } + ], + "title": "Monthly Ram Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly Persistent Volume cost for the selected namespace. Monitor this to identify unused PVCs or opportunities to migrate to cheaper storage classes without impacting application performance.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 7, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n )\n) * 730\n", + "instant": false + } + ], + "title": "Monthly PV Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Daily cost trend for the selected namespace over time. Use this to track how namespace costs evolve, identify cost spikes related to deployments or scaling events, and validate that cost optimization efforts are effective.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10 + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) / (1024 * 1024 * 1024) * 24\n )\n)\n\n+\nsum(\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) * 24\n )\n)\n\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n ) or vector(0)\n) * 24\n\n", + "legendFormat": "Daily Cost" + } + ], + "title": "Daily Cost", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Monthly cost projection trend for the selected namespace. This helps application teams track their projected monthly spending and ensure they remain within their allocated budget throughout the billing period.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10 + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n)\n\n+\nsum(\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) * 730\n )\n)\n\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n ) or vector(0)\n) * 730\n\n", + "legendFormat": "Monthly Cost" + } + ], + "title": "Monthly Cost", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Monthly cost distribution for the selected namespace across resource types (CPU, RAM, Persistent Volumes). This shows which resource category is the primary cost driver for this namespace, helping teams prioritize their optimization efforts.", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 10, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) * 730\n )\n)\n", + "instant": true, + "legendFormat": "CPU" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n }\n )\n by (namespace, instance)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n)\n", + "instant": true, + "legendFormat": "RAM" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n )\n) * 730\n", + "instant": true, + "legendFormat": "PV" + } + ], + "title": "Cost by Resource", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 11, + "title": "Pod Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Top 10 pods by projected monthly cost (based on current hourly rates) with percentage change compared to 7 days and 30 days ago. Positive percentages indicate cost increases (red), negative percentages indicate cost decreases (green). Use this to identify the most expensive pods in the namespace and track how pod costs change over time, especially after deployments or configuration changes.", + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Cost Change vs 7d Ago (%)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cost Change vs 30d Ago (%)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 18, + "x": 0, + "y": 10 + }, + "id": 12, + "options": { + "footer": { + "enablePagination": true + }, + "sortBy": [ + { + "desc": true, + "displayName": "Total Cost (Today)" + } + ] + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730)\n )\n ) by (pod)\n)\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730)\n )\n ) by (pod)\n)\n\n/\ntopk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 7d\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 7d\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (instance) * 730)\n )\n ) by (pod)\n)\n\n* 100\n- 100\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730)\n )\n ) by (pod)\n)\n\n/\ntopk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 30d\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 30d\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 30d\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 30d\n ) by (instance) * 730)\n )\n ) by (pod)\n)\n\n* 100\n- 100\n", + "format": "table", + "instant": true + } + ], + "title": "Pod Monthly Cost", + "transformations": [ + { + "id": "merge" + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "indexByName": { + "Value #A": 1, + "Value #B": 2, + "Value #C": 3, + "pod": 0 + }, + "renameByName": { + "Value #A": "Monthly Cost", + "Value #B": "Cost Change vs 7d Ago (%)", + "Value #C": "Cost Change vs 30d Ago (%)", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Top 10 pods by monthly cost showing the distribution of spending across pods in the namespace. This visualization helps identify which pods consume the most resources and whether costs are evenly distributed or concentrated in a few workloads.", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 13, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, pod)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730)\n )\n ) by (pod)\n)\n", + "instant": true, + "legendFormat": "{{ pod }}" + } + ], + "title": "Cost by Pod", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 14, + "title": "Container Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Top 10 containers by current monthly cost with percentage change compared to 7 days and 30 days ago. Positive percentages indicate cost increases (red), negative percentages indicate cost decreases (green). This granular view helps identify specific containers within pods that are driving costs, useful for optimizing multi-container pod configurations.", + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Cost Change vs 7d Ago (%)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cost Change vs 30d Ago (%)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 18, + "x": 0, + "y": 21 + }, + "id": 15, + "options": { + "footer": { + "enablePagination": true + }, + "sortBy": [ + { + "desc": true, + "displayName": "Monthly Cost" + } + ] + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730\n )\n )\n ) by (container)\n)\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730\n )\n )\n ) by (container)\n)\n\n/\ntopk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 7d\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 7d\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (instance) * 730\n )\n )\n ) by (container)\n)\n\n* 100\n- 100\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730\n )\n )\n ) by (container)\n)\n\n/\ntopk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 30d\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 30d\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"} offset 30d\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 30d\n ) by (instance) * 730\n )\n )\n ) by (container)\n)\n\n* 100\n- 100\n", + "format": "table", + "instant": true + } + ], + "title": "Container Monthly Cost", + "transformations": [ + { + "id": "merge" + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "indexByName": { + "Value #A": 1, + "Value #B": 2, + "Value #C": 3, + "container": 0 + }, + "renameByName": { + "Value #A": "Monthly Cost", + "Value #B": "Cost Change vs 7d Ago (%)", + "Value #C": "Cost Change vs 30d Ago (%)", + "container": "Container" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Top 10 containers by monthly cost showing the distribution of spending across containers in the namespace. This helps identify which container images or workload types are most expensive and whether sidecar containers are adding significant costs.", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 21 + }, + "id": 16, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n (\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) / (1024 * 1024 * 1024) * 730\n )\n )\n +\n (\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n job=\"$job\"}\n )\n by (instance, container)\n * on(instance) group_left()\n (\n avg(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (instance) * 730\n )\n )\n ) by (container)\n)\n", + "instant": true, + "legendFormat": "{{ container }}" + } + ], + "title": "Cost by Container", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 17, + "title": "PV Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "List of Persistent Volumes used by the selected namespace with their capacity (in GiB) and monthly cost, sorted by total cost. Use this to identify large or expensive volumes that may be candidates for cleanup, resizing, or migration to cheaper storage classes.", + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [] + }, + "unit": "decgbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Cost" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 18, + "x": 0, + "y": 32 + }, + "id": 18, + "options": { + "footer": { + "enablePagination": true + }, + "sortBy": [ + { + "desc": true, + "displayName": "Monthly Cost" + } + ] + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\n job=\"$job\"\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace)\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\n job=\"$job\",\n namespace=\"$namespace\"\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n) by (persistentvolume)\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n )\n) by (persistentvolume) * 730\n", + "format": "table", + "instant": true + } + ], + "title": "Persistent Volumes Monthly Cost", + "transformations": [ + { + "id": "merge" + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true, + "namespace": true + }, + "indexByName": { + "Value #A": 1, + "Value #B": 2, + "persistentvolume": 0 + }, + "renameByName": { + "Value #A": "Total GiB", + "Value #B": "Total Cost", + "persistentvolume": "Persistent Volume" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Distribution of monthly storage costs across Persistent Volumes in the namespace. This shows which volumes consume the most storage budget and helps identify if storage costs are concentrated in a few large volumes or distributed across many smaller ones.", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 32 + }, + "id": 19, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n *\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n * on(persistentvolume) group_left(cluster, namespace) (\n label_replace(\n kube_persistentvolumeclaim_info{\n cluster=\"$cluster\",\njob=\"$job\"\n,\nnamespace=\"$namespace\"\n\n },\n \"persistentvolume\", \"$1\",\n \"volumename\", \"(.*)\"\n )\n )\n) by (persistentvolume) * 730\n", + "instant": true, + "legendFormat": "{{ persistentvolume }}" + } + ], + "title": "Cost by Persistent Volume", + "type": "piechart" + } + ], + "schemaVersion": 39, + "tags": [ + "opencost", + "opencost-mixin" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "label": "Data source", + "name": "datasource", + "query": "prometheus", + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "hide": 2, + "label": "Cluster", + "name": "cluster", + "query": "label_values(opencost_build_info{}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "query": "label_values(opencost_build_info{cluster=\"$cluster\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "query": "label_values(kube_namespace_labels{cluster=\"$cluster\", job=\"$job\"}, namespace)", + "refresh": 2, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timezone": "utc", + "title": "OpenCost / Namespace", + "uid": "opencost-mixin-namespace-jkwq", + "gnetId": 22252 +} +\ No newline at end of file diff --git a/apps/talos_cluster/monitoring-stack/dashboards/opencost-overview.json b/apps/talos_cluster/monitoring-stack/dashboards/opencost-overview.json @@ -0,0 +1,1229 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "description": "A comprehensive overview dashboard for OpenCost that displays cluster-wide cost metrics including hourly, daily, and monthly costs broken down by resource type (CPU, RAM, PV), instance type, namespace, and individual nodes. Use this dashboard to monitor overall infrastructure spending, identify cost trends, and detect cost anomalies across your Kubernetes cluster. The dashboards were generated using opencost-mixin. https://github.com/adinhodovic/opencost-mixin. Open issues and create feature requests in the repository.", + "editable": false, + "links": [ + { + "asDropdown": false, + "includeVars": false, + "keepTime": true, + "tags": [ + "opencost", + "opencost-mixin" + ], + "targetBlank": true, + "title": "OpenCost", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "Cluster Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Current hourly infrastructure cost rate across the cluster. This metric provides real-time cost visibility and can be used to project daily and monthly spending. The percentage change helps track cost fluctuations over time.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n) \n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) \n", + "instant": false + } + ], + "title": "Hourly Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total daily infrastructure cost across the cluster, including compute (CPU, RAM) and storage (PV) costs. The percentage change indicates cost variance compared to the previous period, helping identify sudden cost increases or decreases.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 3, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n) * 24\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 24\n", + "instant": false + } + ], + "title": "Daily Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly infrastructure cost based on current hourly rates (730 hours per month). This projection helps with budget planning and cost forecasting. Compare this value against your budget to ensure spending stays within limits.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 4, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n) * 730\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 730\n", + "instant": false + } + ], + "title": "Monthly Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly cost for CPU (compute) resources across all cluster nodes. Compare this with RAM costs to understand your compute vs. memory cost ratio and optimize instance type selection accordingly.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"cpu\",\n unit=\"core\"\n }\n ) by (node)\n * on(node) group_left()\n sum(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n", + "instant": false + } + ], + "title": "Monthly CPU Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly cost for RAM (memory) resources across all cluster nodes. This metric helps identify if memory is a significant cost driver and can guide decisions about node sizing and memory allocation strategies.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 6, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"memory\",\n unit=\"byte\"\n }\n ) by (node)\n / (1024 * 1024 * 1024)\n * on(node) group_left()\n sum(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n", + "instant": false + } + ], + "title": "Monthly Ram Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Projected monthly cost for Persistent Volume (storage) resources across the cluster. Monitor this metric to identify unused or oversized volumes that can be optimized to reduce storage costs.", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 7, + "options": { + "graphMode": "none", + "percentChangeColorMode": "inverted", + "showPercentChange": true + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 730\n", + "instant": false + } + ], + "title": "Monthly PV Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Hourly cost trend over time showing how infrastructure spending fluctuates throughout the day. Use this to identify cost spikes, correlate costs with workload patterns, and detect autoscaling behavior impact on spending.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10 + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n) \n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) \n", + "legendFormat": "Hourly Cost" + } + ], + "title": "Hourly Cost", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Daily cost trend showing infrastructure spending patterns over multiple days. This view helps identify day-over-day cost changes, weekly patterns, and the impact of infrastructure changes on overall spending.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10 + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n) * 24\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 24\n", + "legendFormat": "Daily Cost" + } + ], + "title": "Daily Cost", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Monthly cost projection trend over time. This visualization helps track how your projected monthly spending evolves and whether you are staying within budget throughout the billing period.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10 + }, + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n) * 730\n+\nsum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 730\n", + "legendFormat": "Monthly Cost" + } + ], + "title": "Monthly Cost", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Monthly cost distribution across resource types (CPU, RAM, Persistent Volumes). This breakdown shows which resource category consumes the most budget, helping prioritize optimization efforts. For example, if PV costs dominate, focus on storage optimization.", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 11, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"cpu\",\n unit=\"core\"\n }\n ) by (node)\n * on(node) group_left()\n sum(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n", + "instant": true, + "legendFormat": "CPU" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"memory\",\n unit=\"byte\"\n }\n ) by (node)\n / (1024 * 1024 * 1024)\n * on(node) group_left()\n sum(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n", + "instant": true, + "legendFormat": "RAM" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 730\n", + "instant": true, + "legendFormat": "PV" + } + ], + "title": "Cost by Resource", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Top 10 namespaces by monthly cost showing which teams, applications, or environments consume the most resources. Use this to allocate costs to teams, identify expensive applications, and ensure fair resource distribution across the organization.", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 12, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} / (1024 * 1024 * 1024) * 730\n )\n +\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} * 730\n )\n ) by (namespace)\n)\n", + "instant": true, + "legendFormat": "{{ namespace }}" + } + ], + "title": "Cost by Namespace", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Top 10 instance types by monthly cost showing which VM/node types contribute most to infrastructure spending. This helps evaluate whether your instance type selection is cost-effective and identify opportunities to switch to more economical instance families.", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + } + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 13, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (instance_type) * 730\n)\n", + "instant": true, + "legendFormat": "{{ instance_type }}" + } + ], + "title": "Cost by Instance Type", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Cost variance comparing current hourly costs against 7-day and 30-day historical averages. Positive values indicate costs are higher than average, negative values indicate lower costs. Use this to detect cost anomalies and unusual spending patterns that may require investigation.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10 + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "(\n avg_over_time(\n sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) [1d:1h]\n )\n -\n avg_over_time(\n sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) [7d:1h]\n )\n)\n/\navg_over_time(\n sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) [7d:1h]\n)\n", + "interval": "30m", + "legendFormat": "Current hourly cost vs. 7-day average" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "(\n avg_over_time(\n sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) [1d:1h]\n )\n -\n avg_over_time(\n sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) [30d:1h]\n )\n)\n/\navg_over_time(\n sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) [30d:1h]\n)\n", + "interval": "30m", + "legendFormat": "Current hourly cost vs. 30-day average" + } + ], + "title": "Total Cost Variance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Resource-specific cost variance comparing current CPU, RAM, and PV costs against their 30-day historical averages. This breakdown helps identify which resource type is driving cost changes - useful for pinpointing whether cost increases are due to compute scaling, memory usage, or storage growth.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10 + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "(\n avg_over_time(\n sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"cpu\",\n unit=\"core\"\n }\n ) by (node)\n * on(node) group_left()\n sum(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n [1d:1h]\n )\n -\n avg_over_time(\n sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"cpu\",\n unit=\"core\"\n }\n ) by (node)\n * on(node) group_left()\n sum(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n [30d:1h]\n )\n)\n/\navg_over_time(\n sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"cpu\",\n unit=\"core\"\n }\n ) by (node)\n * on(node) group_left()\n sum(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n [30d:1h]\n)\n", + "interval": "30m", + "legendFormat": "CPU Cost vs. 30-day average" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "(\n avg_over_time(\n sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"memory\",\n unit=\"byte\"\n }\n ) by (node)\n / (1024 * 1024 * 1024)\n * on(node) group_left()\n sum(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n [1d:1h]\n )\n -\n avg_over_time(\n sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"memory\",\n unit=\"byte\"\n }\n ) by (node)\n / (1024 * 1024 * 1024)\n * on(node) group_left()\n sum(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n [30d:1h]\n )\n)\n/\navg_over_time(\n sum(\n sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"memory\",\n unit=\"byte\"\n }\n ) by (node)\n / (1024 * 1024 * 1024)\n * on(node) group_left()\n sum(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node)\n * 730\n)\n [30d:1h]\n)\n", + "interval": "30m", + "legendFormat": "RAM Cost vs. 30-day average" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "exemplar": false, + "expr": "(\n avg_over_time(\n (sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 730\n) [1d:1h]\n )\n -\n avg_over_time(\n (sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 730\n) [30d:1h]\n )\n)\n/\navg_over_time(\n (sum(\n sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n } / (1024 * 1024 * 1024)\n ) by (persistentvolume)\n * on(persistentvolume) group_left()\n sum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (persistentvolume)\n) * 730\n) [30d:1h]\n)\n", + "interval": "30m", + "legendFormat": "PV Cost vs. 30-day average" + } + ], + "title": "Resource Cost Variance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 16, + "title": "Cloud Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Detailed breakdown of monthly costs per node, showing CPU cost, RAM cost, and total cost for each node along with instance type and architecture. Sorted by total cost to highlight the most expensive nodes. Use this to identify underutilized expensive nodes that could be downsized or removed.", + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 21 + }, + "id": 17, + "options": { + "footer": { + "enablePagination": true + }, + "sortBy": [ + { + "desc": true, + "displayName": "Total Cost" + } + ] + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"cpu\",\n unit=\"core\"\n }\n) by (node)\n* on(node) group_left(cluster, instance_type, arch)\n sum(\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n }\n ) by (node, instance_type, arch)\n* 730\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n kube_node_status_capacity{\n cluster=\"$cluster\",\njob=\"$job\"\n,\n resource=\"memory\",\n unit=\"byte\"\n }\n) by (node)\n/ (1024 * 1024 * 1024)\n* on(node) group_left(cluster, instance_type, arch)\n sum(\n node_ram_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n ) by (node, instance_type, arch)\n* 730\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n node_total_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n) by (node, instance_type, arch)\n* 730\n", + "format": "table", + "instant": true + } + ], + "title": "Nodes Monthly Cost", + "transformations": [ + { + "id": "merge" + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "indexByName": { + "Value #A": 3, + "Value #B": 4, + "Value #C": 5, + "arch": 2, + "instance_type": 1, + "node": 0 + }, + "renameByName": { + "Value #A": "CPU Cost", + "Value #B": "RAM Cost", + "Value #C": "Total Cost", + "arch": "Architecture", + "instance_type": "Instance Type", + "node": "Node" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "List of all Persistent Volumes with their capacity (in GiB) and monthly cost, sorted by total cost. Use this to identify large or expensive volumes that may be candidates for cleanup, resizing, or migration to cheaper storage classes.", + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [] + }, + "unit": "decgbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Cost" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 21 + }, + "id": 18, + "options": { + "footer": { + "enablePagination": true + }, + "sortBy": [ + { + "desc": true, + "displayName": "Total Cost" + } + ] + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / 1024 / 1024 / 1024\n) by (persistentvolume)\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n kube_persistentvolume_capacity_bytes{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n / 1024 / 1024 / 1024\n) by (persistentvolume)\n*\nsum(\n pv_hourly_cost{\n cluster=\"$cluster\",\njob=\"$job\"\n\n }\n * 730\n) by (persistentvolume)\n", + "format": "table", + "instant": true + } + ], + "title": "Persistent Volumes Monthly Cost", + "transformations": [ + { + "id": "merge" + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "indexByName": { + "Value #A": 1, + "Value #B": 2, + "persistentvolume": 0 + }, + "renameByName": { + "Value #A": "Total GiB", + "Value #B": "Total Cost", + "persistentvolume": "Persistent Volume" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 19, + "title": "Namespace Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Top 10 namespaces by current monthly cost with percentage change compared to 7 days and 30 days ago. Positive percentages indicate cost increases (red), negative percentages indicate cost decreases (green). Click on a namespace name to drill down into detailed pod and container costs. Use this to track namespace-level spending trends and identify teams or applications with growing costs.", + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Cost Change vs 7d Ago (%)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cost Change vs 30d Ago (%)" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Go To Namespace", + "type": "dashboard", + "url": "/d/opencost-mixin-namespace-jkwq/opencost-namespace?var-job=$job&var-namespace=${__data.fields.Namespace}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 20, + "options": { + "footer": { + "enablePagination": true + }, + "sortBy": [ + { + "desc": true, + "displayName": "Monthly Cost" + } + ] + }, + "pluginVersion": "v11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} / (1024 * 1024 * 1024) * 730\n )\n +\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} * 730\n )\n ) by (namespace)\n)\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} / (1024 * 1024 * 1024) * 730\n )\n +\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} * 730\n )\n ) by (namespace)\n)\n\n/\ntopk(10,\n sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d / (1024 * 1024 * 1024) * 730\n )\n +\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d * 730\n )\n ) by (namespace)\n)\n\n* 100\n- 100\n", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "topk(10,\n sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} / (1024 * 1024 * 1024) * 730\n )\n +\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n job=\"$job\"}\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} * 730\n )\n ) by (namespace)\n)\n\n/\ntopk(10,\n sum(\n sum(\n container_memory_allocation_bytes{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_ram_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d / (1024 * 1024 * 1024) * 730\n )\n +\n sum(\n container_cpu_allocation{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d\n ) by (namespace, instance)\n * on(instance) group_left()\n (\n node_cpu_hourly_cost{\n cluster=\"$cluster\",\n job=\"$job\"} offset 7d * 730\n )\n ) by (namespace)\n)\n\n* 100\n- 100\n", + "format": "table", + "instant": true + } + ], + "title": "Namespace Monthly Cost", + "transformations": [ + { + "id": "merge" + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "indexByName": { + "Value #A": 1, + "Value #B": 2, + "Value #C": 3, + "namespace": 0 + }, + "renameByName": { + "Value #A": "Monthly Cost", + "Value #B": "Cost Change vs 7d Ago (%)", + "Value #C": "Cost Change vs 30d Ago (%)", + "namespace": "Namespace" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 39, + "tags": [ + "opencost", + "opencost-mixin" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "label": "Data source", + "name": "datasource", + "query": "prometheus", + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "hide": 2, + "label": "Cluster", + "name": "cluster", + "query": "label_values(opencost_build_info{}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "query": "label_values(opencost_build_info{cluster=\"$cluster\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timezone": "utc", + "title": "OpenCost / Overview", + "uid": "opencost-mixin-kover-jkwq", + "gnetId": 22208 +} +\ No newline at end of file diff --git a/apps/talos_cluster/namespaces/kustomization.yaml b/apps/talos_cluster/namespaces/kustomization.yaml @@ -30,3 +30,4 @@ resources: - peertube.yaml - persephone.yaml - codeberg.yaml + - opencost.yaml diff --git a/apps/talos_cluster/namespaces/opencost.yaml b/apps/talos_cluster/namespaces/opencost.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: opencost + labels: + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/apps/talos_cluster/opencost/hetzner-pricing.yaml b/apps/talos_cluster/opencost/hetzner-pricing.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hetzner-pricing-csv + namespace: opencost +data: + hetzner.csv: | + Region,Instance Type,vCPUs,RAM (GiB),Storage (GB),GPU,Cost per Hour,PricingUnit + fsn1,cax11,2,4,40,0,0.00742,Hour + hel1,cax11,2,4,40,0,0.00742,Hour + nbg1,cax11,2,4,40,0,0.00742,Hour + hel1,cax21,4,8,80,0,0.01321,Hour + fsn1,cax21,4,8,80,0,0.01321,Hour + nbg1,cax21,4,8,80,0,0.01321,Hour + fsn1,cax31,8,16,160,0,0.02643,Hour + hel1,cax31,8,16,160,0,0.02643,Hour + nbg1,cax31,8,16,160,0,0.02643,Hour + hel1,cax41,16,32,320,0,0.05000,Hour + fsn1,cax41,16,32,320,0,0.05000,Hour + nbg1,cax41,16,32,320,0,0.05000,Hour + hel1,cpx11,2,2,40,0,0.00614,Hour + hel1,cpx21,3,4,80,0,0.01007,Hour + hel1,cpx31,4,8,160,0,0.01793,Hour + hel1,cpx41,8,16,240,0,0.03236,Hour + hel1,cpx51,16,32,360,0,0.06000,Hour diff --git a/apps/talos_cluster/opencost/kustomization.yaml b/apps/talos_cluster/opencost/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: opencost +resources: + - repository.yaml + - release.yaml + - hetzner-pricing.yaml diff --git a/apps/talos_cluster/opencost/release.yaml b/apps/talos_cluster/opencost/release.yaml @@ -0,0 +1,69 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: opencost + namespace: opencost +spec: + releaseName: opencost + interval: 30m + chart: + spec: + chart: opencost + sourceRef: + kind: HelmRepository + name: opencost + namespace: opencost + interval: 1h + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + opencost: + exporter: + defaultCloudBackend: custom + extraEnv: + - name: USE_CSV_PROVIDER + value: "true" + - name: CSV_PATH + value: /etc/opencost/pricing/hetzner.csv + extraVolumeMounts: + - name: hetzner-pricing + mountPath: /etc/opencost/pricing + readOnly: true + prometheus: + internal: + enabled: true + # vmselect service from the victoria-metrics-k8s-stack HelmRelease (name: vm) + serviceName: vmselect-vm-victoria-metrics-k8s-stack + namespaceName: monitoring + port: 8481 + path: /select/0/prometheus/ + serviceMonitor: + enabled: true + namespace: monitoring + extraVolumes: + - name: hetzner-pricing + configMap: + name: hetzner-pricing-csv +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: opencost + namespace: opencost +spec: + parentRefs: + - name: envoy-gateway + namespace: envoy-gateway + hostnames: + - opencost.k8s.midnightthoughts.space + rules: + - backendRefs: + - name: opencost + port: 9090 + timeouts: + request: 240s + backendRequest: 0s diff --git a/apps/talos_cluster/opencost/repository.yaml b/apps/talos_cluster/opencost/repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: opencost + namespace: opencost +spec: + interval: 1h + url: https://opencost.github.io/opencost-helm-chart