Initial metrics skill with aggregation

2026-02-02 22:29:24 +01:00
commit 09fe677f36
3 changed files with 304 additions and 0 deletions
--- a/scripts/run.sh
+++ b/scripts/run.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+set -e
+
+SCRIPT_DIR="$(dirname "$0")"
+METRICS_PORT="${METRICS_PORT:-9090}"
+NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}"
+TARGETS_FILE="/tmp/metrics_targets.txt"
+
+# Start node_exporter in background
+start_node_exporter() {
+    echo "Starting node_exporter on port $NODE_EXPORTER_PORT..."
+    node_exporter --web.listen-address=":$NODE_EXPORTER_PORT" &
+    NODE_EXPORTER_PID=$!
+    sleep 2
+
+    if ! kill -0 $NODE_EXPORTER_PID 2>/dev/null; then
+        echo "Failed to start node_exporter"
+        exit 1
+    fi
+    echo "node_exporter running (PID $NODE_EXPORTER_PID)"
+}
+
+# Create the aggregator CGI script
+create_aggregator_script() {
+    cat > /tmp/aggregate.sh << 'SCRIPT'
+#!/bin/bash
+NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}"
+TARGETS_FILE="/tmp/metrics_targets.txt"
+
+# Collect all metrics
+{
+    # System metrics
+    echo "# HELP vibestack_up Whether the metrics aggregator is up"
+    echo "# TYPE vibestack_up gauge"
+    echo "vibestack_up 1"
+    echo ""
+
+    # node_exporter metrics
+    curl -s "http://localhost:$NODE_EXPORTER_PORT/metrics" 2>/dev/null || true
+
+    # Skill metrics
+    if [ -f "$TARGETS_FILE" ]; then
+        while IFS=' ' read -r skill_name target || [ -n "$skill_name" ]; do
+            [[ "$skill_name" =~ ^#.*$ ]] && continue
+            [ -z "$skill_name" ] && continue
+            [ -z "$target" ] && continue
+
+            echo ""
+            echo "# Metrics from skill: $skill_name"
+            curl -s "http://${target}/metrics" 2>/dev/null || true
+        done < "$TARGETS_FILE"
+    fi
+}
+SCRIPT
+    chmod +x /tmp/aggregate.sh
+}
+
+# Serve metrics using socat
+serve_metrics() {
+    echo "Starting metrics aggregator on port $METRICS_PORT..."
+
+    if ! command -v socat &>/dev/null; then
+        echo "Installing socat..."
+        apt-get update && apt-get install -y socat
+    fi
+
+    # Use socat to handle HTTP requests
+    exec socat TCP-LISTEN:$METRICS_PORT,reuseaddr,fork EXEC:"/tmp/metrics_handler.sh"
+}
+
+# Create HTTP handler script
+create_handler_script() {
+    cat > /tmp/metrics_handler.sh << 'HANDLER'
+#!/bin/bash
+# Read HTTP request (we ignore it, always serve /metrics)
+read -r request_line
+while read -r header; do
+    [ "$header" = $'\r' ] && break
+done
+
+# Get metrics
+metrics=$(/tmp/aggregate.sh)
+content_length=${#metrics}
+
+# Send HTTP response
+printf "HTTP/1.1 200 OK\r\n"
+printf "Content-Type: text/plain; charset=utf-8\r\n"
+printf "Content-Length: %d\r\n" "$content_length"
+printf "Connection: close\r\n"
+printf "\r\n"
+printf "%s" "$metrics"
+HANDLER
+    chmod +x /tmp/metrics_handler.sh
+}
+
+# Cleanup on exit
+cleanup() {
+    echo "Shutting down..."
+    [ -n "$NODE_EXPORTER_PID" ] && kill $NODE_EXPORTER_PID 2>/dev/null
+    exit 0
+}
+trap cleanup SIGTERM SIGINT
+
+start_node_exporter
+create_aggregator_script
+create_handler_script
+serve_metrics