Sovereign Management

#!/bin/bash
# sovereign-setup.sh — Full sovereign management deployment
# Replaces all BMC management with inference-compatible alternatives
# Run as root on each inference server

set -e

BMC_IP=${BMC_IP:-"192.168.1.100"}
BMC_USER=${BMC_USER:-"admin"}
BMC_PASS=${BMC_PASS:-"password"}

echo "=========================================="
echo " Sovereign Management Setup"
echo "=========================================="
echo ""

# ── Layer 1: BMC Cold Standby ──
echo "[Layer 1] Silencing BMC..."

# Unload IPMI kernel modules (eliminates all in-band SMIs)
modprobe -r ipmi_si 2>/dev/null || true
modprobe -r ipmi_devintf 2>/dev/null || true
modprobe -r ipmi_msghandler 2>/dev/null || true
echo "  IPMI kernel modules unloaded"

# Disable watchdog, SOL, video capture via out-of-band IPMI
IPMI="ipmitool -I lanplus -H $BMC_IP -U $BMC_USER -P $BMC_PASS"
$IPMI mc watchdog off 2>/dev/null || true
$IPMI sol deactivate 2>/dev/null || true
$IPMI raw 0x32 0x73 0x01 0x00 2>/dev/null || true
$IPMI raw 0x32 0x71 0x01 0x00 2>/dev/null || true
$IPMI sel clear 2>/dev/null || true

# Set fans to full speed before disabling BMC thermal loop
$IPMI raw 0x30 0x45 0x01 0x01 2>/dev/null || \
  $IPMI raw 0x30 0x30 0x01 0x00 2>/dev/null || true
echo "  BMC silenced (cold standby)"

# ── Layer 2: Sovereign Health Daemon ──
echo "[Layer 2] Installing health daemon..."

# Ensure I2C and MSR access
modprobe i2c-dev 2>/dev/null || true
modprobe msr 2>/dev/null || true

# Install systemd service
cat > /etc/systemd/system/sovereign-health.service <<'UNIT'
[Unit]
Description=Sovereign Health Daemon
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/sovereign-healthd
Restart=always
RestartSec=1
WatchdogSec=30
CPUAffinity=0
Nice=-10
AmbientCapabilities=CAP_SYS_RAWIO CAP_DAC_READ_SEARCH
NoNewPrivileges=true

[Install]
WantedBy=multi-user.target
UNIT

systemctl daemon-reload
systemctl enable sovereign-health
systemctl start sovereign-health
echo "  Health daemon running on Core 0, Prometheus on :9100"

# ── Layer 3: Verify Fleet Integration ──
echo "[Layer 3] Verifying..."

# Test Prometheus endpoint
sleep 2
if curl -sf http://localhost:9100/metrics | grep -q sovereign_cpu_temp; then
  echo "  Prometheus endpoint OK"
else
  echo "  WARNING: Prometheus endpoint not responding yet"
fi

# Verify zero SMIs
SMI_START=$(rdmsr -p 0 0x00000034 2>/dev/null || echo "0")
sleep 5
SMI_END=$(rdmsr -p 0 0x00000034 2>/dev/null || echo "0")
SMI_DELTA=$(( 0x$SMI_END - 0x$SMI_START ))
echo "  SMIs in 5 seconds: $SMI_DELTA (target: 0)"

echo ""
echo "=========================================="
echo " Sovereign Management Active"
echo "=========================================="
echo "  BMC: cold standby (emergency reset only)"
echo "  Sensors: MSR + I2C on Core 0"
echo "  Monitoring: Prometheus :9100"
echo "  Overhead: <0.5%"
echo "  SMIs from management: 0"