Module 4: Core Performance Tuning with Performance Profiles

# Verify you're connected to the SNO cluster
oc whoami
oc get nodes

# Get the SNO node name
TARGET_NODE=$(oc get nodes -o jsonpath='{.items[0].metadata.name}')
echo "Target node: $TARGET_NODE"

# Get detailed node information
echo "=== Node Information ==="
oc describe node $TARGET_NODE | grep -E "(Name:|Roles:|Capacity:|Allocatable:)" -A 10

# Check CPU information
echo "=== CPU Topology ==="
oc debug node/$TARGET_NODE -- chroot /host lscpu | grep -E "(CPU\(s\)|Thread|Core|Socket|NUMA)"

# SNO uses the default master pool
echo "=== Machine Config Pools ==="
oc get mcp

echo ""
echo "📊 Master Pool Details:"
oc get mcp master -o yaml | grep -E "(readyMachineCount|updatedMachineCount|machineCount)" | head -3

# Run the CPU allocation calculator script
bash ~/low-latency-performance-workshop/scripts/module04-calculate-cpu-allocation.sh

# Run the Performance Profile creation script
bash ~/low-latency-performance-workshop/scripts/module04-create-performance-profile.sh

echo "📊 Current master MCP status:"
oc get mcp master
echo ""
echo "⏱️  Starting continuous monitoring (Ctrl+C to stop):"
echo "watch 'oc get mcp master; echo; oc get nodes'"

# Load cluster configuration
source /tmp/cluster-config

echo "=== Detailed Node Update Monitoring ==="

# Check machine config daemon status
echo "📋 Machine Config Daemon Pods:"
oc get pods -n openshift-machine-config-operator | grep daemon

echo ""
echo "📋 Recent Node Events:"
oc get events --sort-by='.lastTimestamp' --field-selector involvedObject.name=$TARGET_NODE | tail -10

echo ""
echo "📋 Machine Config Status:"
oc describe mcp master | grep -A 10 -B 5 "Conditions:"

# Load cluster configuration
source /tmp/cluster-config

echo "=== Waiting for Performance Profile Application ==="
echo "This process may take 5-20 minutes depending on your configuration"

echo ""
echo "⏳ SNO Update Process:"
echo "   1. Machine config generation"
echo "   2. Node cordoning and draining"
if [ "${ENABLE_RT_KERNEL:-false}" = "true" ]; then
    echo "   3. RT kernel installation and reboot (RT kernel enabled)"
    echo "   4. Node rejoin and ready state"
else
    echo "   3. Kernel parameter updates (no reboot required)"
    echo "   4. Node ready state"
fi
echo ""
echo "🔄 Waiting for master MCP to be updated..."
oc wait --for=condition=Updated mcp/master --timeout=1200s

# Only wait for reboot if RT kernel is enabled
if [ "${ENABLE_RT_KERNEL:-false}" = "true" ]; then
    echo "🔄 Waiting for node to be ready after reboot..."
    oc wait --for=condition=Ready node/$TARGET_NODE --timeout=600s
else
    echo "✅ Node update complete (no reboot required)"
fi

echo ""
echo "✅ Performance Profile application completed!"
echo "📊 Final status:"
oc get nodes
oc get mcp

echo ""
echo "🧪 Testing cluster functionality after performance tuning..."

# Create a simple test pod to verify the cluster is still functional
cat << EOF | oc apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: post-tuning-test
  namespace: default
  labels:
    app: post-tuning-test
spec:
  containers:
  - name: test-container
    image: registry.redhat.io/ubi8/ubi-minimal:latest
    command: ["sleep"]
    args: ["30"]
    resources:
      requests:
        memory: "32Mi"
        cpu: "50m"
      limits:
        memory: "64Mi"
        cpu: "100m"
  restartPolicy: Never
EOF

# Wait for pod to be scheduled and running
echo "⏱️  Waiting for test pod to verify cluster functionality..."
if oc wait --for=condition=Ready pod/post-tuning-test --timeout=120s -n default 2>/dev/null; then
    echo "   ✅ Cluster is functional after performance tuning!"
    echo "   📍 Test pod scheduled successfully"
    oc delete pod post-tuning-test -n default --ignore-not-found=true
else
    echo "   ⚠️  Cluster may have scheduling issues after performance tuning"
    echo "   💡 Consider using the revert script if problems persist"
    echo "   🔍 Check pod status: oc describe pod post-tuning-test -n default"
fi

echo "=== Performance Profile Verification ==="
echo "Running comprehensive cluster health check..."
echo ""

# Use Python script for thorough verification
python3 ~/low-latency-performance-workshop/scripts/module04-cluster-health-check.py

echo ""
echo "✅ Comprehensive verification completed!"
echo ""
echo "💡 The health check script validates:"
echo "   - Cluster architecture detection"
echo "   - Performance Profile status"
echo "   - Real-time kernel installation"
echo "   - CPU isolation configuration"
echo "   - Pod scheduling functionality"

echo "=== Performance Tuning Summary ==="
echo ""

# Get color-coded summary of current performance settings
python3 ~/low-latency-performance-workshop/scripts/module04-performance-summary.py

echo ""
echo "💡 This summary shows:"
echo "   - Current CPU allocation strategy"
echo "   - Performance vs stability balance"
echo "   - Recommendations for optimization"

# Calculate CPU allocation based on cluster type
bash ~/low-latency-performance-workshop/scripts/module04-calculate-cpu-allocation.sh

# Create Performance Profile with validated configuration
bash ~/low-latency-performance-workshop/scripts/module04-create-performance-profile.sh

# Validate Performance Profile configuration
python3 ~/low-latency-performance-workshop/scripts/module04-tuning-validator.py

# Check CPU isolation configuration
python3 ~/low-latency-performance-workshop/scripts/module04-cpu-isolation-checker.py

# Validate HugePages configuration
python3 ~/low-latency-performance-workshop/scripts/module04-hugepages-validator.py

# Check specific Performance Profile
python3 ~/low-latency-performance-workshop/scripts/module04-hugepages-validator.py \
    --profile my-performance-profile

cd ~/kube-burner-configs

# Load cluster configuration
source /tmp/cluster-config

echo "=== Creating Tuned Performance Test Configuration ==="
echo "Cluster type: $CLUSTER_TYPE"
echo "Profile name: $PROFILE_NAME"
echo "Target node: $TARGET_NODE"

# Create a new test configuration for the tuned cluster
cat > tuned-config.yml << EOF
global:
  measurements:
    - name: podLatency
      thresholds:
        - conditionType: Ready
          metric: P99
          threshold: 15000ms  # Expect better performance after tuning

metricsEndpoints:
  - indexer:
      type: local
      metricsDirectory: collected-metrics-tuned

jobs:
  - name: tuned-workload
    jobType: create
    jobIterations: 20
    namespace: tuned-workload
    namespacedIterations: true
    cleanup: false
    podWait: false
    waitWhenFinished: true
    verifyObjects: true
    errorOnVerify: false
    objects:
      - objectTemplate: tuned-pod.yml
        replicas: 5
        inputVars:
          containerImage: registry.redhat.io/ubi8/ubi:latest
EOF

# Create a tuned pod template with appropriate node selector
if [ "$CLUSTER_TYPE" = "SNO" ]; then
    NODE_SELECTOR_YAML='nodeSelector:
    node-role.kubernetes.io/master: ""'
    echo "📝 SNO: Using master node selector for pod placement"
elif [ "$CLUSTER_TYPE" = "MULTI_NODE" ]; then
    NODE_SELECTOR_YAML='nodeSelector:
    node-role.kubernetes.io/worker-rt: ""'
    echo "📝 Multi-Node: Using worker-rt node selector for pod placement"
else
    NODE_SELECTOR_YAML='nodeSelector:
    node-role.kubernetes.io/master-rt: ""'
    echo "📝 Multi-Master: Using master-rt node selector for pod placement"
fi

cat > tuned-pod.yml << EOF
apiVersion: v1
kind: Pod
metadata:
  name: tuned-pod-{{.Iteration}}-{{.Replica}}
  labels:
    app: tuned-test
    iteration: "{{.Iteration}}"
    cluster-type: "$CLUSTER_TYPE"
spec:
  $NODE_SELECTOR_YAML
  containers:
  - name: tuned-container
    image: {{.containerImage}}
    command: ["sleep"]
    args: ["300"]
    resources:
      requests:
        memory: "64Mi"
        cpu: "100m"
      limits:
        memory: "128Mi"
        cpu: "200m"
  restartPolicy: Never
EOF

echo ""
echo "🚀 Running Performance Test on Tuned Cluster"
echo "   - Test configuration: tuned-config.yml"
echo "   - Pod template: tuned-pod.yml"
echo "   - Target: $CLUSTER_TYPE cluster with $PROFILE_NAME"
echo ""

# Run the performance test
kube-burner init -c tuned-config.yml --log-level=info

echo ""
echo "✅ Tuned performance test completed!"
echo "📊 Results stored in: collected-metrics-tuned/"

cd ~/kube-burner-configs

echo "🔍 Analyzing tuned performance results..."
echo ""

# Use Python script for clean, color-coded analysis
python3 ~/low-latency-performance-workshop/scripts/analyze-performance.py --single collected-metrics-tuned

echo ""
echo "✅ Tuned performance analysis completed!"
echo ""
echo "💡 The enhanced analysis provides:"
echo "   � Educational context explaining what each metric means"
echo "   🎯 Performance explanations (why some metrics are slower)"
echo "   🚀 Color-coded results: Excellent/Good/Needs Attention"
echo "   📋 Suggested next steps based on your results"

cd ~/kube-burner-configs

echo "📊 Comparing baseline vs tuned performance..."
echo ""

# Use module-specific analysis script for clean Module 4 results
echo "🎯 Module 4 Focused Analysis (Container Performance Only)..."
python3 ~/low-latency-performance-workshop/scripts/module-specific-analysis.py 4

echo ""
echo "📄 Generating Module 4 Performance Report..."
echo "   🎯 Focus: Container performance optimization (baseline vs tuned)"
echo "   📊 Scope: CPU isolation and HugePages impact on pod startup"
echo ""

# Generate Module 4 specific markdown report (baseline vs tuned only)
REPORT_FILE="module4-performance-comparison-$(date +%Y%m%d-%H%M).md"
python3 ~/low-latency-performance-workshop/scripts/analyze-performance.py \
    --baseline collected-metrics \
    --tuned collected-metrics-tuned \
    --report "$REPORT_FILE"

echo ""
echo "💡 Module 4 Analysis Scope:"
echo "   ✅ Baseline container performance (from Module 3)"
echo "   ✅ Tuned container performance (from Module 4)"
echo "   ⚠️  Note: If VMI data exists from Module 5, it will also be shown"
echo "   🎯 Focus on the 'Performance Comparison' section for Module 4 results"
echo "   ℹ️  Comprehensive analysis across all modules happens in Module 6"

echo ""
echo "📚 How to Read the Analysis:"
echo "   1. Individual test sections show raw performance data"
echo "   2. 'Performance Comparison' section shows Module 4 improvements"
echo "   3. VMI data (if shown) is for reference - focus on container metrics"

echo ""
echo "📊 Performance Comparison Summary:"
echo "=================================="
if [ -f "$REPORT_FILE" ]; then
    # Display key sections of the report
    head -30 "$REPORT_FILE"
    echo ""
    echo "📄 Full report available at: $REPORT_FILE"
else
    echo "⚠️  Report generation failed - check if both baseline and tuned metrics exist"
fi

echo ""
echo "✅ Performance comparison completed!"
echo ""
echo "💡 The comparison analysis explains:"
echo "   � What P99/P95/P50 improvements mean for your workloads"
echo "   🎯 Why scheduling became instant (0ms) with CPU isolation"
echo "   ⚖️ Why container operations may be slower (expected trade-off)"
echo "   🏆 Overall assessment of your performance tuning effectiveness"

# Load cluster configuration
source /tmp/cluster-config

echo "=== Validating kube-burner Test Pod Scheduling ==="
echo "Cluster type: $CLUSTER_TYPE"
echo "Target node: $TARGET_NODE"

# Check if tuned test pods were created successfully
echo ""
echo "📋 Tuned Test Pod Status:"
TUNED_PODS=$(oc get pods -A -l app=tuned-test --no-headers 2>/dev/null | wc -l)
if [ $TUNED_PODS -gt 0 ]; then
    echo "   ✅ Found $TUNED_PODS tuned test pods"
    echo ""
    echo "   📊 Pod Distribution by Node:"
    oc get pods -A -l app=tuned-test -o wide --no-headers | awk '{print $8}' | sort | uniq -c | sed 's/^/      /'

    echo ""
    echo "   📊 Pod Status Summary:"
    oc get pods -A -l app=tuned-test --no-headers | awk '{print $4}' | sort | uniq -c | sed 's/^/      /'

    # Check for any failed pods
    FAILED_PODS=$(oc get pods -A -l app=tuned-test --no-headers | grep -v "Running\|Completed" | wc -l)
    if [ $FAILED_PODS -gt 0 ]; then
        echo ""
        echo "   ⚠️  Found $FAILED_PODS pods not in Running/Completed state:"
        oc get pods -A -l app=tuned-test --no-headers | grep -v "Running\|Completed" | sed 's/^/      /'
    fi
else
    echo "   ⚠️  No tuned test pods found - this may indicate scheduling issues"
fi

# Check baseline test pods as well
echo ""
echo "📋 Baseline Test Pod Status:"
BASELINE_PODS=$(oc get pods -A -l app=baseline-test --no-headers 2>/dev/null | wc -l)
if [ $BASELINE_PODS -gt 0 ]; then
    echo "   ✅ Found $BASELINE_PODS baseline test pods"
    echo "   📊 Baseline Pod Status Summary:"
    oc get pods -A -l app=baseline-test --no-headers | awk '{print $4}' | sort | uniq -c | sed 's/^/      /'
else
    echo "   ℹ️  No baseline test pods found (may have been cleaned up)"
fi

# Load cluster configuration
source /tmp/cluster-config

echo "=== Testing Cluster Responsiveness ==="

# Create a simple test pod to verify scheduling works
echo "🧪 Creating test pod to verify cluster functionality..."

cat << EOF | oc apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: cluster-health-test
  namespace: default
  labels:
    app: cluster-health-test
spec:
  containers:
  - name: test-container
    image: registry.redhat.io/ubi8/ubi-minimal:latest
    command: ["sleep"]
    args: ["60"]
    resources:
      requests:
        memory: "32Mi"
        cpu: "50m"
      limits:
        memory: "64Mi"
        cpu: "100m"
  restartPolicy: Never
EOF

# Wait for pod to be scheduled and running
echo "⏱️  Waiting for test pod to start..."
oc wait --for=condition=Ready pod/cluster-health-test --timeout=60s -n default

if [ $? -eq 0 ]; then
    echo "   ✅ Test pod started successfully - cluster is responsive"

    # Check which node it was scheduled on
    TEST_POD_NODE=$(oc get pod cluster-health-test -n default -o jsonpath='{.spec.nodeName}')
    echo "   📍 Test pod scheduled on node: $TEST_POD_NODE"

    # Clean up test pod
    oc delete pod cluster-health-test -n default --ignore-not-found=true
else
    echo "   ❌ Test pod failed to start - cluster may have scheduling issues"
    echo "   🔍 Pod events:"
    oc describe pod cluster-health-test -n default | grep -A 10 "Events:" | sed 's/^/      /'
fi

# Load cluster configuration
source /tmp/cluster-config

echo "=== Creating Performance Tuning Revert Script ==="

# Create revert script
cat > ~/revert-performance-tuning.sh << 'EOF'
#!/bin/bash

# Load cluster configuration
if [ -f /tmp/cluster-config ]; then
    source /tmp/cluster-config
    echo "=== Reverting Performance Tuning ==="
    echo "Cluster type: $CLUSTER_TYPE"
    echo "Profile name: $PROFILE_NAME"
    echo "Target node: $TARGET_NODE"
else
    echo "❌ Cluster configuration not found. Cannot revert automatically."
    echo "💡 You can manually delete performance profiles with:"
    echo "   oc get performanceprofile"
    echo "   oc delete performanceprofile <profile-name>"
    exit 1
fi

# Delete the Performance Profile
echo ""
echo "🗑️  Removing Performance Profile: $PROFILE_NAME"
if oc get performanceprofile $PROFILE_NAME >/dev/null 2>&1; then
    oc delete performanceprofile $PROFILE_NAME
    echo "   ✅ Performance Profile deleted"
else
    echo "   ℹ️  Performance Profile not found (may already be deleted)"
fi

# Remove custom Machine Config Pool (if created)
if [ "$CLUSTER_TYPE" = "MULTI_NODE" ]; then
    echo ""
    echo "🗑️  Removing worker-rt Machine Config Pool..."
    if oc get mcp worker-rt >/dev/null 2>&1; then
        oc delete mcp worker-rt
        echo "   ✅ worker-rt Machine Config Pool deleted"
    else
        echo "   ℹ️  worker-rt Machine Config Pool not found"
    fi

    # Remove worker-rt label from nodes
    echo ""
    echo "🏷️  Removing worker-rt labels from nodes..."
    oc label nodes -l node-role.kubernetes.io/worker-rt node-role.kubernetes.io/worker-rt- --ignore-not-found=true

elif [ "$CLUSTER_TYPE" = "MULTI_MASTER" ]; then
    echo ""
    echo "🗑️  Removing master-rt Machine Config Pool..."
    if oc get mcp master-rt >/dev/null 2>&1; then
        oc delete mcp master-rt
        echo "   ✅ master-rt Machine Config Pool deleted"
    else
        echo "   ℹ️  master-rt Machine Config Pool not found"
    fi

    # Remove master-rt label from nodes
    echo ""
    echo "🏷️  Removing master-rt labels from nodes..."
    oc label nodes -l node-role.kubernetes.io/master-rt node-role.kubernetes.io/master-rt- --ignore-not-found=true
fi

echo ""
echo "⏳ Waiting for nodes to revert to standard kernel..."
echo "   💡 This process will take 10-15 minutes as nodes reboot"
echo "   📊 Monitor progress with: watch 'oc get nodes; oc get mcp'"

# Wait for machine config pools to be updated
if [ "$CLUSTER_TYPE" = "SNO" ]; then
    echo "   🔄 Waiting for master MCP to update..."
    oc wait --for=condition=Updated mcp/master --timeout=1200s
elif [ "$CLUSTER_TYPE" = "MULTI_NODE" ]; then
    echo "   🔄 Waiting for worker MCP to update..."
    oc wait --for=condition=Updated mcp/worker --timeout=1200s
else
    echo "   🔄 Waiting for master MCP to update..."
    oc wait --for=condition=Updated mcp/master --timeout=1200s
fi

echo ""
echo "✅ Performance tuning revert completed!"
echo "🔍 Verify with: oc debug node/$TARGET_NODE -- chroot /host uname -r"
echo "   (Should show standard kernel without 'rt')"

EOF

chmod +x ~/revert-performance-tuning.sh

echo "✅ Revert script created: ~/revert-performance-tuning.sh"
echo ""
echo "💡 To revert performance tuning later, run:"
echo "   ~/revert-performance-tuning.sh"
echo ""
echo "⚠️  Note: Reverting will cause nodes to reboot back to standard kernel"

echo "=== Performance Tuning Revert Decision ==="
echo ""
echo "🤔 Do you want to revert the performance tuning now?"
echo ""
echo "✅ Keep Performance Tuning if:"
echo "   - Cluster is stable and responsive"
echo "   - Test pods are scheduling successfully"
echo "   - You want to continue with optimized performance"
echo ""
echo "🔄 Revert Performance Tuning if:"
echo "   - Experiencing cluster stability issues"
echo "   - Pods are failing to schedule"
echo "   - Want to continue workshop with standard settings"
echo ""
echo "💡 You can always re-apply performance tuning later by re-running this module"
echo ""
echo "To revert now, run: ~/revert-performance-tuning.sh"
echo "To keep current settings, continue to the next step"

# Load cluster configuration
source /tmp/cluster-config

echo "=== Creating Performance Tuning Revert Script ==="

# Create revert script
cat > ~/revert-performance-tuning.sh << 'EOF'
#!/bin/bash

# Load cluster configuration
if [ -f /tmp/cluster-config ]; then
    source /tmp/cluster-config
    echo "=== Reverting Performance Tuning ==="
    echo "Cluster type: $CLUSTER_TYPE"
    echo "Profile name: $PROFILE_NAME"
    echo "Target node: $TARGET_NODE"
else
    echo "❌ Cluster configuration not found. Attempting manual cleanup..."
    PROFILE_NAME=$(oc get performanceprofile -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
    if [ -z "$PROFILE_NAME" ]; then
        echo "No Performance Profiles found to delete."
        exit 0
    fi
fi

# Delete the Performance Profile
echo ""
echo "🗑️  Removing Performance Profile: $PROFILE_NAME"
if oc get performanceprofile $PROFILE_NAME >/dev/null 2>&1; then
    oc delete performanceprofile $PROFILE_NAME
    echo "   ✅ Performance Profile deleted"
else
    echo "   ℹ️  Performance Profile not found (may already be deleted)"
fi

# Remove custom Machine Config Pool and labels (if created)
if [ "$CLUSTER_TYPE" = "MULTI_NODE" ]; then
    echo ""
    echo "🗑️  Cleaning up worker-rt configuration..."
    oc delete mcp worker-rt --ignore-not-found=true
    oc label nodes -l node-role.kubernetes.io/worker-rt node-role.kubernetes.io/worker-rt- --ignore-not-found=true
    echo "   ✅ Worker-rt configuration cleaned up"

elif [ "$CLUSTER_TYPE" = "MULTI_MASTER" ]; then
    echo ""
    echo "🗑️  Cleaning up master-rt configuration..."
    oc delete mcp master-rt --ignore-not-found=true
    oc label nodes -l node-role.kubernetes.io/master-rt node-role.kubernetes.io/master-rt- --ignore-not-found=true
    echo "   ✅ Master-rt configuration cleaned up"
fi

echo ""
echo "⏳ Waiting for nodes to revert to standard kernel..."
echo "   💡 This process will take 10-15 minutes as nodes reboot"
echo "   📊 Monitor progress with: watch 'oc get nodes; oc get mcp'"

# Wait for machine config pools to be updated
if [ "$CLUSTER_TYPE" = "SNO" ] || [ "$CLUSTER_TYPE" = "MULTI_MASTER" ]; then
    echo "   🔄 Waiting for master MCP to update..."
    oc wait --for=condition=Updated mcp/master --timeout=1200s
elif [ "$CLUSTER_TYPE" = "MULTI_NODE" ]; then
    echo "   🔄 Waiting for worker MCP to update..."
    oc wait --for=condition=Updated mcp/worker --timeout=1200s
fi

echo ""
echo "✅ Performance tuning revert completed!"
echo "🔍 Verify standard kernel with:"
echo "   oc debug node/\$(oc get nodes -o jsonpath='{.items[0].metadata.name}') -- chroot /host uname -r"
echo "   (Should show standard kernel without 'rt')"

EOF

chmod +x ~/revert-performance-tuning.sh

echo "✅ Revert script created: ~/revert-performance-tuning.sh"
echo ""
echo "💡 To revert performance tuning at any time, run:"
echo "   ~/revert-performance-tuning.sh"
echo ""
echo "⚠️  Note: Reverting will cause nodes to reboot back to standard kernel"