Module 5: Low-Latency Virtualization

  # Check the HyperConverged operator status
oc get hyperconverged -n openshift-cnv

  # Verify virtualization components are running
oc get pods -n openshift-cnv --field-selector=status.phase=Running | head -10

  # Check if KVM virtualization is available on the cluster
oc get nodes -o jsonpath='{.items[*].status.allocatable.devices\.kubevirt\.io/kvm}' | grep -q "1k" && echo "✅ KVM available on cluster nodes" || echo "❌ KVM not available"

  # Verify the operator CSV status
oc get csv -n openshift-cnv | grep kubevirt-hyperconverged

  # Check available VM templates
echo "Available Fedora VM templates:"
oc get templates -n openshift --field-selector metadata.name=fedora-server-small

  # Check cluster node configuration
echo "--- Cluster Node Information ---"
oc get nodes -o wide

  # Check available CPU resources
echo ""
echo "--- CPU Resources ---"
oc debug node/$(oc get nodes -o jsonpath='{.items[0].metadata.name}') -- chroot /host nproc

  # Check if Fedora VM DataSource is available
echo ""
echo "--- Available VM DataSources ---"
oc get datasource -n openshift-virtualization-os-images | grep fedora

# Check if performance profile exists (from Module 4)
echo "--- Performance Profile Status ---"
PERF_PROFILES=$(oc get performanceprofile --no-headers 2>/dev/null | wc -l)
if [ "$PERF_PROFILES" -gt 0 ]; then
    echo "✅ Performance profile found:"
    oc get performanceprofile -o custom-columns=NAME:.metadata.name,ISOLATED:.spec.cpu.isolated,RESERVED:.spec.cpu.reserved

    # Get current HugePages from Performance Profile
    echo ""
    echo "--- Current HugePages Configuration ---"
    PROFILE_NAME=$(oc get performanceprofile -o jsonpath='{.items[0].metadata.name}')
    HUGEPAGES_COUNT=$(oc get performanceprofile "$PROFILE_NAME" -o jsonpath='{.spec.hugepages.pages[0].count}' 2>/dev/null || echo "0")
    echo "Performance Profile HugePages: ${HUGEPAGES_COUNT}GB"

    if [ "$HUGEPAGES_COUNT" -lt 8 ]; then
        echo ""
        echo "⚠️  Current HugePages (${HUGEPAGES_COUNT}GB) may be insufficient for Module 5"
        echo "   Module 4 allocates minimal HugePages (1GB) for demonstration"
        echo "   Module 5 needs more HugePages to run multiple VMs"
        echo ""
        echo "💡 Recommended HugePages for Module 5:"
        echo "   • SNO (64GB+ RAM): 8-16GB HugePages"
        echo "   • Multi-Node (64GB+ RAM): 16-24GB HugePages"
        echo ""
        echo "   The next step will update HugePages automatically!"
    else
        echo "✅ HugePages sufficient for Module 5 (${HUGEPAGES_COUNT}GB)"
    fi
else
    echo "⚠️  No performance profile found"
    echo "   This is expected if Module 4 hasn't been completed yet"
    echo "   VMI tests will use default cluster resources"
    echo ""
    echo "💡 Want to see enhanced VM performance?"
    echo "   You can go back to Module 4 to configure performance profiles"
    echo "   This will enable:"
    echo "   • CPU isolation and dedicated CPU placement for VMs"
    echo "   • HugePages for reduced memory latency"
    echo "   • NUMA alignment for optimal performance"
    echo "   • Significant improvement in VMI startup times"
    echo ""
    echo "   After completing Module 4, return here to see the performance difference!"
fi

# Update HugePages to support multiple VMs
bash ~/low-latency-performance-workshop/scripts/module05-update-hugepages.sh

# Validate that your cluster has sufficient resources for VMI testing
bash ~/low-latency-performance-workshop/scripts/module05-validate-vmi-resources.sh

  # Create a namespace for our VM testing
oc new-project vmi-performance-test || oc project vmi-performance-test

  # Clean up any existing VMs to avoid PVC conflicts
echo "🧹 Cleaning up any existing performance test VMs..."
oc delete vm --selector=app=vmi-performance-test --ignore-not-found=true
oc delete dv --selector=app=vmi-performance-test --ignore-not-found=true

  # Wait a moment for cleanup to complete
sleep 5

  # Create a Fedora VM using the existing template with performance optimizations
  # Generate unique name to avoid PVC conflicts
VM_NAME="fedora-perf-$(date +%s)"
echo "Creating VM with unique name: $VM_NAME"

cat << EOF | oc apply -f -
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: $VM_NAME
  labels:
    app: vmi-performance-test
    vm.kubevirt.io/template: fedora-server-small
spec:
  dataVolumeTemplates:
  - apiVersion: cdi.kubevirt.io/v1beta1
    kind: DataVolume
    metadata:
      name: $VM_NAME
    spec:
      sourceRef:
        kind: DataSource
        name: fedora
        namespace: openshift-virtualization-os-images
      storage:
        resources:
          requests:
            storage: 30Gi
  runStrategy: Manual
  template:
    metadata:
      labels:
        kubevirt.io/domain: $VM_NAME
        kubevirt.io/size: small
    spec:
      domain:
        cpu:
          cores: 2
          sockets: 1
          threads: 1
          # Enable performance features if performance profile exists
          dedicatedCpuPlacement: false  # Will be enabled conditionally
          model: host-model  # More compatible than host-passthrough
        memory:
          guest: 2Gi
          # HugePages will be enabled conditionally based on availability
        devices:
          disks:
          - disk:
              bus: virtio
            name: rootdisk
          - disk:
              bus: virtio
            name: cloudinitdisk
          interfaces:
          - masquerade: {}
            model: virtio
            name: default
          rng: {}
        features:
          smm:
            enabled: true
        firmware:
          bootloader:
            efi: {}
      networks:
      - name: default
        pod: {}
      terminationGracePeriodSeconds: 180
      volumes:
      - dataVolume:
          name: $VM_NAME
        name: rootdisk
      - cloudInitNoCloud:
          userData: |
            #cloud-config
            user: fedora
            password: workshop123
            chpasswd: { expire: False }
            packages:
              - qemu-guest-agent
            runcmd:
              - systemctl enable --now qemu-guest-agent
              - echo "VM ready for performance testing" > /tmp/vm-ready
        name: cloudinitdisk
EOF

echo "✅ Fedora VM '$VM_NAME' created for performance testing"

  # Verify the VM and DataVolume were created
echo ""
echo "📋 Verifying VM creation:"
oc get vm $VM_NAME
echo ""
echo "📋 Verifying DataVolume creation:"
oc get dv $VM_NAME

  # Check for any PVC binding issues
echo ""
echo "📋 Checking for PVC issues:"
if oc get events -n vmi-performance-test | grep -i "bound incorrectly\|pvc.*conflict" >/dev/null 2>&1; then
    echo "⚠️  PVC binding issues detected. This may be due to duplicate VM names."
    echo "   The cleanup step above should have resolved this."
    echo "   If issues persist, check: oc get events -n vmi-performance-test"
else
    echo "✅ No PVC binding issues detected"
fi

  # Compare the two approaches in your cluster
echo "--- VirtualMachine Objects (Management Layer) ---"
oc get VirtualMachine -A
echo ""
echo "--- VirtualMachineInstance Objects (Running VMs) ---"
oc get VirtualMachineInstance -A
echo ""
echo "--- Owner Relationships ---"
echo "VM-managed VMI (has owner reference):"
oc get vmi fedora-perf-1759292486 -n vmi-performance-test -o jsonpath='{.metadata.ownerReferences[0].kind}' 2>/dev/null && echo " ← Managed by VirtualMachine" || echo "No owner reference"

echo ""
echo "Direct VMI (no owner reference):"
oc get vmi fedora-vmi-0-1 -n vmi-latency-test-0 -o jsonpath='{.metadata.ownerReferences}' 2>/dev/null
if [ $? -eq 0 ] && [ -n "$(oc get vmi fedora-vmi-0-1 -n vmi-latency-test-0 -o jsonpath='{.metadata.ownerReferences}' 2>/dev/null)" ]; then
    echo "Has owner reference"
else
    echo "No owner reference ← Created directly by kube-burner"
fi

cd ~/kube-burner-configs

cat << EOF > vmi-latency-config.yml
global:
  measurements:
    - name: vmiLatency
      thresholds:
        - conditionType: VMIRunning
          metric: P99
          threshold: 90000ms  # Increased for SNO environment
        - conditionType: VMIScheduled
          metric: P99
          threshold: 60000ms  # Increased for SNO environment

metricsEndpoints:
  - indexer:
      type: local
      metricsDirectory: collected-metrics-vmi

jobs:
  - name: vmi-latency-test
    jobType: create
    jobIterations: 5  # Reduced for SNO environment
    namespace: vmi-latency-test
    namespacedIterations: true
    cleanup: false
    podWait: false
    waitWhenFinished: true
    verifyObjects: true
    errorOnVerify: false
    objects:
      - objectTemplate: fedora-vmi.yml
        replicas: 2  # Small scale for SNO
EOF

  # Create VMI template using containerDisk for faster, ephemeral testing
  # This approach is ideal for performance testing as it doesn't require PVC provisioning
echo "Creating Fedora VMI template for kube-burner testing"

cat << EOF > fedora-vmi.yml
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
  name: fedora-vmi-{{.Iteration}}-{{.Replica}}
  labels:
    app: vmi-latency-test
    iteration: "{{.Iteration}}"
spec:
  # No nodeSelector for SNO - will schedule on the single node
  domain:
    cpu:
      cores: 1
      sockets: 1
      threads: 1
      # Performance features will be enabled conditionally
      # Using host-model instead of host-passthrough for better compatibility
      model: host-model
    memory:
      guest: 2Gi  # Minimum required for Fedora
      # HugePages will be added conditionally if available
    devices:
      disks:
      - name: containerdisk
        disk:
          bus: virtio
      - name: cloudinitdisk
        disk:
          bus: virtio
      interfaces:
      - name: default
        masquerade: {}
        model: virtio
      rng: {}
    features:
      smm:
        enabled: true
    firmware:
      bootloader:
        efi: {}
  networks:
  - name: default
    pod: {}
  terminationGracePeriodSeconds: 180
  volumes:
  - name: containerdisk
    containerDisk:
      image: quay.io/containerdisks/fedora:latest
  - name: cloudinitdisk
    cloudInitNoCloud:
      userData: |
        #cloud-config
        user: fedora
        password: workshop123
        chpasswd: { expire: False }
        bootcmd:
          - "echo 'Fedora VMI started at' \$(date) > /tmp/vmi-start-time"
EOF

# Generate optimized VMI configuration
bash ~/low-latency-performance-workshop/scripts/module05-configure-vmi.sh

  # Clean up any existing VMI test resources to avoid PVC conflicts
echo "🧹 Cleaning up any existing VMI test resources..."
oc delete vmi --selector=app=vmi-latency-test --all-namespaces --ignore-not-found=true
oc delete dv --selector=app=vmi-latency-test --all-namespaces --ignore-not-found=true

  # Wait for cleanup to complete
sleep 5

echo "✅ Cleanup completed"

  # Execute the VMI latency test with containerDisk approach
echo "Starting Fedora VMI latency performance test..."
echo "   Test approach: Direct VMI creation with containerDisk (no PVC provisioning)"
echo "   Test scale: 5 iterations × 2 replicas = 10 VMIs total"
echo "   Environment: Single Node OpenShift (SNO)"
echo "   Unique namespaces: vmi-latency-test-0 through vmi-latency-test-4"
echo ""

kube-burner init -c vmi-latency-config.yml --log-level=info

  # The test will:
  # 1. Create VMIs directly in each namespace using containerDisk
  # 2. Measure pure VMI startup latency (no storage provisioning overhead)
  # 3. Track VMI lifecycle phases from creation to running
  # 4. Generate performance metrics in collected-metrics-vmi/

  # View the key metrics from the test
echo "VMI Latency Test Results Summary:"
echo ""
echo "Key Metrics Measured:"
echo "• VMICreated: Time to create VMI object (should be ~0ms)"
echo "• VMIPending: Time VMI spends in Pending state"
echo "• VMIScheduling: Time to schedule VMI to a node"
echo "• VMIScheduled: Time until VMI is scheduled (containerDisk pull + pod creation)"
echo "• VMIRunning: Total time until VMI is fully running (includes OS boot)"
echo ""
echo "Expected Results for SNO Environment with containerDisk:"
echo "• VMIScheduled P99: ~30-45 seconds (container image pull + pod start)"
echo "• VMIRunning P99: ~45-60 seconds (full VM boot from containerDisk)"
echo "• VMIScheduling P99: <1 second (fast on SNO)"
echo ""
echo "📁 Detailed metrics saved in: collected-metrics-vmi/"
ls -la collected-metrics-vmi/

  # Watch VMIs being created (press Ctrl+C to exit watch)
echo "Monitoring VMI creation progress..."
echo "   Use Ctrl+C to exit the watch command when test completes"
echo ""

  # Watch VMIs and their launcher pods being created
watch -n 5 "echo '--- VMIs ---' && oc get vmi --all-namespaces --selector=app=vmi-latency-test && echo '' && echo '--- Launcher Pods ---' && oc get pods --all-namespaces --selector=kubevirt.io=virt-launcher | grep vmi-latency"

  # Comprehensive verification of VMI test results
echo "=================================================="
echo "📋 VMI Latency Test - Current Status"
echo "=================================================="
echo ""
echo "✅ VirtualMachine Objects (Management Layer):"
oc get VirtualMachine -A 2>/dev/null || echo "No VMs found"
echo ""
echo "✅ VirtualMachineInstance Objects (Running VMs):"
oc get VirtualMachineInstance -A 2>/dev/null || echo "No VMIs found"
echo ""
echo "=================================================="
echo "� Kube-burner Test Results"
echo "=================================================="
echo ""
echo "VMIs created by kube-burner test:"
oc get vmi --all-namespaces --selector=app=vmi-latency-test -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,PHASE:.status.phase,IP:.status.interfaces[0].ipAddress,READY:.status.conditions[?\(@.type==\"Ready\"\)].status 2>/dev/null || echo "No test VMIs found"
echo ""
echo "📋 DataVolume Status (should be empty with containerDisk):"
oc get dv --all-namespaces --selector=app=vmi-latency-test 2>/dev/null || echo "No DataVolumes found (expected with containerDisk)"
echo ""
echo "📋 VMI Launcher Pods:"
oc get pods --all-namespaces --selector=kubevirt.io=virt-launcher -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,NODE:.spec.nodeName 2>/dev/null | grep -E "NAMESPACE|vmi-latency" || echo "No launcher pods found"
echo ""
echo "=================================================="
echo "✅ Test Results Summary"
echo "=================================================="
TOTAL_VMIS=\$(oc get vmi --all-namespaces --selector=app=vmi-latency-test --no-headers 2>/dev/null | wc -l)
RUNNING_VMIS=\$(oc get vmi --all-namespaces --selector=app=vmi-latency-test --no-headers 2>/dev/null | grep -c "Running" || echo "0")
echo "Total VMIs created: \$TOTAL_VMIS"
echo "VMIs in Running phase: \$RUNNING_VMIS"
echo ""
if [ "\$RUNNING_VMIS" -eq 10 ]; then
    echo "🎉 SUCCESS! All 10 test VMIs are running!"
    echo "📊 This demonstrates direct VMI creation with containerDisk"
    echo "✅ No DataVolumes needed - faster startup for performance testing"
    echo ""
    echo "Key Observations:"
    echo "• All VMIs have IP addresses assigned"
    echo "• All VMIs are in Ready state"
    echo "• No PVC/DataVolume provisioning delays"
    echo "• Pure VMI startup latency measured"
elif [ "\$TOTAL_VMIS" -eq 10 ]; then
    echo "⚠️  All 10 VMIs created, \$RUNNING_VMIS are running"
    echo "   Some may still be pulling containerDisk images"
    echo "   Check: oc get pods --all-namespaces | grep virt-launcher"
else
    echo "⚠️  Expected 10 VMIs, found \$TOTAL_VMIS"
    echo "   Review kube-burner logs for errors"
    echo ""
    echo "💡 If VMIs failed, see troubleshooting section below"
fi

# Get detailed status of all VMIs
echo "=== VMI Status ==="
oc get vmi --all-namespaces --selector=app=vmi-latency-test

echo ""
echo "=== virt-launcher Pod Status ==="
oc get pods --all-namespaces --selector=kubevirt.io=virt-launcher | grep vmi-latency

echo ""
echo "=== Failed/OOMKilled Pods ==="
oc get pods --all-namespaces --selector=kubevirt.io=virt-launcher | grep -E "OOMKilled|Error|CrashLoop" || echo "No failed pods"

# Check if VMIs are OOMKilled due to insufficient HugePages
echo "=== Checking for OOMKilled VMIs ==="
OOMKILLED_COUNT=$(oc get pods --all-namespaces --selector=kubevirt.io=virt-launcher -o jsonpath='{.items[*].status.containerStatuses[*].lastState.terminated.reason}' 2>/dev/null | grep -o "OOMKilled" | wc -l)

if [ "$OOMKILLED_COUNT" -gt 0 ]; then
    echo "❌ Found $OOMKILLED_COUNT OOMKilled virt-launcher pods"
    echo ""
    echo "Root Cause: Insufficient HugePages for VMI memory + overhead"
    echo ""
    echo "Explanation:"
    echo "  • Each VMI needs: 2GB guest + 1GB virt-launcher overhead = 3GB total"
    echo "  • Test creates: 10 VMIs × 3GB = 30GB required"
    echo "  • Available HugePages: $(oc get node -o jsonpath='{.items[0].status.allocatable.hugepages-1Gi}' | sed 's/Gi//g')GB"
    echo ""
    echo "Solutions:"
    echo ""
    echo "  Option 1: Increase HugePages (Recommended)"
    echo "  ─────────────────────────────────────────"
    echo "  bash ~/low-latency-performance-workshop/scripts/module05-update-hugepages.sh"
    echo ""
    echo "  This will:"
    echo "  • Calculate optimal HugePages for your cluster"
    echo "  • Update Performance Profile"
    echo "  • Trigger node reboot (wait 10-15 minutes)"
    echo "  • Allocate sufficient HugePages for 10 VMIs"
    echo ""
    echo "  Option 2: Reduce Test Scale"
    echo "  ───────────────────────────"
    echo "  Edit ~/kube-burner-configs/vmi-latency-config.yml:"
    echo ""
    echo "  Current:"
    echo "    jobIterations: 5"
    echo "    replicas: 2"
    echo "    Total: 10 VMIs"
    echo ""
    echo "  Recommended for 16GB HugePages:"
    echo "    jobIterations: 2"
    echo "    replicas: 2"
    echo "    Total: 4 VMIs (fits in 16GB)"
    echo ""
    echo "  Then clean up and re-run:"
    echo "    oc delete vmi --selector=app=vmi-latency-test --all-namespaces"
    echo "    kube-burner init -c vmi-latency-config.yml"
    echo ""
else
    echo "✅ No OOMKilled pods found"
fi

# Detailed HugePages analysis
echo "=== HugePages Allocation Analysis ==="
echo ""

# Get HugePages from Performance Profile
PERF_PROFILE=$(oc get performanceprofile -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -n "$PERF_PROFILE" ]; then
    HUGEPAGES_COUNT=$(oc get performanceprofile "$PERF_PROFILE" -o jsonpath='{.spec.hugepages.pages[0].count}' 2>/dev/null)
    HUGEPAGES_SIZE=$(oc get performanceprofile "$PERF_PROFILE" -o jsonpath='{.spec.hugepages.pages[0].size}' 2>/dev/null)
    echo "Performance Profile: $PERF_PROFILE"
    echo "  Configured: ${HUGEPAGES_COUNT} × ${HUGEPAGES_SIZE} = ${HUGEPAGES_COUNT}GB"
fi

echo ""

# Get HugePages from node
NODE=$(oc get nodes -o jsonpath='{.items[0].metadata.name}')
HUGEPAGES_CAPACITY=$(oc get node "$NODE" -o jsonpath='{.status.capacity.hugepages-1Gi}' | sed 's/Gi//g')
HUGEPAGES_ALLOCATABLE=$(oc get node "$NODE" -o jsonpath='{.status.allocatable.hugepages-1Gi}' | sed 's/Gi//g')

echo "Node: $NODE"
echo "  Capacity: ${HUGEPAGES_CAPACITY}GB"
echo "  Allocatable: ${HUGEPAGES_ALLOCATABLE}GB"

echo ""
echo "VMI Capacity Calculation:"
echo "  • VMI memory requirement: 3GB per VMI (2GB guest + 1GB overhead)"
echo "  • Available HugePages: ${HUGEPAGES_ALLOCATABLE}GB"
echo "  • Max concurrent VMIs: ~$((HUGEPAGES_ALLOCATABLE / 3))"
echo "  • Test requires: 10 VMIs = 30GB"
echo ""

if [ "$HUGEPAGES_ALLOCATABLE" -ge 30 ]; then
    echo "✅ Sufficient HugePages for 10 VMIs"
elif [ "$HUGEPAGES_ALLOCATABLE" -ge 24 ]; then
    echo "⚠️  Sufficient for 8 VMIs, reduce test scale to 8"
elif [ "$HUGEPAGES_ALLOCATABLE" -ge 18 ]; then
    echo "⚠️  Sufficient for 6 VMIs, reduce test scale to 6"
elif [ "$HUGEPAGES_ALLOCATABLE" -ge 12 ]; then
    echo "⚠️  Sufficient for 4 VMIs, reduce test scale to 4"
else
    echo "❌ Insufficient HugePages, increase allocation to at least 24GB"
fi

# Check events for failed VMIs
echo "=== Recent VMI Events ==="
oc get events --all-namespaces --field-selector involvedObject.kind=VirtualMachineInstance --sort-by='.lastTimestamp' | tail -20

# Get logs from a failed virt-launcher pod
echo "=== virt-launcher Pod Logs (first failed pod) ==="
FAILED_POD=$(oc get pods --all-namespaces --selector=kubevirt.io=virt-launcher -o jsonpath='{.items[?(@.status.phase!="Running")].metadata.name}' | awk '{print $1}' | head -1)
FAILED_NS=$(oc get pods --all-namespaces --selector=kubevirt.io=virt-launcher -o jsonpath='{.items[?(@.status.phase!="Running")].metadata.namespace}' | awk '{print $1}' | head -1)

if [ -n "$FAILED_POD" ]; then
    echo "Pod: $FAILED_POD (namespace: $FAILED_NS)"
    echo ""
    oc logs -n "$FAILED_NS" "$FAILED_POD" --tail=50 2>/dev/null || echo "No logs available"
else
    echo "No failed pods found"
fi

cd ~/kube-burner-configs

  # Check what metrics were generated
echo "📊 VMI Latency Test Results:"
ls -la collected-metrics-vmi/

  # View the summary of VMI latency measurements
echo ""
echo "📋 VMI Latency Quantiles (Key Performance Indicators):"
echo "   All times in milliseconds (ms)"
echo ""
if [ -f "collected-metrics-vmi/vmiLatencyQuantilesMeasurement-vmi-latency-test.json" ]; then
    cat collected-metrics-vmi/vmiLatencyQuantilesMeasurement-vmi-latency-test.json | jq -r '.[] | "\(.quantileName) - P99: \(.P99)ms | P50: \(.P50)ms | Avg: \(.avg)ms"' | grep -v "VMReady" | sort
else
    echo "VMI latency quantiles file not found"
fi

  # Show job summary
echo ""
echo "📈 Test Execution Summary:"
if [ -f "collected-metrics-vmi/jobSummary.json" ]; then
    cat collected-metrics-vmi/jobSummary.json | jq -r '.[] | "Job: \(.jobConfig.name) | Status: \(if .passed then "✅ PASSED" else "❌ FAILED" end) | Duration: \(.elapsedTime)s | QPS: \(.achievedQps)"'
else
    echo "Job summary file not found"
fi

cd ~/kube-burner-configs

  # Analyze the detailed VMI latency measurements
echo "🔍 Detailed VMI Startup Phase Analysis:"
echo ""

if [ -f "collected-metrics-vmi/vmiLatencyMeasurement-vmi-latency-test.json" ]; then
    echo "VMI Startup Phases (in chronological order):"
    echo "1. VMICreated → VMIPending: Object creation time"
    echo "2. VMIPending → VMIScheduling: Waiting for scheduling"
    echo "3. VMIScheduling → VMIScheduled: Node assignment + pod creation"
    echo "4. VMIScheduled → VMIRunning: containerDisk pull + VM boot"
    echo ""

    # Show actual timing data
    echo "📊 Actual Timing Results (Average across all VMIs):"
    cat collected-metrics-vmi/vmiLatencyMeasurement-vmi-latency-test.json | jq -r '
        [.[] | {
            vmiCreated: .vmiCreatedLatency,
            vmiPending: .vmiPendingLatency,
            vmiScheduling: .vmiSchedulingLatency,
            vmiScheduled: .vmiScheduledLatency,
            vmiRunning: .vmiRunningLatency,
            podCreated: .podCreatedLatency,
            podScheduled: .podScheduledLatency,
            podInitialized: .podInitializedLatency,
            podContainersReady: .podContainersReadyLatency
        }] |
        {
            vmiCreated: ([.[].vmiCreated] | add / length),
            vmiPending: ([.[].vmiPending] | add / length),
            vmiScheduling: ([.[].vmiScheduling] | add / length),
            vmiScheduled: ([.[].vmiScheduled] | add / length),
            vmiRunning: ([.[].vmiRunning] | add / length),
            podCreated: ([.[].podCreated] | add / length),
            podScheduled: ([.[].podScheduled] | add / length),
            podInitialized: ([.[].podInitialized] | add / length),
            podContainersReady: ([.[].podContainersReady] | add / length)
        } |
        to_entries |
        .[] |
        "  \(.key): \(.value | floor)ms"
    '

    echo ""
    echo "🎯 Performance Analysis (containerDisk approach):"
    echo "• VMICreated should be ~0ms (object creation)"
    echo "• VMIScheduling should be <2000ms (fast scheduling on SNO)"
    echo "• VMIScheduled includes containerDisk image pull time (major component)"
    echo "• VMIRunning includes full Fedora boot time from containerDisk (~45-55s typical)"
    echo ""
    echo "💡 Key Insight: With containerDisk, most time is spent pulling the container"
    echo "   image and booting the OS. No PVC provisioning or DataVolume import delays!"

else
    echo "❌ VMI latency measurement file not found"
    echo "This may indicate the test didn't complete successfully"
fi

cd ~/kube-burner-configs

  # Use the main performance analyzer for VMI metrics
echo "🎓 Running VMI Performance Analysis..."
python3 ~/low-latency-performance-workshop/scripts/analyze-performance.py \
    --single collected-metrics-vmi

  # This analysis provides:
  # • VMI startup phase breakdown and timing analysis
  # • Performance bottleneck identification
  # • Statistical analysis of latency variations
  # • Comparison with performance thresholds
  # • Color-coded performance assessment

cd ~/kube-burner-configs

  # Generate comprehensive comparison between VMs and containers
echo "📊 VMI vs Container Performance Comparison..."

  # Check what metrics are available for comparison
BASELINE_AVAILABLE=false
TUNED_AVAILABLE=false

if [ -d "collected-metrics" ]; then
    echo "✅ Container baseline metrics found"
    BASELINE_AVAILABLE=true
fi

if [ -d "collected-metrics-tuned" ]; then
    echo "✅ Container tuned metrics found"
    TUNED_AVAILABLE=true
fi

if [ -d "collected-metrics-vmi" ]; then
    echo "✅ VMI metrics found"
else
    echo "❌ VMI metrics not found - check test execution above"
    exit 1
fi

echo ""

  # Module 5 focused analysis - VMI performance with intelligent container context
echo "🎯 Module 5 Focused Analysis (VMI Performance with Context)..."
python3 ~/low-latency-performance-workshop/scripts/module-specific-analysis.py 5

echo ""
echo "💡 Module 5 Learning Focus:"
echo "   🔍 VMI startup phases and timing"
echo "   ⚖️  Virtualization vs containerization trade-offs"
echo "   🎯 When to choose VMs vs containers for workloads"
if [ "$TUNED_AVAILABLE" = true ]; then
    echo "   🚀 How performance profiles benefit both VMs and containers"
else
    echo "   ℹ️  Performance profiles (Module 4) would improve both VMs and containers"
fi

echo ""
echo "📚 How to Read the Module 5 Analysis:"
echo "   1. Individual sections show raw performance for each test type"
echo "   2. VMI metrics (🖥️ section) are the focus of this module"
echo "   3. Container metrics provide context for comparison"
echo "   4. Look for VMI-specific phases: VMICreated → VMIPending → VMIScheduled → VMIRunning"

echo ""
echo "💡 This comparison explains:"
echo "• Why VMs take longer to start than containers (OS boot vs process start)"
echo "• The performance trade-offs of virtualization (isolation vs overhead)"
echo "• When to use VMs vs containers for different workloads"
echo "• How performance profiles affect both VMs and containers"

cd ~/kube-burner-configs

  # Generate a comprehensive markdown report with all available metrics
echo "Generating Comprehensive Performance Report..."

  # Determine what metrics are available and generate appropriate report
BASELINE_AVAILABLE=false
TUNED_AVAILABLE=false
VMI_AVAILABLE=false

[ -d "collected-metrics" ] && BASELINE_AVAILABLE=true
[ -d "collected-metrics-tuned" ] && TUNED_AVAILABLE=true
[ -d "collected-metrics-vmi" ] && VMI_AVAILABLE=true

  # Generate Module 5 specific report with available metrics
REPORT_FILE="module5-vmi-performance-report-$(date +%Y%m%d-%H%M).md"

echo "📄 Generating Module 5 VMI Performance Report..."
echo "   🎯 Focus: Virtual machine performance analysis"
echo "   📊 Context: VMI startup vs container performance"

if [ "$BASELINE_AVAILABLE" = true ] && [ "$TUNED_AVAILABLE" = true ] && [ "$VMI_AVAILABLE" = true ]; then
    echo "   📈 Scope: VMI + Container baseline + Container tuned"
    python3 ~/low-latency-performance-workshop/scripts/analyze-performance.py \
        --baseline collected-metrics \
        --tuned collected-metrics-tuned \
        --vmi collected-metrics-vmi \
        --report "$REPORT_FILE"
elif [ "$BASELINE_AVAILABLE" = true ] && [ "$VMI_AVAILABLE" = true ]; then
    echo "   📈 Scope: VMI + Container baseline"
    python3 ~/low-latency-performance-workshop/scripts/analyze-performance.py \
        --baseline collected-metrics \
        --vmi collected-metrics-vmi \
        --report "$REPORT_FILE"
elif [ "$VMI_AVAILABLE" = true ]; then
    echo "   📈 Scope: VMI standalone analysis"
    python3 ~/low-latency-performance-workshop/scripts/analyze-performance.py \
        --single collected-metrics-vmi \
        --report "$REPORT_FILE"
else
    echo "❌ No VMI performance metrics found for report generation"
    exit 1
fi

echo ""
echo "📄 Performance Report Generated: $REPORT_FILE"
echo "📊 Report Summary:"
if [ -f "$REPORT_FILE" ]; then
    head -20 "$REPORT_FILE"
    echo ""
    echo "💡 View the complete report: cat $REPORT_FILE"
else
    echo "❌ Report generation failed"
fi

# Check SR-IOV operator installation
oc get csv -n openshift-sriov-network-operator

# Verify SR-IOV operator pods
oc get pods -n openshift-sriov-network-operator

# Check if SR-IOV capable nodes are detected
oc get sriovnetworknodestates -n openshift-sriov-network-operator

# List available SR-IOV network node policies
oc get sriovnetworknodepolicy -n openshift-sriov-network-operator

# Check for SR-IOV networks configured for VMs
oc get sriovnetwork -n openshift-sriov-network-operator

cat << EOF | oc apply -f -
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetwork
metadata:
  name: vm-sriov-network
  namespace: openshift-sriov-network-operator
spec:
  resourceName: vm_sriov_net
  networkNamespace: default
  vlan: 100  # Optional: VLAN tagging
  capabilities: '{"ips": true, "mac": true}'
  # Important: This network will be used by VMs
  ipam: |
    {
      "type": "host-local",
      "subnet": "192.168.100.0/24",
      "rangeStart": "192.168.100.10",
      "rangeEnd": "192.168.100.100",
      "gateway": "192.168.100.1"
    }
EOF

cat << EOF | oc apply -f -
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: high-performance-vm-sriov
  namespace: default
spec:
  running: true
  template:
    metadata:
      labels:
        kubevirt.io/vm: high-performance-vm-sriov
    spec:
      domain:
        cpu:
          cores: 4
          dedicatedCpuPlacement: true  # Pin CPUs for low latency
        memory:
          hugepages:
            pageSize: 2Mi  # Use HugePages
          guest: 4Gi
        devices:
          disks:
            - name: containerdisk
              disk:
                bus: virtio
            - name: cloudinitdisk
              disk:
                bus: virtio
          interfaces:
            # Primary interface: Pod network (for management)
            - name: default
              masquerade: {}
            # Secondary interface: SR-IOV (for high-performance data plane)
            - name: sriov-net
              sriov: {}
      networks:
        # Pod network for management traffic
        - name: default
          pod: {}
        # SR-IOV network for data plane traffic
        - name: sriov-net
          multus:
            networkName: vm-sriov-network
      volumes:
        - name: containerdisk
          containerDisk:
            image: quay.io/containerdisks/fedora:latest
        - name: cloudinitdisk
          cloudInitNoCloud:
            userData: |
              #cloud-config
              password: fedora
              chpasswd: { expire: False }
EOF

# Wait for VM to be running
oc wait --for=condition=Ready vmi/high-performance-vm-sriov --timeout=300s

# Check VM network interfaces
oc get vmi high-performance-vm-sriov -o jsonpath='{.status.interfaces}' | jq

# Verify SR-IOV VF is attached
oc describe vmi high-performance-vm-sriov | grep -A 10 "Interfaces"

# Check that VM has both pod network and SR-IOV
echo "VM Network Configuration:"
oc get vmi high-performance-vm-sriov -o jsonpath='{.spec.networks}' | jq

# Access the VM console
virtctl console high-performance-vm-sriov

# Inside the VM, check network interfaces
ip addr show

# You should see:
# - eth0: Pod network interface (management)
# - eth1: SR-IOV interface (high-performance)

# Check SR-IOV interface details
ethtool -i eth1

# Test network performance (requires iperf3 installed)
# From another VM or pod, run iperf3 server
# Then from this VM: iperf3 -c <server-ip> -i 1 -t 30

# Test networking to the SR-IOV-enabled VM
python3 ~/low-latency-performance-workshop/scripts/module05-vmi-network-tester.py \
    --namespace default

# This will test connectivity to VMs including SR-IOV-enabled ones
# Expected results:
# - Pod network interface: 2-5ms latency
# - SR-IOV interface: <1ms latency (if tested directly)

cd ~/kube-burner-configs

cat << EOF > network-policy-latency-config.yml
global:
  measurements:
    - name: netpolLatency

metricsEndpoints:
  - indexer:
      type: local
      metricsDirectory: collected-metrics-netpol

jobs:
  # Job 1: Create pods and namespaces (reduced scale for SNO)
  - name: network-policy-setup
    jobType: create
    jobIterations: 3  # Reduced for SNO
    namespace: network-policy-perf
    namespacedIterations: true
    cleanup: false
    podWait: true
    waitWhenFinished: true
    verifyObjects: true
    errorOnVerify: false
    namespaceLabels:
      kube-burner.io/skip-networkpolicy-latency: "true"
    objects:
      - objectTemplate: network-test-pod.yml
        replicas: 2  # Reduced for SNO
        inputVars:
          containerImage: registry.redhat.io/ubi8/ubi-minimal:latest

  # Job 2: Apply network policies and test connectivity
  - name: network-policy-test
    jobType: create
    jobIterations: 3  # Reduced for SNO
    namespace: network-policy-perf
    namespacedIterations: false
    cleanup: false
    podWait: false
    waitWhenFinished: true
    verifyObjects: true
    errorOnVerify: false
    jobPause: 30s  # Reduced pause for faster testing
    objects:
      - objectTemplate: ingress-network-policy.yml
        replicas: 1  # Reduced for SNO
        inputVars:
          namespaces: 3  # Reduced for SNO
EOF

cat << EOF > network-test-pod.yml
apiVersion: v1
kind: Pod
metadata:
  name: network-test-pod-{{.Iteration}}-{{.Replica}}
  labels:
    app: network-test
    iteration: "{{.Iteration}}"
    replica: "{{.Replica}}"
spec:
  # No nodeSelector for SNO - will schedule on the single node
  containers:
  - name: network-test-container
    image: {{.containerImage}}
    command: ["/bin/bash"]
    args: ["-c", "microdnf install -y httpd && echo 'Hello from pod {{.Iteration}}-{{.Replica}}' > /var/www/html/index.html && httpd -D FOREGROUND"]
    ports:
    - containerPort: 80
      protocol: TCP
    resources:
      requests:
        memory: "128Mi"  # Increased for httpd
        cpu: "100m"
      limits:
        memory: "256Mi"
        cpu: "200m"
    readinessProbe:
      httpGet:
        path: /
        port: 80
      initialDelaySeconds: 10
      periodSeconds: 5
  restartPolicy: Never
EOF

cat << EOF > ingress-network-policy.yml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: ingress-policy-{{.Iteration}}-{{.Replica}}
spec:
  podSelector:
    matchLabels:
      app: network-test
  policyTypes:
  - Ingress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          name: network-policy-perf-{{.Iteration}}
    - podSelector:
        matchLabels:
          app: network-test
    ports:
    - protocol: TCP
      port: 80  # Updated to match httpd default port
  # Allow egress for DNS resolution and package installation
  - from: []
    ports:
    - protocol: TCP
      port: 53
    - protocol: UDP
      port: 53
EOF

  # Execute the network policy latency test adapted for SNO
echo "Starting network policy latency test..."
echo "   Test scale: 3 iterations × 2 replicas = 6 pods total"
echo "   Environment: Single Node OpenShift (SNO)"
echo ""

kube-burner init -c network-policy-latency-config.yml --log-level=info

  # This test will:
  # 1. Create pods in multiple namespaces (reduced scale for SNO)
  # 2. Apply network policies with ingress rules
  # 3. Measure network policy enforcement latency

  # Watch network policies being created (press Ctrl+C to exit)
echo "Monitoring network policy test progress..."
echo "   Use Ctrl+C to exit the watch command when test completes"
echo ""

watch -n 5 "echo '--- Network Policies ---' && oc get networkpolicy --all-namespaces | grep network-policy-perf && echo '' && echo '--- Test Pods ---' && oc get pods --all-namespaces | grep network-test"

  # Check final network policy status
echo "📋 Final Network Policy Status:"
oc get networkpolicy --all-namespaces | grep network-policy-perf

  # Check pod status
echo ""
echo "📋 Test Pod Status:"
oc get pods --all-namespaces | grep network-test

  # Check if pods are ready and accessible
echo ""
echo "📊 Pod Readiness:"
oc get pods --all-namespaces -o custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[0].ready,STATUS:.status.phase | grep network-test

# Compare VMs and containers comprehensively
python3 ~/low-latency-performance-workshop/scripts/module05-vm-vs-container-comparison.py

# Disable colored output for documentation
python3 ~/low-latency-performance-workshop/scripts/module05-vm-vs-container-comparison.py --no-color

# Test networking against all VMIs in the cluster
python3 ~/low-latency-performance-workshop/scripts/module05-vmi-network-tester.py

# Test VMIs in specific namespace
python3 ~/low-latency-performance-workshop/scripts/module05-vmi-network-tester.py \
    --namespace vmi-latency-test-0

# Skip educational explanations
python3 ~/low-latency-performance-workshop/scripts/module05-vmi-network-tester.py \
    --skip-explanation

cd ~/low-latency-performance-workshop/scripts

  # Run the educational network policy latency analyzer
echo "🔍 Analyzing Network Policy Performance Impact..."
python3 ~/low-latency-performance-workshop/scripts/module05-network-policy-analyzer.py \
    --metrics-dir ~/kube-burner-configs \
    --analysis-type latency

  # The script provides:
  # 1. Educational analysis of policy enforcement overhead
  # 2. Color-coded performance assessment
  # 3. Performance vs security trade-off explanations
  # 4. Recommendations for policy optimization

cd ~/low-latency-performance-workshop/scripts

  # Create detailed educational analysis with report generation
echo "📊 Generating Comprehensive Network Policy Analysis..."
python3 ~/low-latency-performance-workshop/scripts/module05-network-policy-analyzer.py \
    --metrics-dir ~/kube-burner-configs \
    --analysis-type comprehensive \
    --output-format educational

  # This educational analysis includes:
  # • Statistical analysis of policy enforcement latency
  # • Performance vs security trade-off explanations
  # • Best practices for low-latency network policies
  # • Detailed markdown report with optimization strategies
  # • Educational insights about CNI performance impact