Zeroboot 启动优化分析：亚毫秒级 VM 沙箱实现

1
2
3

// 传统方式：需要分配和复制完整内存
let mem = allocate_memory(256 * 1024 * 1024);  // 分配 256MB
memcpy(mem, template_mem, 256 * 1024 * 1024);   // 复制整个模板

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

let fork_mem = unsafe {
    libc::mmap(
        ptr::null_mut(),
        snapshot.mem_size,
        libc::PROT_READ | libc::PROT_WRITE,
        libc::MAP_PRIVATE | libc::MAP_NORESERVE,  // CoW 映射
        memfd,
        0,
    )
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

// 1. sregs (Special Registers)
// 设置控制寄存器 CR4.OSXSAVE，启用 XSAVE 指令集
vcpu_fd.set_sregs(&snapshot.sregs)?;

// 2. XCRS (Extended Control Registers)
// XCR0 寄存器，控制哪些 SIMD 扩展被启用
vcpu_fd.set_xcrs(&snapshot.xcrs)?;

// 3. XSAVE (FPU/SSE/AVX 状态)
// 保存浮点寄存器、SSE 寄存器、AVX 寄存器的状态
vcpu_fd.set_xsave(&snapshot.xsave)?;

// 4. regs (General Registers)
// RAX, RBX, RCX, RDX, RSI, RDI, RSP, RBP, R8-R15
// RIP (指令指针), RFLAGS (标志寄存器)
vcpu_fd.set_regs(&snapshot.regs)?;

// 5. LAPIC (Local APIC)
// 本地高级可编程中断控制器状态
vcpu_fd.set_lapic(&snapshot.lapic)?;

// 6. MSRs (Model Specific Registers)
// CPU 特定的寄存器，如 SYSENTER_CS/EIP/ESP
for msr in &snapshot.msrs {
    vcpu_fd.set_msr(msr)?;
}

// 7. MP_STATE (Multiprocessor State)
// 设置为 RUNNABLE，让 CPU 开始执行
vcpu_fd.set_mp_state(kvm_mp_state { mp_state: KVM_MP_STATE_RUNNABLE });

1
2
3
4
5
6
7
8

let kvm = Kvm::new()?;                    // 打开 /dev/kvm
let vm_fd = kvm.create_vm()?;             // KVM_CREATE_VM
vm_fd.create_irq_chip()?;                 // KVM_CREATE_IRQCHIP
vm_fd.create_pit2(kvm_pit_config::default())?;  // KVM_CREATE_PIT2
let vcpu_fd = vm_fd.create_vcpu(0)?;      // KVM_CREATE_VCPU
vcpu_fd.set_sregs(&snapshot.sregs)?;      // KVM_SET_SREGS
// ... 更多状态恢复
vcpu_fd.run()?;                          // KVM_RUN

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

const COM1_PORT: u16 = 0x3F8;
const COM1_PORT_END: u16 = 0x3F8 + 7;

loop {
    match self.vcpu_fd.run()? {
        // Guest 写入串口（输出数据）
        VcpuExit::IoOut(port, data) => {
            if port >= COM1_PORT && port <= COM1_PORT_END {
                self.serial.write(port - COM1_PORT, data[0]);
            }
        }
        // Guest 读取串口（输入数据）
        VcpuExit::IoIn(port, data) => {
            if port >= COM1_PORT && port <= COM1_PORT_END {
                data[0] = self.serial.read(port - COM1_PORT);
            }
        }
        VcpuExit::Hlt => { /* CPU 停止，yield */ }
        _ => {}
    }
}

1
2
3
4
5

# Guest 中执行代码
echo "print('Hello')" > /dev/ttyS0

# Host 接收输出
# 从串口读取 → 返回给 Client

1
2
3
4

let fd = memfd_create(
    CStr::from_bytes_with_nul(b"zeroboot-memfd\0")?,
    MemfdFlags::MFD_CLOEXEC | MemfdFlags::MFD_ALLOW_SEALING
)?;

1
2
3

// 设置密封，防止写入和缩减
let seals = SealFlags::F_SEAL_WRITE | SealFlags::F_SEAL_SHRINK;
fcntl_add_seals(fd, seals)?;

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

pub fn create_snapshot_memfd(mem_ptr: *const u8, mem_size: usize) -> Result<i32> {
    let name = std::ffi::CString::new("zeroboot-snapshot").unwrap();
    let fd = unsafe { libc::memfd_create(name.as_ptr(), libc::MFD_CLOEXEC) };
    
    // 设置大小
    ftruncate(fd, mem_size as i64)?;
    
    // 写入快照数据
    let dst = mmap(fd, mem_size, PROT_WRITE, MAP_SHARED)?;
    std::ptr::copy_nonoverlapping(mem_ptr, dst, mem_size);
    
    Ok(fd)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

// 继承 memfd 文件描述符
let memfd = unsafe { File::from_raw_fd(3) };

// CoW 映射到虚拟机内存
let addr = mmap(
    memfd.as_raw_fd(),
    MAP_PRIVATE | MAP_NORESERVE
)?;

// 设置为虚拟机物理内存
vm.set_user_memory_region(addr, size)?;

1
2
3
4
5

// ❌ 错误：零初始化会导致问题
let mut irqchip = kvm_irqchip::default();
irqchip.chip_id = KVM_IRQCHIP_IOAPIC;
// memset(&irqchip, 0, sizeof(irqchip)) ← 问题所在
vm_fd.set_irqchip(&irqchip)?;  // 失败或导致不稳定

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

struct kvm_irqchip {
    union {
        struct kvm_pic pic;          // 8259 PIC (不需要)
        struct kvm_ioapic ioapic;    // IOAPIC (需要恢复)
        char chip[512];              // 原始字节
    };
};

struct kvm_ioapic {
    __u64 base_address;              // 0xFEC00000 (固定)
    __u32 ioregsel;                  // 寄存器选择
    __u32 id;                        // APIC ID
    __u32 irr;                       // 中断请求寄存器
    __u64 redirtbl[24];              // 重定向表 (核心！)
    // ... 其他字段
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

let mut irqchip = kvm_irqchip::default();
irqchip.chip_id = KVM_IRQCHIP_IOAPIC;

// ✅ 先获取内核默认值
vm_fd.get_irqchip(&mut irqchip)?;

// ✅ 只修改重定向表
for i in 0..24 {
    irqchip.chip.ioapic.redirtbl[i].bits = snapshot.ioapic_redirtbl[i];
}

// ✅ 写回修改后的状态
vm_fd.set_irqchip(&irqchip)?;

1
2

# Firecracker Cargo.toml
versionize = { version = "0.1.6" }

1
2
3
4
5
6
7
8
9

// IOAPIC 基地址是固定值 0xFEC00000
const IOAPIC_BASE_PATTERN: [u8; 8] = 0xFEC00000u64.to_le_bytes();

// 在 vmstate 中搜索锚点
fn find_ioapic_offset(vmstate: &[u8]) -> Option<usize> {
    vmstate
        .windows(8)
        .position(|w| w == IOAPIC_BASE_PATTERN)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

struct VmstateOffsets {
    sregs: usize,     // 总是 0
    xcrs: usize,      // sregs + sizeof(kvm_sregs)
    msrs: usize,      // xcrs + sizeof(kvm_xcrs)
    ioapic: usize,    // 从搜索得到的锚点位置
    // ... 其他字段
}

impl VmstateOffsets {
    fn detect(vmstate: &[u8]) -> Result<Self> {
        // 1. 固定偏移
        let sregs = 0;
        let xcrs = sregs + std::mem::size_of::<kvm_sregs>();
        
        // 2. 搜索锚点
        let ioapic = find_ioapic_offset(vmstate)
            .ok_or(Error::InvalidVmstate)?;
        
        // 3. 推导 msrs 位置
        // msrs 在 ioapic 之前，需要根据版本调整
        let msrs = ioapic - ESTIMATED_MSRS_SIZE;
        
        Ok(Self { sregs, xcrs, msrs, ioapic })
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

enum FirecrackerVersion {
    V1_0,
    V1_5,
    V1_7,
    Unknown,
}

impl VmstateOffsets {
    fn detect_with_version(vmstate: &[u8]) -> Result<(Self, FirecrackerVersion)> {
        let ioapic = find_ioapic_offset(vmstate)?;
        
        // 根据 ioapic 位置推断版本
        let version = match ioapic {
            144 => FirecrackerVersion::V1_0,
            152 => FirecrackerVersion::V1_5,
            152..=160 => FirecrackerVersion::V1_7,
            _ => FirecrackerVersion::Unknown,
        };
        
        // 根据版本选择解析策略
        let offsets = match version {
            FirecrackerVersion::V1_0 => Self::for_v1_0(),
            FirecrackerVersion::V1_5 | FirecrackerVersion::V1_7 => Self::for_v1_5(),
            FirecrackerVersion::Unknown => Self::detect_heuristically(vmstate)?,
        };
        
        Ok((offsets, version))
    }
}

1
2
3
4
5
6
7

fn parse_sregs(vmstate: &[u8], offset: usize) -> Result<kvm_sregs> {
    // kvm_sregs 是 C 结构体，可以直接从字节解析
    let bytes = &vmstate[offset..offset + std::mem::size_of::<kvm_sregs>()];
    unsafe {
        Ok(std::ptr::read(bytes.as_ptr() as *const kvm_sregs))
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

fn parse_msrs(vmstate: &[u8], offset: usize, count: usize) -> Result<Vec<kvm_msr_entry>> {
    // MSR 是变长列表，需要逐个解析
    let mut msrs = Vec::with_capacity(count);
    let mut pos = offset;
    
    for _ in 0..count {
        let msr = kvm_msr_entry {
            index: u32::from_le_bytes(read_bytes(&vmstate[pos..pos+4])?),
            data: u64::from_le_bytes(read_bytes(&vmstate[pos+8..pos+16])?),
            ..Default::default()
        };
        msrs.push(msr);
        pos += std::mem::size_of::<kvm_msr_entry>();
    }
    
    Ok(msrs)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

fn parse_vmstate(vmstate_path: &Path) -> Result<VmState> {
    let vmstate = std::fs::read(vmstate_path)?;
    
    // 1. 验证最小大小
    if vmstate.len() < MIN_VMSTATE_SIZE {
        return Err(Error::VmstateTooSmall);
    }
    
    // 2. 搜索锚点
    let ioapic_offset = find_ioapic_offset(&vmstate)
        .ok_or(Error::InvalidVmstate)?;
    
    // 3. 验证锚点位置合理性
    if ioapic_offset < MIN_CPU_STATE_SIZE {
        return Err(Error::InvalidVmstate);
    }
    
    // 4. 解析各字段
    let sregs = parse_sregs(&vmstate, 0)?;
    let xcrs = parse_xcrs(&vmstate, std::mem::size_of::<kvm_sregs>())?;
    // ... 其他字段
    
    // 5. 验证解析结果
    validate_sregs(&sregs)?;
    
    Ok(VmState { sregs, xcrs, ... })
}