can't call VMXON on more than 1 processor - c

Question

I'm trying to build my own hypervisor for Linux (5.0.x kernel) on an Intel chip and I'm running into an odd problem.
Whenever I try to execute VMXON on more than one processor, it fails.
I made sure that VMX is enabled, that I allocated an aligned page and wrote VMCS REV ID into the VMXON region, but I'm not sure where the issue is.
This is my code:
vmx.c:
#include "vmx.h"
typedef struct vmstate {
bool vmx_enabled;
void *vmxon_region;
phys_addr_t vmxon_physical;
void *vmcs_region;
phys_addr_t vmcs_physical;
} vmstate;
static DEFINE_PER_CPU(vmstate*, cpu_vms);
static inline int VMXON(phys_addr_t phys){
uint8_t ret;
// TODO: Signal VMX to PT, to avoid PT crashes (Processor Trace)
__asm__ __volatile__ (
"vmxon %[pa]; setna %[ret]"
: [ret]"=rm"(ret)
: [pa]"m"(phys)
: "cc", "memory"
);
return ret;
}
static inline void VMXOFF(void){
__asm__ __volatile__("vmxoff" : : : "cc");
}
static void enable_vmx_operation_cr(void){
// Enable 14th bit in CR4
__write_cr4(__read_cr4() | 0x2000);
}
static void disable_vmx_operation_cr(void){
__write_cr4(__read_cr4() & ~(0x2000));
}
static vmstate *create_vmstate(void){
vmstate *vms = kzalloc(sizeof(vmstate), GFP_KERNEL);
vms->vmx_enabled = false;
return vms;
}
static void alloc_vmxon_region(vmstate *vms){
// TODO: respect physical width as set by the IA32_VMX_BASIC[48] bit for 32bit support
uint32_t vmcs_revid = 0;
uint32_t hi = 0;
void *vmxon_region = kmalloc(4096, GFP_KERNEL);
rdmsr_safe(MSR_IA32_VMX_BASIC, &vmcs_revid, &hi);
memcpy(vmxon_region, &vmcs_revid, 4);
vms->vmxon_region = vmxon_region;
vms->vmxon_physical = virt_to_phys(vmxon_region);
}
static void teardown_vmstate(vmstate *vms){
if(vms->vmxon_region)
kfree(vms->vmxon_region);
}
void vmx_teardown(void){
int i;
vmstate* vms;
for_each_possible_cpu(i){
vms = per_cpu(cpu_vms, i);
if(vms->vmx_enabled == true) {
VMXOFF();
vms->vmx_enabled = false;
}
disable_vmx_operation_cr();
teardown_vmstate(vms);
kfree(vms);
}
}
int vmx_setup(void){
int i;
vmstate* vms;
printk(KERN_INFO "NUM CPUS: %d", num_possible_cpus());
for_each_possible_cpu(i) {
// Allocate vmstate for every processor
per_cpu(cpu_vms, i) = create_vmstate();
vms = per_cpu(cpu_vms, i);
alloc_vmxon_region(vms);
enable_vmx_operation_cr();
if(VMXON(vms->vmxon_physical)){
printk(KERN_ALERT "VMXON operation failed!");
vms->vmx_enabled = false;
}
else
vms->vmx_enabled = true;
}
for_each_possible_cpu(i){
vms = per_cpu(cpu_vms, i);
if(vms->vmx_enabled == false) {
printk(KERN_ALERT "Tearing down after VMXON fail!");
vmx_teardown();
return -1;
}
}
return 0;
}
vmx_setup is called by a device open file operation:
static int hyper_dev_open(struct inode* inode, struct file *filep){
int err;
printk(KERN_INFO "Enabling VMX operation!\n");
if((err = vmx_setup()))
return err;
return 0;
}
When I execute VMXON on another processor, the carry flag is set to 1, zero flag is 0.
The driver works however if I add a VMXOFF() right after VMXON() so two VMX operations aren't enabled in parallel.
Any suggestions would help :)

Answer 1

for_each_possible_cpu simply iterates over the available CPUs; it doesn’t change execution to run on each CPU in turn. The entire loop runs on a single CPU.
Because of this, you are trying to execute vmxon repeatedly on the same CPU, which is why it is failing.