Unable to read PCI BAR from kernel mode - c

Question

I have written some code to probe registers of an AMD Vega from userspace using a mmap of the /sys resource. This is working and I am able to read the register values without any problems. However when I implement the kernel mode version of this, all reads return 0xffffffff. I feel like I am missing something simple as the userspace code is returning expected values, where the kernel code fails to read the registers.
Here is the working userspace code:
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <stdint.h>
#define RES_PATH "/sys/devices/pci0000:40/0000:40:01.3/0000:42:00.0/0000:43:00.0/0000:44:00.0/resource5"
#define RES_SIZE (512*1024)
#define MP0_BASE 0x16000
#define mmMP0_SMN_C2PMSG_33 ((MP0_BASE + 0x61) * 4)
#define mmMP0_SMN_C2PMSG_64 ((MP0_BASE + 0x80) * 4)
#define mmMP0_SMN_C2PMSG_81 ((MP0_BASE + 0x91) * 4)
void * map;
uint32_t _preadl(const char * name, uint32_t reg)
{
uint32_t val = *(uint32_t*)(map + reg);
printf("readl %20s (0x%08x / 0x%08x) = 0x%08x\n", name, reg / 4, reg, val);
return val;
}
#define preadl(x) _preadl(#x, x)
int main(int argc, char * argv[])
{
int fd = open(RES_PATH, O_RDWR | O_SYNC, 0);
map = mmap(NULL, RES_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
printf("mmap = %p\n", map);
preadl(mmMP0_SMN_C2PMSG_33);
preadl(mmMP0_SMN_C2PMSG_64);
preadl(mmMP0_SMN_C2PMSG_81);
return 0;
}
And here is the non working kernel code:
static int reset_amdgpu_vega(struct pci_dev *dev, int probe)
{
#define MP0_BASE 0x16000
#define mmMP0_SMN_C2PMSG_33 ((MP0_BASE + 0x61) * 4)
#define mmMP0_SMN_C2PMSG_64 ((MP0_BASE + 0x80) * 4)
#define mmMP0_SMN_C2PMSG_81 ((MP0_BASE + 0x91) * 4)
resource_size_t mmio_base, mmio_size;
void __iomem *mmio;
int ret;
int i;
uint32_t val;
if (probe)
return 0;
mmio_base = pci_resource_start(dev, 5);
mmio_size = pci_resource_len(dev, 5);
mmio = pci_iomap(dev, 5, 0);
if (mmio == NULL) {
printk(KERN_ERR
"[reset_amdgpu_vega] failed to map the resource\n");
ret = -ENOMEM;
goto out;
}
printk(KERN_INFO "mmio: %llx %llx %lx\n",
mmio_base,
mmio_size,
(uintptr_t)mmio);
printk(KERN_INFO "regs: %08x %08x %08x\n",
mmMP0_SMN_C2PMSG_33,
mmMP0_SMN_C2PMSG_64,
mmMP0_SMN_C2PMSG_81);
printk(KERN_INFO "vals: %08x %08x %08x\n",
ioread32(mmio + mmMP0_SMN_C2PMSG_33),
ioread32(mmio + mmMP0_SMN_C2PMSG_64),
ioread32(mmio + mmMP0_SMN_C2PMSG_81));
pci_iounmap(dev, mmio);
ret = 0;
out:
return ret;
}

Answer 1

The problem was that before the pci reset quirk is called, the kernel turns off the PCI_COMMAND_MEMORY flag on the COMMAND register. By doing so the device is no longer readable. The solution is to re-enable the flag first:
pci_read_config_word(dev, PCI_COMMAND, &cfg);
cfg |= PCI_COMMAND_MEMORY;
pci_write_config_word(dev, PCI_COMMAND, cfg);