CXL devices are not listed by the cxl command

[ikitayama]

root@localhost:~/ndctl/build# ./cxl/cxl list -R
  Warning: no matching devices found

[]
root@localhost:~/ndctl/build# ./cxl/cxl list -M
  Warning: no matching devices found

[]
root@localhost:~/ndctl/build# lspci -mm
00:00.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Host bridge" -p00 "Red Hat, Inc." "Device 1100"
00:01.0 "SCSI storage controller" "Red Hat, Inc." "Virtio block device" -p00 "Red Hat, Inc." "Device 0002"
00:02.0 "Ethernet controller" "Intel Corporation" "82540EM Gigabit Ethernet Controller" -r03 -p00 "Red Hat, Inc." "QEMU Virtual Machine"
00:03.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Expander bridge" -p00 "Red Hat, Inc." "Device 1100"
00:04.0 "Host bridge" "Red Hat, Inc." "QEMU PCIe Expander bridge" -p00 "Red Hat, Inc." "Device 1100"
35:00.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
35:01.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
36:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a128" -p00 "" ""
36:00.1 "Serial bus controller [0c0b]" "Huawei Technologies Co., Ltd." "Device a123" -p00 "Red Hat, Inc." "Device 1100"
37:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
37:01.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
37:02.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
37:03.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
38:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100"
3a:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100"
bf:00.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
bf:01.0 "PCI bridge" "Intel Corporation" "Device 7075" -p00 "Intel Corporation" "Device 0000"
c0:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a128" -p00 "" ""
c0:00.1 "Serial bus controller [0c0b]" "Huawei Technologies Co., Ltd." "Device a123" -p00 "Red Hat, Inc." "Device 1100"
c1:00.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
c1:01.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
c1:02.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
c1:03.0 "PCI bridge" "Huawei Technologies Co., Ltd." "Device a129" -p00 "" ""
c2:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100"
c4:00.0 "CXL" "Intel Corporation" "Device 0d93" -r01 -p10 "Red Hat, Inc." "Device 1100"

[ikitayama]

Below is the QEMU options:

/home/realm/projects/qemu/build/qemu-system-aarch64 -machine virt,accel=tcg,cxl=on,highmem=on,compact-highmem=on,highmem-ecam=on,highmem-mmio=on -m 2048M,slots=0,maxmem=6144M -smp 2,sockets=1,cores=2,threads=1 -display none -nographic -drive if=pflash,format=raw,unit=0,file=AAVMF_CODE.fd,readonly=on -drive if=pflash,format=raw,unit=1,file=AAVMF_VARS.fd -drive file=root.img,format=raw,if=none,id=rootdisk -device virtio-blk-pci,drive=rootdisk,id=root-device -kernel ./mkosi.extra/boot/vmlinuz -initrd mkosi.extra/boot/initramfs-6.15.0-rc4-00040-g128ad8fa385b.img -append selinux=0 audit=0 console=tty0 console=ttyS0 root=PARTUUID=fb44336e-2709-4072-9fe8-ccd976eeb178 ignore_loglevel rw initcall_debug log_buf_len=20M memory_hotplug.memmap_on_memory=force cxl_acpi.dyndbg=+fplm cxl_pci.dyndbg=+fplm cxl_core.dyndbg=+fplm cxl_mem.dyndbg=+fplm cxl_pmem.dyndbg=+fplm cxl_port.dyndbg=+fplm cxl_region.dyndbg=+fplm cxl_test.dyndbg=+fplm cxl_mock.dyndbg=+fplm cxl_mock_mem.dyndbg=+fplm systemd.set_credential=agetty.autologin:root systemd.set_credential=login.noauth:yes -device e1000,netdev=net0,mac=52:54:00:12:34:56 -netdev user,id=net0,hostfwd=tcp::10022-:22 -cpu max -object memory-backend-file,id=cxl-mem0,share=on,mem-path=cxltest0.raw,size=256M -object memory-backend-file,id=cxl-mem1,share=on,mem-path=cxltest1.raw,size=256M -object memory-backend-file,id=cxl-mem2,share=on,mem-path=cxltest2.raw,size=256M -object memory-backend-file,id=cxl-mem3,share=on,mem-path=cxltest3.raw,size=256M -object memory-backend-file,id=cxl-lsa0,share=on,mem-path=lsa0.raw,size=128K -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=lsa1.raw,size=128K -object memory-backend-file,id=cxl-lsa2,share=on,mem-path=lsa2.raw,size=128K -object memory-backend-file,id=cxl-lsa3,share=on,mem-path=lsa3.raw,size=128K -device pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=53 -device pxb-cxl,id=cxl.1,bus=pcie.0,bus_nr=191 -device cxl-rp,id=hb0rp0,bus=cxl.0,chassis=0,slot=0,port=0 -device cxl-rp,id=hb0rp1,bus=cxl.0,chassis=0,slot=1,port=1 -device cxl-rp,id=hb1rp0,bus=cxl.1,chassis=0,slot=2,port=0 -device cxl-rp,id=hb1rp1,bus=cxl.1,chassis=0,slot=3,port=1 -device cxl-upstream,port=4,bus=hb0rp0,id=cxl-up0,multifunction=on,addr=0.0,sn=12345678 -device cxl-switch-mailbox-cci,bus=hb0rp0,addr=0.1,target=cxl-up0 -device cxl-upstream,port=4,bus=hb1rp0,id=cxl-up1,multifunction=on,addr=0.0,sn=12341234 -device cxl-switch-mailbox-cci,bus=hb1rp0,addr=0.1,target=cxl-up1 -device cxl-downstream,port=0,bus=cxl-up0,id=swport0,chassis=0,slot=4 -device cxl-downstream,port=1,bus=cxl-up0,id=swport1,chassis=0,slot=5 -device cxl-downstream,port=2,bus=cxl-up0,id=swport2,chassis=0,slot=6 -device cxl-downstream,port=3,bus=cxl-up0,id=swport3,chassis=0,slot=7 -device cxl-downstream,port=0,bus=cxl-up1,id=swport4,chassis=0,slot=8 -device cxl-downstream,port=1,bus=cxl-up1,id=swport5,chassis=0,slot=9 -device cxl-downstream,port=2,bus=cxl-up1,id=swport6,chassis=0,slot=10 -device cxl-downstream,port=3,bus=cxl-up1,id=swport7,chassis=0,slot=11 -device cxl-type3,bus=swport0,persistent-memdev=cxl-mem0,id=cxl-pmem0,lsa=cxl-lsa0 -device cxl-type3,bus=swport2,persistent-memdev=cxl-mem1,id=cxl-pmem1,lsa=cxl-lsa1 -device cxl-type3,bus=swport4,volatile-memdev=cxl-mem2,id=cxl-vmem2,lsa=cxl-lsa2 -device cxl-type3,bus=swport6,volatile-memdev=cxl-mem3,id=cxl-vmem3,lsa=cxl-lsa3 -M cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k,cxl-fmw.1.targets.0=cxl.0,cxl-fmw.1.targets.1=cxl.1,cxl-fmw.1.size=4G,cxl-fmw.1.interleave-granularity=8k -snapshot -object memory-backend-ram,id=mem0,size=2048M -numa node,nodeid=0,memdev=mem0, -numa cpu,node-id=0,socket-id=0 -numa dist,src=0,dst=0,val=10

[ikitayama]

The cxl command be is Debian provides or built on the fly list zero CXL devices.

root@localhost:~# lsmod|grep test
cxl_test               36864  0
cxl_mem                16384  1 cxl_test
cxl_pmem               24576  1 cxl_test
cxl_acpi               24576  1 cxl_test
cxl_port               16384  1 cxl_test
cxl_mock               20480  5 cxl_pmem,cxl_port,cxl_mem,cxl_test,cxl_acpi
cxl_core              208896  6 cxl_pmem,cxl_port,cxl_mem,cxl_test,cxl_acpi,cxl_mock

Any other things I should check?

[marc-hb]

Surprising, can you try cxl list --debug?

Also, can you try #185 and run cxl list immediately after boot?. That seems to make a difference for me.

[marc-hb]

Someone also recommended cxl list -i.

PS: none of this seems related to run_qemu. Please use https://lore.kernel.org/linux-cxl/ for questions like this.

[marc-hb]

none of this seems related to run_qemu.

I will re-open if you prove me wrong :-)