Synthetic reproducer on Focal ===
Ubuntu Server cloud image: ---
SERIES=focal
wget https://cloud-images.ubuntu.com/$SERIES/current/${SERIES}-server-cloudimg-amd64.img qemu-img create -F qcow2 -b ${SERIES}-server-cloudimg-amd64.img -f qcow2 $SERIES.qcow2 8G
cloud-init image: ---
echo 'local-hostname: test' >meta-data
PUBKEY=$(cat $HOME/.ssh/id_rsa.pub)
cat <<EOF >user-data #cloud-config users: - name: ubuntu ssh-authorized-keys: ["$PUBKEY"] sudo: ['ALL=(ALL) NOPASSWD:ALL'] groups: sudo shell: /bin/bash runcmd: - echo 'AllowUsers ubuntu' >>/etc/ssh/sshd_config - restart ssh EOF
genisoimage -output test-cidata.iso -volid cidata -joliet -rock user-data meta-data
QEMU VM: --- - multipath disk on virtio-scsi: - host forwarding for ssh login:
truncate -s 1G multipath.img
qemu-system-x86_64 \ -machine q35 -accel kvm -cpu host -m 2048 \ -nodefaults -no-user-config \ -nographic -serial stdio \ \ -net nic -net user,hostfwd=::22222-:22 \ \ -drive file=$SERIES.qcow2 \ -drive file=test-cidata.iso,media=cdrom \ \ -device virtio-scsi,id=scsi1 \ \ -drive file=multipath.img,if=none,cache=none,format=file,locking=off,id=path0 \ -device scsi-hd,bus=scsi1.0,serial=helloworld,drive=path0 \ \ -drive file=multipath.img,if=none,cache=none,format=file,locking=off,id=path1 \ -device scsi-hd,bus=scsi1.0,serial=helloworld,drive=path1
$ ssh ubuntu@127.0.0.1 -p 22222
$ sudo multipath -l mpatha (0QEMU_QEMU_HARDDISK_helloworld) dm-0 QEMU,QEMU HARDDISK size=1.0G features='0' hwhandler='0' wp=rw |-+- policy='service-time 0' prio=0 status=active | `- 0:0:0:0 sda 8:0 active undef running `-+- policy='service-time 0' prio=0 status=enabled `- 0:0:1:0 sdb 8:16 active undef running
Latest version: ---
sudo add-apt-repository -y 'deb http://archive.ubuntu.com/ubuntu focal-proposed main' sudo apt install -y multipath-tools/focal-proposed
sudo systemctl restart multipathd.service
$ dpkg -s multipath-tools | grep Version: Version: 0.8.3-1ubuntu2.2
Debug symbols ---
V1=$(dpkg-query -Wf '${Version}' multipath-tools) V2=$(dpkg-query -Wf '${Version}' libdevmapper1.02.1 | cut -d: -f2-)
wget https://launchpad.net/ubuntu/+archive/primary/+files/multipath-tools-dbgsym_${V1}_amd64.ddeb wget https://launchpad.net/ubuntu/+archive/primary/+files/libdevmapper1.02.1-dbgsym_${V2}_amd64.ddeb
sudo dpkg -i ./multipath-tools-dbgsym_${V1}_amd64.ddeb ./libdevmapper1.02.1-dbgsym_${V2}_amd64.ddeb
Debugger ---
sudo apt install -y gdb
sudo gdb -p $(pidof multipathd)
(gdb) b dm_get_map Breakpoint 1 at ... (gdb) c Thread 5 "multipathd" hit Breakpoint 1, dm_get_map (name=0x558d02dc6880 "mpatha", ...
(gdb) b dm_get_next_target Breakpoint 2 at ... (gdb) c Thread 5 "multipathd" hit Breakpoint 2, dm_get_next_target (...
(gdb) finish Run till exit from #0 dm_get_next_target (... Value returned is $1 = (void *) 0x0
(gdb) p params $2 = 0x7fef90004db0 "0 0 2 1 service-time 0 1 2 8:0 1 1 service-time 0 1 2 8:16 1 1 " (gdb) set params = 0 (gdb) p params $3 = 0x0
Now, NOTE the value returned from both functions (ZERO, i.e., "OK", even though empty map parameters is invalid)
(gdb) finish Run till exit from #0 dm_get_map (name=0x563ca34f5f70 "mpatha", ... Value returned is $4 = 0
(gdb) p params $5 = "(null)", '\000' <repeats 4089 times>
(gdb) finish Run till exit from #0 0x00007f28d78b0911 in update_multipath_table (...
... Value returned is $6 = 0
Now, the multipath map state has NULL in all parameters:
(gdb) b disassemble_map (gdb) c
Thread 5 "multipathd" hit Breakpoint 3, disassemble_map (... (gdb) finish
(gdb) p mpp.features $8 = 0x7f2e80000e40 "(null)"
(gdb) p mpp.hwhandler $10 = 0x7f2e80000f40 "(null)"
(gdb) p mpp.pg $11 = (vector) 0x0
(gdb) p mpp.selector $12 = (vector) 0x0
(gdb) q
Compare with a normal run, where params is not NULL:
(gdb) p mpp.features $2 = 0x55f7670f3940 "0"
(gdb) p mpp.hwhandler $3 = 0x55f7670f8500 "0"
(gdb) p *mpp.pg $4 = {allocated = 2, slot = 0x55f7670f3990}
(gdb) p mpp.selector $5 = 0x55f7670f9e00 "service-time 0"
Synthetic reproducer on Focal
===
Ubuntu Server cloud image:
---
SERIES=focal
wget https:/ /cloud- images. ubuntu. com/$SERIES/ current/ ${SERIES} -server- cloudimg- amd64.img -server- cloudimg- amd64.img -f qcow2 $SERIES.qcow2 8G
qemu-img create -F qcow2 -b ${SERIES}
cloud-init image:
---
echo 'local-hostname: test' >meta-data
PUBKEY=$(cat $HOME/. ssh/id_ rsa.pub)
cat <<EOF >user-data authorized- keys: ["$PUBKEY"] ssh/sshd_ config
#cloud-config
users:
- name: ubuntu
ssh-
sudo: ['ALL=(ALL) NOPASSWD:ALL']
groups: sudo
shell: /bin/bash
runcmd:
- echo 'AllowUsers ubuntu' >>/etc/
- restart ssh
EOF
genisoimage -output test-cidata.iso -volid cidata -joliet -rock user-data meta-data
QEMU VM:
---
- multipath disk on virtio-scsi:
- host forwarding for ssh login:
truncate -s 1G multipath.img
qemu-system-x86_64 \ ::22222- :22 \ cidata. iso,media= cdrom \ scsi,id= scsi1 \ img,if= none,cache= none,format= file,locking= off,id= path0 \ bus=scsi1. 0,serial= helloworld, drive=path0 \ img,if= none,cache= none,format= file,locking= off,id= path1 \ bus=scsi1. 0,serial= helloworld, drive=path1
-machine q35 -accel kvm -cpu host -m 2048 \
-nodefaults -no-user-config \
-nographic -serial stdio \
\
-net nic -net user,hostfwd=
\
-drive file=$SERIES.qcow2 \
-drive file=test-
\
-device virtio-
\
-drive file=multipath.
-device scsi-hd,
\
-drive file=multipath.
-device scsi-hd,
$ ssh ubuntu@127.0.0.1 -p 22222
$ sudo multipath -l QEMU_HARDDISK_ helloworld) dm-0 QEMU,QEMU HARDDISK 'service- time 0' prio=0 status=active 'service- time 0' prio=0 status=enabled
mpatha (0QEMU_
size=1.0G features='0' hwhandler='0' wp=rw
|-+- policy=
| `- 0:0:0:0 sda 8:0 active undef running
`-+- policy=
`- 0:0:1:0 sdb 8:16 active undef running
Latest version:
---
sudo add-apt-repository -y 'deb http:// archive. ubuntu. com/ubuntu focal-proposed main' tools/focal- proposed
sudo apt install -y multipath-
sudo systemctl restart multipathd.service
$ dpkg -s multipath-tools | grep Version:
Version: 0.8.3-1ubuntu2.2
Debug symbols
---
V1=$(dpkg-query -Wf '${Version}' multipath-tools)
V2=$(dpkg-query -Wf '${Version}' libdevmapper1.02.1 | cut -d: -f2-)
wget https:/ /launchpad. net/ubuntu/ +archive/ primary/ +files/ multipath- tools-dbgsym_ ${V1}_amd64.ddeb /launchpad. net/ubuntu/ +archive/ primary/ +files/ libdevmapper1. 02.1-dbgsym_ ${V2}_amd64.ddeb
wget https:/
sudo dpkg -i ./multipath- tools-dbgsym_ ${V1}_amd64. ddeb ./libdevmapper1 .02.1-dbgsym_ ${V2}_amd64. ddeb
Debugger
---
sudo apt install -y gdb
sudo gdb -p $(pidof multipathd)
(gdb) b dm_get_map c6880 "mpatha", ...
Breakpoint 1 at ...
(gdb) c
Thread 5 "multipathd" hit Breakpoint 1, dm_get_map (name=0x558d02d
(gdb) b dm_get_next_target
Breakpoint 2 at ...
(gdb) c
Thread 5 "multipathd" hit Breakpoint 2, dm_get_next_target (...
(gdb) finish
Run till exit from #0 dm_get_next_target (...
Value returned is $1 = (void *) 0x0
(gdb) p params
$2 = 0x7fef90004db0 "0 0 2 1 service-time 0 1 2 8:0 1 1 service-time 0 1 2 8:16 1 1 "
(gdb) set params = 0
(gdb) p params
$3 = 0x0
Now, NOTE the value returned from both functions
(ZERO, i.e., "OK", even though empty map parameters is invalid)
(gdb) finish f5f70 "mpatha", ...
Run till exit from #0 dm_get_map (name=0x563ca34
Value returned is $4 = 0
(gdb) p params
$5 = "(null)", '\000' <repeats 4089 times>
(gdb) finish multipath_ table (...
Run till exit from #0 0x00007f28d78b0911 in update_
...
Value returned is $6 = 0
Now, the multipath map state has NULL in all parameters:
(gdb) b disassemble_map
(gdb) c
Thread 5 "multipathd" hit Breakpoint 3, disassemble_map (...
(gdb) finish
(gdb) p mpp.features
$8 = 0x7f2e80000e40 "(null)"
(gdb) p mpp.hwhandler
$10 = 0x7f2e80000f40 "(null)"
(gdb) p mpp.pg
$11 = (vector) 0x0
(gdb) p mpp.selector
$12 = (vector) 0x0
(gdb) q
Compare with a normal run, where params is not NULL:
(gdb) p mpp.features
$2 = 0x55f7670f3940 "0"
(gdb) p mpp.hwhandler
$3 = 0x55f7670f8500 "0"
(gdb) p *mpp.pg
$4 = {allocated = 2, slot = 0x55f7670f3990}
(gdb) p mpp.selector
$5 = 0x55f7670f9e00 "service-time 0"