Migration of a nested guest =========================== In this setup both L1 guest hypervisors (source & destination) and the L2 guest are started with QEMU '-cpu host' (or in libvirt parlance: 'host-passthrough'). The guests are launched via libvirt. Destination Compute QEMU command-line ----------------------------------------------------------------------- /usr/bin/qemu-system-x86_64 -machine accel=kvm -name guest=f26-devstack,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-16-f26-devstack/master-key.aes -machine pc-i440fx-2.9,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 8192 -realtime mlock=off -smp 1,sockets=1,cores=1,threads=1 -uuid c2c55fa7-e08a-4b03-bdc7-aba6dcb59a9e -display none -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-16-f26-devstack/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x3.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x3 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x3.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x3.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x4 -drive file=/home/kashyapc/vmimages/f26-devstack.qcow2,format=qcow2,if=none,id=drive-virtio-disk0 -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=29 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:77:41:44,bus=pci.0,addr=0x2 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-16-f26-devstack/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x6 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x7 -msg timestamp=on ----------------------------------------------------------------------- Source Compute QEMU command-line ----------------------------------------------------------------------- /usr/bin/qemu-system-x86_64 -machine accel=kvm -name guest=f26-devstack,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-16-f26-devstack/master-key.aes -machine pc-i440fx-2.9,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 8192 -realtime mlock=off -smp 1,sockets=1,cores=1,threads=1 -uuid c2c55fa7-e08a-4b03-bdc7-aba6dcb59a9e -display none -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-16-f26-devstack/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x3.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x3 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x3.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x3.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x4 -drive file=/home/kashyapc/vmimages/f26-devstack.qcow2,format=qcow2,if=none,id=drive-virtio-disk0 -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=29 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:77:41:44,bus=pci.0,addr=0x2 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-16-f26-devstack/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x6 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x7 -msg timestamp=on ----------------------------------------------------------------------- Nested guest (to be migrated) QEMU command-line; running on source Compute: ----------------------------------------------------------------------- /usr/bin/qemu-system-x86_64 -machine accel=kvm -name guest=cvm1,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-2-cvm1/master-key.aes -machine pc-i440fx-2.9,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 512 -realtime mlock=off -smp 1,sockets=1,cores=1,threads=1 -uuid b46f0d5f-388a-45f2-a1e8-2867be466391 -display none -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-2-cvm1/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x3.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x3 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x3.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x3.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x4 -drive file=/var/lib/libvirt/images/cirros.qcow2,format=qcow2,if=none,id=drive-virtio-disk0 -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=32,id=hostnet0,vhost=on,vhostfd=34 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:28:a4:e8,bus=pci.0,addr=0x2 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-2-cvm1/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x6 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x7 -msg timestamp=on ----------------------------------------------------------------------- == Do the migration == [Source hostname: f26-devstack; Destination hostname: f26-vm2] $ virsh migrate --verbose --copy-storage-all --live cvm1 qemu+tcp://root@f26-vm2/system Migration: [100 %] $ echo $? 0 * * * Config setup ------------ [For the above migration to work, you need the following config setup.] Assuming you're in a trusted network (but there's no such thing as "trusted network", though), have this config in destination's libvirtd.conf: $ cat /etc/libvirt/libvirtd.conf | grep -v ^$ | grep -v ^# listen_tls = 0 listen_tcp = 1 auth_tcp = "none" Start the libvirtd daemon on the destination with: $ cat /etc/sysconfig/libvirtd | grep -v ^$ | grep -v ^# LIBVIRTD_ARGS="--listen" Since I'm testing in a "trusted network", I also have SSH access (via public/private keys) to destination host without any password prompts. # Create the SSH keys with empty passphrase $ ssh-keygen -t rsa # Copy the key to the remote host $ ssh-copy-id root@f26-vm2 Versions -------- L0: $ uname -r; qemu-system-x86_64 --version; rpm -q libvirt-daemon-kvm 4.13.13-300.fc27.x86_64+debug QEMU emulator version 2.11.0(qemu-2.11.0-4.fc27) Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers libvirt-daemon-kvm-3.10.0-2.fc27.x86_64 On both source and destination L1s: $ uname -r; qemu-system-x86_64 --version; rpm -q libvirt-daemon-kvm 4.14.16-200.fc26.x86_64 QEMU emulator version 2.10.0(qemu-2.10.0-4.fc26) Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers libvirt-daemon-kvm-3.7.0-2.fc26.x86_64