From 78a15c67b0a9bc1131339eb2071cc0b8254c2b17 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:10 +0200 Subject: [PATCH 01/79] vmm: Revert "skip pending memory sends on worker failure" This reverts commit 4fb29bc98bf25c7a38a5ac99356f84245bf145f9. This change is independent of the migration worker, but its code sits inside the reverted region. It is re-applied unchanged later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/migration_transport.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vmm/src/migration_transport.rs b/vmm/src/migration_transport.rs index dbaf785c02..42c9017fff 100644 --- a/vmm/src/migration_transport.rs +++ b/vmm/src/migration_transport.rs @@ -727,9 +727,7 @@ impl SendAdditionalConnections { })?; match message { SendMemoryThreadMessage::Memory(table) => { - if external_cancel.load(Ordering::Acquire) - || worker_error.load(Ordering::Acquire) - { + if external_cancel.load(Ordering::Acquire) { continue; } From bc6d7e144ccc02b3fd8db24d5358f8d217a15a87 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:10 +0200 Subject: [PATCH 02/79] vmm: Revert "fix migration sender hang on worker failure" This reverts commit 705ed464860b7d23581a0655329411a9cb91d13b. This change is independent of the migration worker, but its code sits inside the reverted region. It is re-applied unchanged later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/migration_transport.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmm/src/migration_transport.rs b/vmm/src/migration_transport.rs index 42c9017fff..95ea03b9fa 100644 --- a/vmm/src/migration_transport.rs +++ b/vmm/src/migration_transport.rs @@ -885,7 +885,7 @@ impl SendAdditionalConnections { // All threads may have terminated, leading to a dropped receiver. Thus we ignore // errors here. self.message_tx - .send(SendMemoryThreadMessage::Disconnect) + .try_send(SendMemoryThreadMessage::Disconnect) .ok(); } From 61bb025a40933498bc3939e40e631a84fdd107e7 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:10 +0200 Subject: [PATCH 03/79] vmm: Revert "Reset throttle thread when do_memory_iteration fails" This reverts commit 636637c9edc7fdc2263beb83b23fed4cc92e46b9. This change is independent of the migration worker, but its code sits inside the reverted region. It is re-applied unchanged later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index ef7d16b690..22e7a07caf 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1819,16 +1819,12 @@ impl Vmm { mem_send, postponed_lifecycle_event, return_if_cancelled_cb, - ); - + )?; let downtime_begin = Instant::now(); // End throttle thread info!("stopping vcpu throttling"); vm.reset_vcpu_throttle_thread(); info!("stopped vcpu throttling"); - - let remaining = remaining?; - info!("pausing VM"); vm.pause()?; info!("paused VM"); From 8ce9c5bc4818c57106e215546e66fa05180c1f8f Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:10 +0200 Subject: [PATCH 04/79] vmm: Revert "Do not stop vCPU throttle thread after migration" This reverts commit e38e586c2dc3daff8a630a4a4801b3bba6396103. This change is independent of the migration worker, but its code sits inside the reverted region. It is re-applied unchanged later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 6 ++--- vmm/src/vcpu_throttling.rs | 50 ++++---------------------------------- vmm/src/vm.rs | 8 +++--- 3 files changed, 12 insertions(+), 52 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 22e7a07caf..c87c3cce0f 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1822,9 +1822,9 @@ impl Vmm { )?; let downtime_begin = Instant::now(); // End throttle thread - info!("stopping vcpu throttling"); - vm.reset_vcpu_throttle_thread(); - info!("stopped vcpu throttling"); + info!("stopping vcpu thread"); + vm.stop_vcpu_throttling(); + info!("stopped vcpu thread"); info!("pausing VM"); vm.pause()?; info!("paused VM"); diff --git a/vmm/src/vcpu_throttling.rs b/vmm/src/vcpu_throttling.rs index 840fa06e67..464728b9e6 100644 --- a/vmm/src/vcpu_throttling.rs +++ b/vmm/src/vcpu_throttling.rs @@ -30,13 +30,13 @@ use std::cell::Cell; use std::cmp::min; -use std::sync::mpsc::{RecvTimeoutError, SyncSender}; +use std::sync::mpsc::RecvTimeoutError; use std::sync::{Arc, Mutex, mpsc}; use std::thread; use std::thread::JoinHandle; use std::time::{Duration, Instant}; -use log::{debug, error, info, warn}; +use log::{debug, warn}; use vm_migration::Pausable; use crate::cpu::CpuManager; @@ -50,11 +50,6 @@ enum ThrottleCommand { Throttle(u8 /* `1..=99` */), /// Gracefully shutdown the vCPU throttling thread. Exit, - /// Exit the throttle loop then rendezvous with the receiver before proceeding to wait for the next command. - /// - /// In other words the `report` is used to synchronize the throttle thread reset event with the thread that - /// sent this command. - Reset { report: SyncSender<()> }, } /// Helper to adapt the throttling timeslice as we go, depending on the time it @@ -263,7 +258,6 @@ impl ThrottleWorker { None } Some(cmd @ (ThrottleCommand::Exit | ThrottleCommand::Wait)) => Some(cmd), - Some(ThrottleCommand::Reset { report }) => Some(ThrottleCommand::Reset { report }), } } @@ -351,31 +345,10 @@ impl ThrottleWorker { &callback_pause_vcpus, &callback_resume_vcpus, ); - match next_task { - ThrottleCommand::Exit => { - break 'control; - } - // else: thread needs to go into waiting state - ThrottleCommand::Reset { report } => { - // Inform sender that we are back in the waiting state: Since `report` has capacity 0 - // this call will block until the command sender has received our message. - if let Err(e) = report.send(()) { - error!( - "Unable to synchronize throttle thread reset event: error = {e:#?}" - ); - } - } - _ => { - continue 'control; - } - } - } - ThrottleCommand::Reset { report } => { - if let Err(e) = report.send(()) { - error!( - "Unable to synchronize throttle thread reset event: error = {e:#?}" - ); + if matches!(next_task, ThrottleCommand::Exit) { + break 'control; } + // else: thread is in Waiting state } } } @@ -554,19 +527,6 @@ impl ThrottleThreadHandle { ); } } - - /// Stops throttling and returns the throttle thread to the waiting state. - /// - /// This blocks until the throttling thread has exited the throttling loop. - pub fn reset(&self) { - let (report, recv) = mpsc::sync_channel(0); - self.state_sender - .send(ThrottleCommand::Reset { report }) - .expect("channel should not be closed"); - self.current_throttle.set(0); - info!("Waiting for throttle thread to acknowledge reset"); - recv.recv().expect("The throttle thread should acknowledge the reset event before dropping rendezvous channel"); - } } impl Drop for ThrottleThreadHandle { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index c7c17fb41d..fdc341a0aa 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -1495,11 +1495,11 @@ impl Vm { self.vcpu_throttler.throttle_percent() } - /// Sets the vCPU throttling thread back to its initial waiting state. + /// Stops and terminates the thread gracefully. /// - /// Blocks until the throttling thread acknowledges the reset event. - pub fn reset_vcpu_throttle_thread(&self) { - self.vcpu_throttler.reset(); + /// Waits for the thread to finish. + pub fn stop_vcpu_throttling(&mut self) { + self.vcpu_throttler.shutdown(); } pub fn set_post_migration_lifecycle_event( From ac70bd669f3ef9654b150be5c58d15a3e0a77791 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:10 +0200 Subject: [PATCH 05/79] vmm: Revert "clean up migration threads on failure" This reverts commit d00bdc095ee08e686ba4cf4c05818dbc20d8ea23. This change is independent of the migration worker, but its code sits inside the reverted region. It is re-applied unchanged later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/migration_transport.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/vmm/src/migration_transport.rs b/vmm/src/migration_transport.rs index 95ea03b9fa..e73b8bfcd5 100644 --- a/vmm/src/migration_transport.rs +++ b/vmm/src/migration_transport.rs @@ -543,9 +543,6 @@ impl ReceiveAdditionalConnections { impl Drop for ReceiveAdditionalConnections { fn drop(&mut self) { - if let Err(error) = self.terminate_fd.write(1) { - warn!("Failed to write to termination fd: {error}"); - } if self.accept_thread.is_some() { warn!( "ReceiveAdditionalConnections was not cleaned up! Either cleanup() was never called (programming error) or it failed before completing." From 968428970303e3326a765bcc6433c77fa454d0ca Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:37 +0200 Subject: [PATCH 06/79] vmm, vm-migration: Revert "streamline error chain message with upstream" This reverts commit e6cd3d883727450d9c30cf3648175f030616db3a. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/lib.rs | 18 ----------------- vm-migration/src/progress.rs | 4 +--- vmm/src/lib.rs | 38 +++++++++++++++++++++++++++++++----- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 1941e6f830..60b1a47496 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -3,9 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause // -use std::error::Error; -use std::iter; - use anyhow::anyhow; pub use context::{ CompletedMigrationContext, DowntimeContext, MemoryMigrationContext, MigrationContextError, @@ -23,21 +20,6 @@ pub mod progress; pub mod protocol; pub mod tls; -/// Mimics the error chain printing of CH for migration-related errors, where we -/// do not exit the VMM (which would print the error chain). -pub fn nested_error_to_flat_chain_as_string(top_error: &dyn Error) -> String { - iter::successors(Some(top_error), |sub_error| { - // Dereference necessary to mitigate rustc compiler bug. - // See - (*sub_error).source() - }) - // Important to use the plain Display impl to not interfere - // with anyhow's "smart" printing - .map(|e| format!("{e}")) - .collect::>() - .join(" => ") -} - #[derive(Error, Debug)] pub enum UffdError { #[error("Snapshot ranges are not page-aligned")] diff --git a/vm-migration/src/progress.rs b/vm-migration/src/progress.rs index dc2f358b82..c5babdad92 100644 --- a/vm-migration/src/progress.rs +++ b/vm-migration/src/progress.rs @@ -22,8 +22,6 @@ use std::fmt::Display; use std::num::NonZeroU32; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use crate::nested_error_to_flat_chain_as_string; - #[derive( Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] @@ -331,7 +329,7 @@ impl MigrationProgress { self.timestamp_snapshot_ms = current_unix_timestamp_ms(); self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; self.state = MigrationState::Failed { - error_msg: nested_error_to_flat_chain_as_string(error), + error_msg: format!("{error}",), error_msg_debug: format!("{error:?}",), }; } diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index c87c3cce0f..74fb8ffaad 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -56,7 +56,7 @@ use vm_migration::progress::{ use vm_migration::protocol::*; use vm_migration::{ MemoryMigrationContext, Migratable, MigratableError, OngoingMigrationContext, Pausable, - Snapshot, Snapshottable, Transportable, nested_error_to_flat_chain_as_string, + Snapshot, Snapshottable, Transportable, }; use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::unblock_signal; @@ -2213,6 +2213,37 @@ impl Vmm { self.vm.vm_mut().unwrap().restore() } + /// Prints the error chain to `error!()` level, akin to user-facing errors when Cloud Hypervisor + /// or ch-remote fail. + // TODO: For upstreaming, we should unify this with the code-paths used by ch-remote and + // Cloud Hypervisor on failure. + fn log_print_error_chain<'a>(top_error: &'a (dyn std::error::Error + 'static)) { + // Print chain of errors + if top_error.source().is_none() { + error!("Migration failed with the following error:"); + error!(" {top_error}"); + } else { + // In cli_print_error_chain(), we also print the + // ::fmt() as oneliner so that we can see all + // properties. As we use anyhow errors in the migration path, + // Debug::fmt() is not helpful for us as it doesn't print the + // underlying properties (like the default Debug::fmt() impl would + // do). Instead, it would print a trace itself, which is not what + // we want to do here. + + error!("Migration failed with the following chain of errors:"); + std::iter::successors(Some(top_error), |sub_error| { + // Dereference necessary to mitigate rustc compiler bug. + // See + (*sub_error).source() + }) + .enumerate() + .for_each(|(level, error)| { + error!(" {level}: {error}"); + }); + } + } + /// Checks the migration result. /// /// This should be called when the migration thread indicated a state @@ -2314,10 +2345,7 @@ impl Vmm { } } Err(e) => { - error!( - "Migration failed: {}", - nested_error_to_flat_chain_as_string(&e) - ); + Self::log_print_error_chain(&e); try_resume_vm(vm); // Update migration progress snapshot From 8a9e565014b047dfd7391d5ae0c4244e36b18fe7 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:37 +0200 Subject: [PATCH 07/79] vmm, vm-migration: Revert "add MigrationStateOngoingPhase::Started" This reverts commit 05827716664bcc83d578f051a683f06185e5551d. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/progress.rs | 7 +------ vmm/src/lib.rs | 10 ---------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/vm-migration/src/progress.rs b/vm-migration/src/progress.rs index c5babdad92..8a5083068d 100644 --- a/vm-migration/src/progress.rs +++ b/vm-migration/src/progress.rs @@ -76,12 +76,8 @@ pub struct MemoryTransmissionInfo { Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] pub enum MigrationStateOngoingPhase { - /// The migration process is initiated. No checks or connections are - /// established yet. + /// The migration starts. Handshake and transfer of VM config. Starting, - /// The initial connection is established and the migration protocol - /// handshake succeeded. - Started, /// Transfer of memory FDs. /// /// Only used for local migrations. @@ -100,7 +96,6 @@ impl Display for MigrationStateOngoingPhase { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Starting => write!(f, "starting"), - Self::Started => write!(f, "started"), Self::MemoryFds => write!(f, "memory FDs"), Self::MemoryPrecopy => write!(f, "memory (precopy)"), Self::Completing => write!(f, "completing"), diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 74fb8ffaad..8ca9bb21a9 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1897,16 +1897,6 @@ impl Vmm { MigratableError::MigrateSend(anyhow!("Error starting migration (got bad response)")), )?; - // Signal that the migration connection has been established. Management - // software can use this to distinguish short send/receive races from a - // real migration startup failure. - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .update(MigrationStateOngoingPhase::Started, None, None, None); - } - return_if_cancelled_cb(&mut socket)?; // Send config From 58e8d5b48a1c7ed2da54e6e412184667d913e3fe Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:37 +0200 Subject: [PATCH 08/79] vmm: Revert "Remove duplicated update_migration_progress" This reverts commit afa060f4dde5cfc71b2fc4a7b373da581cfcef96. The original subject was "vmm: Remove duplicated update_migration_progress in do_memory_iterations". This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 8ca9bb21a9..4db780992c 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1658,13 +1658,16 @@ impl Vmm { }; ctx.update_metrics_before_transfer(iteration_begin, &iteration_table); - // Update before we either exit the loop or transfer memory + // Update before we might exit the loop. update_migration_progress(ctx, vm); if is_converged(ctx)? { info!("Precopy converged: {ctx}"); break Ok(iteration_table); } + // Update with new metrics before transmission. + update_migration_progress(ctx, vm); + // Send the current dirty pages let transfer_begin = Instant::now(); mem_send.send_memory(iteration_table, socket, return_if_cancelled_cb)?; From aed01fb42654bcd702af512916e9d389c6edebe1 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:38 +0200 Subject: [PATCH 09/79] vmm: Revert "Fix memory_transmission_bps value in do_memory_iterations" This reverts commit 0b2ba9d5e96f1afd35c0e296c49b1738224f85c9. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 4db780992c..ca6c40775d 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1610,7 +1610,7 @@ impl Vmm { MigrationStateOngoingPhase::MemoryPrecopy, Some(MemoryTransmissionInfo { memory_iteration: s.iteration as u64, - memory_transmission_bps: s.bandwidth_bytes_per_second as u64, + memory_transmission_bps: s.current_iteration_total_bytes, memory_bytes_total: total_memory_size_bytes, memory_bytes_transmitted: s.total_sent_bytes, memory_pages_4k_transmitted: s.total_sent_bytes.div_ceil(PAGE_SIZE as u64), From ea46800dc79da7d605c283e4d6f9a43a8b517815 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:38 +0200 Subject: [PATCH 10/79] vmm: Revert "Fix memory_bytes_total value in do_memory_iterations" This reverts commit 6c38ad2ddfecb73914eb235e9f3f36ae935550aa. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index ca6c40775d..e842e4d4ef 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1596,12 +1596,6 @@ impl Vmm { postponed_lifecycle_event: &Mutex>, return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> result::Result<(), MigratableError>, ) -> result::Result { - let total_memory_size_bytes = vm - .memory_range_table()? - .ranges() - .iter() - .map(|range| range.length) - .sum::(); let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); lock.as_mut() @@ -1611,7 +1605,7 @@ impl Vmm { Some(MemoryTransmissionInfo { memory_iteration: s.iteration as u64, memory_transmission_bps: s.current_iteration_total_bytes, - memory_bytes_total: total_memory_size_bytes, + memory_bytes_total: s.bandwidth_bytes_per_second as u64, memory_bytes_transmitted: s.total_sent_bytes, memory_pages_4k_transmitted: s.total_sent_bytes.div_ceil(PAGE_SIZE as u64), memory_pages_4k_remaining_iteration: s From e31b846595ca92cef9091168a8603df44f3e6ac4 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:36:58 +0200 Subject: [PATCH 11/79] vmm: Revert "keep virtio activation alive in migration" This reverts commit 8c86589f9f53f9aee91c4772c0a3f55e60d25e27. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. This also drops the payload from the match arm in vm_post_migration_announce(), which was added later by 0f8e07333 and is therefore not part of the reverted diff. Without that the tree would not compile. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 130 ++++++++++++++++++------------------------------- vmm/src/vm.rs | 4 -- 2 files changed, 48 insertions(+), 86 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index e842e4d4ef..f348e1d02f 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -21,7 +21,7 @@ use std::panic::AssertUnwindSafe; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, RecvError, SendError, Sender}; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::Duration; #[cfg(not(target_arch = "riscv64"))] @@ -71,7 +71,6 @@ use crate::config::{MemoryRestoreMode, RestoreConfig, add_to_config}; use crate::coredump::GuestDebuggable; #[cfg(feature = "kvm")] use crate::cpu::IS_IN_SHUTDOWN; -use crate::device_manager::DeviceManager; use crate::landlock::Landlock; use crate::memory_manager::MemoryManager; use crate::migration::{get_vm_snapshot, recv_vm_config, recv_vm_state}; @@ -782,40 +781,13 @@ pub struct VmmThreadHandle { pub http_api_handle: Option, } -struct MigrationVmState { - // The migration worker owns the VM during migration, so this should stop - // working once that VM has been dropped. - device_manager: Weak>, -} - -impl MigrationVmState { - fn new(vm: &Vm) -> Self { - Self { - device_manager: Arc::downgrade(vm.device_manager()), - } - } - - fn activate_virtio_devices(&self) -> result::Result<(), VmError> { - self.device_manager - .upgrade() - .expect("device manager should remain alive during migration") - .lock() - .unwrap() - .activate_virtio_devices() - .map_err(VmError::ActivateVirtioDevices) - } -} - /// Describes the current ownership of a running VM. #[allow(clippy::large_enum_variant)] -enum MaybeVmOwnership { +pub enum MaybeVmOwnership { /// The VMM holds the ownership of the VM. Vmm(Vm), /// The VM is temporarily blocked by the current ongoing migration. - /// - /// We still keep the device manager reachable so the epoll thread can - /// drain pending virtio activations while the migration worker owns the VM. - Migration(MigrationVmState), + Migration, /// No VM is running. None, } @@ -826,12 +798,13 @@ impl MaybeVmOwnership { /// # Panics /// This method panics if `self` is not [`Self::Vmm`]. fn take_vm_for_migration(&mut self) -> Vm { - match mem::replace(self, Self::None) { - Self::Vmm(vm) => { - *self = Self::Migration(MigrationVmState::new(&vm)); - vm - } - _ => panic!("should only be called when a migration can start"), + if !matches!(self, Self::Vmm(_)) { + panic!("should only be called when a migration can start"); + } + + match mem::replace(self, Self::Migration) { + MaybeVmOwnership::Vmm(vm) => vm, + _ => unreachable!(), } } @@ -2129,7 +2102,7 @@ impl Vmm { prefault: bool, memory_restore_mode: MemoryRestoreMode, ) -> std::result::Result<(), VmError> { - if matches!(self.vm, MaybeVmOwnership::Migration(_)) { + if matches!(self.vm, MaybeVmOwnership::Migration) { return Err(VmError::VmMigrating); } @@ -2395,7 +2368,7 @@ impl Vmm { // Consume the event. self.reset_evt.read().map_err(Error::EventFdRead)?; // Workaround for guest-induced shutdown during a live-migration. - if matches!(self.vm, MaybeVmOwnership::Migration(_)) { + if matches!(self.vm, MaybeVmOwnership::Migration) { self.postpone_lifecycle_event_during_migration( PostMigrationLifecycleEvent::VmReboot, ); @@ -2407,7 +2380,7 @@ impl Vmm { info!("VM guest exit event"); self.guest_exit_evt.read().map_err(Error::EventFdRead)?; // Workaround for guest-induced shutdown during a live-migration. - if matches!(self.vm, MaybeVmOwnership::Migration(_)) { + if matches!(self.vm, MaybeVmOwnership::Migration) { self.postpone_lifecycle_event_during_migration( PostMigrationLifecycleEvent::VmShutdown, ); @@ -2421,18 +2394,11 @@ impl Vmm { } } EpollDispatch::ActivateVirtioDevices => { - let count = self.activate_evt.read().map_err(Error::EventFdRead)?; - info!("Trying to activate pending virtio devices: count = {count}"); - match &self.vm { - MaybeVmOwnership::Vmm(vm) => vm - .activate_virtio_devices() - .map_err(Error::ActivateVirtioDevices)?, - MaybeVmOwnership::Migration(state) => { - state - .activate_virtio_devices() - .map_err(Error::ActivateVirtioDevices)?; - } - MaybeVmOwnership::None => {} + if let MaybeVmOwnership::Vmm(ref vm) = self.vm { + let count = self.activate_evt.read().map_err(Error::EventFdRead)?; + info!("Trying to activate pending virtio devices: count = {count}"); + vm.activate_virtio_devices() + .map_err(Error::ActivateVirtioDevices)?; } } EpollDispatch::Api => { @@ -2527,7 +2493,7 @@ impl RequestHandler for Vmm { info!("Booting VM"); event!("vm", "booting"); - if matches!(self.vm, MaybeVmOwnership::Migration(_)) { + if matches!(self.vm, MaybeVmOwnership::Migration) { return Err(VmError::VmMigrating); } @@ -2592,7 +2558,7 @@ impl RequestHandler for Vmm { fn vm_pause(&mut self) -> result::Result<(), VmError> { match self.vm { MaybeVmOwnership::Vmm(ref mut vm) => vm.pause().map_err(VmError::Pause), - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?, + MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, } } @@ -2600,7 +2566,7 @@ impl RequestHandler for Vmm { fn vm_resume(&mut self) -> result::Result<(), VmError> { match self.vm { MaybeVmOwnership::Vmm(ref mut vm) => vm.resume().map_err(VmError::Resume), - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?, + MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, } } @@ -2615,7 +2581,7 @@ impl RequestHandler for Vmm { vm.post_migration_announce(); Ok(()) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?, + MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, } } @@ -2632,7 +2598,7 @@ impl RequestHandler for Vmm { .map_err(VmError::SnapshotSend) }) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?, + MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, } } @@ -2640,7 +2606,7 @@ impl RequestHandler for Vmm { fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> result::Result<(), VmError> { match &self.vm { MaybeVmOwnership::Vmm(_vm) => return Err(VmError::VmAlreadyCreated), - MaybeVmOwnership::Migration(_) => return Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => return Err(VmError::VmMigrating), MaybeVmOwnership::None => (), } @@ -2708,7 +2674,7 @@ impl RequestHandler for Vmm { MaybeVmOwnership::Vmm(ref mut vm) => { vm.coredump(destination_url).map_err(VmError::Coredump) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2716,7 +2682,7 @@ impl RequestHandler for Vmm { fn vm_shutdown(&mut self) -> result::Result<(), VmError> { let vm = match self.vm { MaybeVmOwnership::Vmm(ref mut vm) => vm, - MaybeVmOwnership::Migration(_) => return Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => return Err(VmError::VmMigrating), MaybeVmOwnership::None => return Err(VmError::VmNotRunning), }; // Drain console_info so that the FDs are not reused @@ -2737,7 +2703,7 @@ impl RequestHandler for Vmm { // First we stop the current VM let vm = match self.vm { MaybeVmOwnership::Vmm(ref mut vm) => vm, - MaybeVmOwnership::Migration(_) => return Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => return Err(VmError::VmMigrating), MaybeVmOwnership::None => return Err(VmError::VmNotRunning), }; let config = vm.get_config(); @@ -2811,7 +2777,7 @@ impl RequestHandler for Vmm { let state = match &self.vm { MaybeVmOwnership::Vmm(vm) => vm.get_state(), // TODO in theory one could live-migrate a non-running VM .. - MaybeVmOwnership::Migration(_) => VmState::Running, + MaybeVmOwnership::Migration => VmState::Running, MaybeVmOwnership::None => VmState::Created, }; @@ -2822,14 +2788,14 @@ impl RequestHandler for Vmm { memory_actual_size = memory_actual_size.saturating_sub(vm.balloon_size()); memory_actual_size += vm.virtio_mem_plugged_size(); } - MaybeVmOwnership::Migration(_) => {} + MaybeVmOwnership::Migration => {} MaybeVmOwnership::None => {} } let device_tree = match &self.vm { MaybeVmOwnership::Vmm(vm) => Some(vm.device_tree().lock().unwrap().clone()), // TODO we need to fix this - MaybeVmOwnership::Migration(_) => None, + MaybeVmOwnership::Migration => None, MaybeVmOwnership::None => None, }; @@ -2871,7 +2837,7 @@ impl RequestHandler for Vmm { MaybeVmOwnership::None => { self.vm_config = None; } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?, + MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, } Ok(()) @@ -2901,7 +2867,7 @@ impl RequestHandler for Vmm { .inspect_err(|e| error!("Error when resizing VM: {e:?}"))?; Ok(()) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); if let Some(desired_vcpus) = desired_vcpus { @@ -2934,7 +2900,7 @@ impl RequestHandler for Vmm { Ok(()) } } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => Err(VmError::ResizeDisk), } } @@ -2948,7 +2914,7 @@ impl RequestHandler for Vmm { .inspect_err(|e| error!("Error when resizing zone: {e:?}"))?; Ok(()) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by setting the new desired ram. let memory_config = &mut self.vm_config.as_ref().unwrap().lock().unwrap().memory; @@ -2990,7 +2956,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3022,7 +2988,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3039,7 +3005,7 @@ impl RequestHandler for Vmm { .inspect_err(|e| error!("Error when removing device from the VM: {e:?}"))?; Ok(()) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { if let Some(ref config) = self.vm_config { let mut config = config.lock().unwrap(); @@ -3074,7 +3040,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3103,7 +3069,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3141,7 +3107,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3170,7 +3136,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3199,7 +3165,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3233,7 +3199,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -3253,7 +3219,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => Err(VmError::VmNotRunning), } } @@ -3261,7 +3227,7 @@ impl RequestHandler for Vmm { fn vm_power_button(&mut self) -> result::Result<(), VmError> { match self.vm { MaybeVmOwnership::Vmm(ref mut vm) => vm.power_button(), - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => Err(VmError::VmNotRunning), } } @@ -3269,7 +3235,7 @@ impl RequestHandler for Vmm { fn vm_nmi(&mut self) -> result::Result<(), VmError> { match self.vm { MaybeVmOwnership::Vmm(ref mut vm) => vm.nmi(), - MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating), + MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => Err(VmError::VmNotRunning), } } @@ -3369,7 +3335,7 @@ impl RequestHandler for Vmm { match self.vm { MaybeVmOwnership::Vmm(_) => (), - MaybeVmOwnership::Migration(_) => { + MaybeVmOwnership::Migration => { return Err(MigratableError::MigrateSend(anyhow!( "There is already an ongoing migration" ))); @@ -3481,7 +3447,7 @@ impl RequestHandler for Vmm { fn vm_cancel_migration(&mut self) -> result::Result<(), MigratableError> { match self.vm { - MaybeVmOwnership::Migration(_) => (), + MaybeVmOwnership::Migration => (), _ => { return Err(MigratableError::CancelMigration(anyhow!( "There is no ongoing migration" diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index fdc341a0aa..513cf6bb1e 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -3263,10 +3263,6 @@ impl Vm { Ok(()) } - pub fn device_manager(&self) -> &Arc> { - &self.device_manager - } - pub fn activate_virtio_devices(&self) -> Result<()> { self.device_manager .lock() From 89db6ab82312c2dca781b1e9761a69f03c2b2602 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:45 +0200 Subject: [PATCH 12/79] vmm: Revert "reduce API event verbosity" This reverts commit 09c63995470dbce52335b0ee80ceead588bba38c. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 6d1f7c6d7e..280196157b 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -40,7 +40,7 @@ use std::str::FromStr; use std::sync::mpsc::{RecvError, SendError, Sender, channel}; use std::time::Duration; -use log::{info, trace}; +use log::{debug, info}; use micro_http::Body; use option_parser::{OptionParser, OptionParserError, Toggle}; use serde::{Deserialize, Serialize}; @@ -1283,7 +1283,7 @@ impl ApiAction for VmCounters { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - trace!("API request event: VmCounters"); + info!("API request event: VmCounters"); let response = vmm .vm_counters() @@ -1388,7 +1388,7 @@ impl ApiAction for VmInfo { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - trace!("API request event: VmInfo"); + debug!("API request event: VmInfo"); let response = vmm .vm_info() @@ -2049,7 +2049,7 @@ impl ApiAction for VmMigrationProgress { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - trace!("API request event: VmMigrationProgress"); + debug!("API request event: VmMigrationProgress"); let snapshot = Ok(vmm.vm_migration_progress()); let response = snapshot From 3e5b8bb7d26e350f11e3cab8b4b439aef07e7173 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:45 +0200 Subject: [PATCH 13/79] vmm: Revert "defer guest exit during migration" This reverts commit 0795c73d024b4078fd826b4697f79408c9e815a7. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 26 +++++++++++++------------- vmm/src/vm.rs | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index f348e1d02f..47fdb3089e 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1286,10 +1286,10 @@ impl Vmm { .context("Failed writing reset eventfd after migration") .map_err(MigratableError::MigrateReceive)?; } - Some(PostMigrationLifecycleEvent::VmShutdown) => { - self.guest_exit_evt + Some(PostMigrationLifecycleEvent::VmmShutdown) => { + self.exit_evt .write(1) - .context("Failed writing guest exit eventfd after migration") + .context("Failed writing exit eventfd after migration") .map_err(MigratableError::MigrateReceive)?; } } @@ -2258,10 +2258,10 @@ impl Vmm { .inspect_err(|write_err| error!("{write_err}")) .ok(); } - PostMigrationLifecycleEvent::VmShutdown => { - self.guest_exit_evt + PostMigrationLifecycleEvent::VmmShutdown => { + self.exit_evt .write(1) - .context("Failed replaying guest exit event after failed migration") + .context("Failed replaying shutdown event after failed migration") .inspect_err(|write_err| error!("{write_err}")) .ok(); } @@ -2359,6 +2359,13 @@ impl Vmm { info!("VM exit event"); // Consume the event. self.exit_evt.read().map_err(Error::EventFdRead)?; + // Workaround for guest-induced shutdown during a live-migration. + if matches!(self.vm, MaybeVmOwnership::Migration) { + self.postpone_lifecycle_event_during_migration( + PostMigrationLifecycleEvent::VmmShutdown, + ); + continue; + } self.vmm_shutdown().map_err(Error::VmmShutdown)?; break 'outer; @@ -2379,13 +2386,6 @@ impl Vmm { EpollDispatch::GuestExit => { info!("VM guest exit event"); self.guest_exit_evt.read().map_err(Error::EventFdRead)?; - // Workaround for guest-induced shutdown during a live-migration. - if matches!(self.vm, MaybeVmOwnership::Migration) { - self.postpone_lifecycle_event_during_migration( - PostMigrationLifecycleEvent::VmShutdown, - ); - continue; - } if self.no_shutdown { self.vm_shutdown().map_err(Error::VmShutdown)?; } else { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 513cf6bb1e..18f9200755 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -587,7 +587,7 @@ pub struct Vm { #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum PostMigrationLifecycleEvent { VmReboot, - VmShutdown, + VmmShutdown, } impl Vm { From a2326fb60eb7d66584f83af367a6b413c7cbf5a8 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:46 +0200 Subject: [PATCH 14/79] vmm: Revert "migration: properly print error chain on failure" This reverts commit 0aebb7b3d70f84327d070283ce6d19446b8642e1. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 47fdb3089e..faf1b88704 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -2173,37 +2173,6 @@ impl Vmm { self.vm.vm_mut().unwrap().restore() } - /// Prints the error chain to `error!()` level, akin to user-facing errors when Cloud Hypervisor - /// or ch-remote fail. - // TODO: For upstreaming, we should unify this with the code-paths used by ch-remote and - // Cloud Hypervisor on failure. - fn log_print_error_chain<'a>(top_error: &'a (dyn std::error::Error + 'static)) { - // Print chain of errors - if top_error.source().is_none() { - error!("Migration failed with the following error:"); - error!(" {top_error}"); - } else { - // In cli_print_error_chain(), we also print the - // ::fmt() as oneliner so that we can see all - // properties. As we use anyhow errors in the migration path, - // Debug::fmt() is not helpful for us as it doesn't print the - // underlying properties (like the default Debug::fmt() impl would - // do). Instead, it would print a trace itself, which is not what - // we want to do here. - - error!("Migration failed with the following chain of errors:"); - std::iter::successors(Some(top_error), |sub_error| { - // Dereference necessary to mitigate rustc compiler bug. - // See - (*sub_error).source() - }) - .enumerate() - .for_each(|(level, error)| { - error!(" {level}: {error}"); - }); - } - } - /// Checks the migration result. /// /// This should be called when the migration thread indicated a state @@ -2305,7 +2274,7 @@ impl Vmm { } } Err(e) => { - Self::log_print_error_chain(&e); + error!("Migration failed: {e}"); try_resume_vm(vm); // Update migration progress snapshot From afb625afeacc06e4861bbc8a648d6f82962815d7 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:12:46 +0200 Subject: [PATCH 15/79] ch-remote: Revert "add cancel-migration" This reverts commit 5283a0c9013044bbf45eb90efc15de2ab35510de. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/src/bin/ch-remote.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index dd2eefb79e..f8075de113 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -636,8 +636,6 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu )?; simple_api_command(socket, "PUT", "create", Some(&data)).map_err(Error::HttpApiClient) } - Some("cancel-migration") => simple_api_command(socket, "PUT", "cancel-migration", None) - .map_err(Error::HttpApiClient), _ => unreachable!(), } } @@ -1134,7 +1132,6 @@ fn get_cli_commands_sorted() -> Box<[Command]> { .about("Add vsock device") .arg(Arg::new("vsock_config").index(1).help(VsockConfig::SYNTAX)), Command::new("boot").about("Boot a created VM"), - Command::new("cancel-migration").about("Cancel any ongoing migration"), Command::new("coredump") .about("Create a coredump from VM") .arg(Arg::new("coredump_config").index(1).help("")), From 8af0f21f30f76789d310788b92b0df12545d061b Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:13:06 +0200 Subject: [PATCH 16/79] vmm: Revert "migration cancellation: integrate into TCP threads" This reverts commit a2abcf55f4b05242311c9b1bb01cd49283ef1b39. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/migration_transport.rs | 55 ++++++++-------------------------- 1 file changed, 13 insertions(+), 42 deletions(-) diff --git a/vmm/src/migration_transport.rs b/vmm/src/migration_transport.rs index e73b8bfcd5..437ae4b8ab 100644 --- a/vmm/src/migration_transport.rs +++ b/vmm/src/migration_transport.rs @@ -12,9 +12,7 @@ use std::os::unix::net::{UnixListener, UnixStream}; use std::path::{Path, PathBuf}; use std::result::Result; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{ - Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, -}; +use std::sync::mpsc::{Receiver, Sender, SyncSender, TrySendError, channel, sync_channel}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; @@ -585,9 +583,6 @@ pub(crate) struct SendAdditionalConnections { /// this using this flag. Only the main thread checks this variable, the worker /// threads will be stopped during cleanup. worker_error: Arc, - /// Externally triggered cancellation. Workers drain queued memory messages - /// after this is set and wait for the disconnect message. - external_cancel: Arc, /// After the main thread sent all memory chunks to the sender threads, it waits /// until one of the workers notifies it. Either because an error occurred, or /// because they arrived at the gate. @@ -630,7 +625,6 @@ impl SendAdditionalConnections { let buffer_size = Self::BUFFERED_REQUESTS_PER_THREAD * configured_connections as usize; let (message_tx, message_rx) = sync_channel::(buffer_size); let worker_error = Arc::new(AtomicBool::new(false)); - let external_cancel = Arc::new(AtomicBool::new(false)); let (notify_tx, notify_rx) = channel::(); // If one connection is configured, we don't have to create any additional threads. @@ -641,7 +635,6 @@ impl SendAdditionalConnections { threads, message_tx, worker_error, - external_cancel, notify_rx, }); } @@ -655,7 +648,6 @@ impl SendAdditionalConnections { let guest_memory = guest_memory.clone(); let message_rx = message_rx.clone(); let worker_error = worker_error.clone(); - let external_cancel = external_cancel.clone(); let notify_tx = notify_tx.clone(); let thread = thread::Builder::new() @@ -666,7 +658,6 @@ impl SendAdditionalConnections { &guest_memory, &message_rx, &worker_error, - &external_cancel, ¬ify_tx, ) }) @@ -689,7 +680,6 @@ impl SendAdditionalConnections { threads, message_tx, worker_error, - external_cancel, notify_rx, }) } @@ -699,7 +689,6 @@ impl SendAdditionalConnections { guest_memory: &GuestMemoryAtomic, message_rx: &Mutex>, worker_error: &AtomicBool, - external_cancel: &AtomicBool, notify_tx: &Sender, ) -> Result<(), MigratableError> { info!("Spawned thread to send VM memory."); @@ -724,10 +713,6 @@ impl SendAdditionalConnections { })?; match message { SendMemoryThreadMessage::Memory(table) => { - if external_cancel.load(Ordering::Acquire) { - continue; - } - send_memory_ranges(guest_memory, &table, socket) .inspect_err(|_| { worker_error.store(true, Ordering::Relaxed); @@ -783,14 +768,12 @@ impl SendAdditionalConnections { // The chunk size is chosen to be big enough so that even very fast links need some // milliseconds to send it. for chunk in table.partition(Self::CHUNK_SIZE) { - return_if_cancelled_cb(socket).inspect_err(|_| { - info!("cancelling migration during memory iteration"); - self.external_cancel.store(true, Ordering::Release); - })?; + return_if_cancelled_cb(socket) + .inspect_err(|_| info!("cancelling migration during memory iteration"))?; self.send_chunk(chunk)?; } - self.wait_for_pending_data(socket, return_if_cancelled_cb)?; + self.wait_for_pending_data()?; Ok(true) } @@ -825,11 +808,7 @@ impl SendAdditionalConnections { } /// Wait until all data that is in-flight has actually been sent and acknowledged. - fn wait_for_pending_data( - &mut self, - socket: &mut SocketStream, - return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> Result<(), MigratableError>, - ) -> Result<(), MigratableError> { + fn wait_for_pending_data(&mut self) -> Result<(), MigratableError> { let gate = Arc::new(Gate::new()); for _ in 0..self.threads.len() { self.message_tx @@ -843,34 +822,26 @@ impl SendAdditionalConnections { // they arrived at the gate. let mut seen_threads = 0; loop { - return_if_cancelled_cb(socket).inspect_err(|_| { - gate.open(); - self.external_cancel.store(true, Ordering::Release); - })?; - - thread::sleep(Duration::from_millis(2)); - - match self.notify_rx.try_recv() { - Ok(SendMemoryThreadNotify::Gate) => { + match self + .notify_rx + .recv() + .context("Error receiving message from workers") + .map_err(MigratableError::MigrateSend)? + { + SendMemoryThreadNotify::Gate => { seen_threads += 1; if seen_threads == self.threads.len() { gate.open(); return Ok(()); } } - Ok(SendMemoryThreadNotify::Error) => { + SendMemoryThreadNotify::Error => { // If an error occurred in one of the worker threads, we open // the gate to make sure that no thread hangs. After that, we // receive the error from Self::cleanup() and return it. gate.open(); return self.cleanup(); } - Err(TryRecvError::Empty) => {} - Err(TryRecvError::Disconnected) => { - return Err(MigratableError::MigrateSend(anyhow!( - "All senders died unexpectedly." - ))); - } } } } From c1c4ccbdd4a16b021c2da8e562a5c70af330bdef Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:13:06 +0200 Subject: [PATCH 17/79] vmm: Revert "migration: early cancellation (add more checks)" This reverts commit a9253633d47bf2b03f7f4917a4a1de421073aa95. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 18 ++---------------- vmm/src/migration_transport.rs | 9 +-------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index faf1b88704..3710c36442 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1567,7 +1567,6 @@ impl Vmm { is_converged: impl Fn(&MemoryMigrationContext) -> result::Result, mem_send: &mut SendAdditionalConnections, postponed_lifecycle_event: &Mutex>, - return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> result::Result<(), MigratableError>, ) -> result::Result { let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -1600,8 +1599,6 @@ impl Vmm { }; loop { - return_if_cancelled_cb(socket)?; - // todo: check if auto-converge is enabled at all? if Self::can_increase_autoconverge_step(ctx) && vm.throttle_percent() < AUTO_CONVERGE_MAX @@ -1637,7 +1634,7 @@ impl Vmm { // Send the current dirty pages let transfer_begin = Instant::now(); - mem_send.send_memory(iteration_table, socket, return_if_cancelled_cb)?; + mem_send.send_memory(iteration_table, socket)?; let transfer_duration = transfer_begin.elapsed(); ctx.update_metrics_after_transfer(transfer_begin, transfer_duration); @@ -1775,7 +1772,6 @@ impl Vmm { mem_send: &mut SendAdditionalConnections, ctx: &mut OngoingMigrationContext, postponed_lifecycle_event: &Mutex>, - return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> result::Result<(), MigratableError>, ) -> result::Result<(), MigratableError> { let mut mem_ctx = MemoryMigrationContext::new(); @@ -1788,7 +1784,6 @@ impl Vmm { |ctx| Self::is_precopy_converged(ctx, send_data_migration), mem_send, postponed_lifecycle_event, - return_if_cancelled_cb, )?; let downtime_begin = Instant::now(); // End throttle thread @@ -1808,7 +1803,7 @@ impl Vmm { mem_ctx.update_metrics_before_transfer(iteration_begin, &final_table); let transfer_begin = Instant::now(); - mem_send.send_memory(final_table, socket, return_if_cancelled_cb)?; + mem_send.send_memory(final_table, socket)?; let transfer_duration = transfer_begin.elapsed(); mem_ctx.update_metrics_after_transfer(transfer_begin, transfer_duration); mem_ctx.iteration += 1; @@ -1867,8 +1862,6 @@ impl Vmm { MigratableError::MigrateSend(anyhow!("Error starting migration (got bad response)")), )?; - return_if_cancelled_cb(&mut socket)?; - // Send config let vm_config = vm.get_config(); #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -1907,8 +1900,6 @@ impl Vmm { .map_err(MigratableError::MigrateSend)? }; - return_if_cancelled_cb(&mut socket)?; - if send_data_migration.local { match &mut socket { SocketStream::Unix(unix_socket) => { @@ -1928,8 +1919,6 @@ impl Vmm { } } - return_if_cancelled_cb(&mut socket)?; - let vm_migration_config = VmMigrationConfig { vm_config, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -1938,8 +1927,6 @@ impl Vmm { }; migration_transport::send_config(&mut socket, &vm_migration_config)?; - return_if_cancelled_cb(&mut socket)?; - // Let every Migratable object know about the migration being started. vm.start_migration()?; @@ -1970,7 +1957,6 @@ impl Vmm { &mut mem_send, &mut ctx, postponed_lifecycle_event, - &return_if_cancelled_cb, ) .inspect_err(|_| { // Calling cleanup multiple times is fine, thus here we just make sure diff --git a/vmm/src/migration_transport.rs b/vmm/src/migration_transport.rs index 437ae4b8ab..e001e328ed 100644 --- a/vmm/src/migration_transport.rs +++ b/vmm/src/migration_transport.rs @@ -749,7 +749,6 @@ impl SendAdditionalConnections { &mut self, table: MemoryRangeTable, socket: &mut SocketStream, - return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> Result<(), MigratableError>, ) -> Result { if table.regions().is_empty() { return Ok(false); @@ -757,19 +756,13 @@ impl SendAdditionalConnections { // If we use only one connection, we send the memory directly. if self.threads.is_empty() { - for chunk in table.partition(Self::CHUNK_SIZE) { - return_if_cancelled_cb(socket) - .inspect_err(|_| info!("cancelling migration during memory iteration"))?; - send_memory_ranges(&self.guest_memory, &chunk, socket)?; - } + send_memory_ranges(&self.guest_memory, &table, socket)?; return Ok(true); } // The chunk size is chosen to be big enough so that even very fast links need some // milliseconds to send it. for chunk in table.partition(Self::CHUNK_SIZE) { - return_if_cancelled_cb(socket) - .inspect_err(|_| info!("cancelling migration during memory iteration"))?; self.send_chunk(chunk)?; } From 4fef0d3f6eb1abad1812bba32c3ba3950a5173f3 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:13:06 +0200 Subject: [PATCH 18/79] vmm: Revert "migration: actually support cancellation" This reverts commit 806b51b96a0121136605a1edf13efc857f309cb0. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/lib.rs | 3 -- vmm/src/lib.rs | 61 +---------------------------------------- 2 files changed, 1 insertion(+), 63 deletions(-) diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 60b1a47496..89cbcecf87 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -95,9 +95,6 @@ pub enum MigratableError { #[error("Failed to complete migration for migratable component")] CompleteMigration(#[source] anyhow::Error), - #[error("Failed to continue the migration as it was cancelled")] - Cancelled, - #[error("Failed to release a disk lock")] UnlockError(#[source] anyhow::Error), diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 3710c36442..7801139f83 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -19,7 +19,6 @@ use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::panic::AssertUnwindSafe; #[cfg(feature = "guest_debug")] use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, RecvError, SendError, Sender}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -639,20 +638,9 @@ impl VmmVersionInfo { struct MigrationWorkerHandle { // Option to take the inner handle handle: Option>, - cancel: Arc, } impl MigrationWorkerHandle { - /// Cancels the migration. - /// - /// Note that timing issues in the very last phase of the migration allow a - /// tiny window in that migration succeeds before they could be canceled. - fn trigger_cancellation(&self) { - info!("Will cancel ongoing live-migration"); - self.cancel.store(true, Ordering::Release); - // we just dispatch here and do not block for the migration thread - } - /// Joins the thread and returns the result. fn join(mut self) -> MigrationThreadOut { self.handle @@ -685,7 +673,6 @@ struct MigrationWorker { postponed_lifecycle_event: Arc>>, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: Arc, - cancel: Arc, } impl MigrationWorker { @@ -700,7 +687,6 @@ impl MigrationWorker { self.hypervisor.as_ref(), &self.config, self.postponed_lifecycle_event.as_ref(), - self.cancel.clone(), ) .inspect(|_| event!("vm", "migration-finished")) .inspect_err(|e| { @@ -729,7 +715,6 @@ impl MigrationWorker { dyn hypervisor::Hypervisor, >, ) -> result::Result { - let cancel = Arc::new(AtomicBool::new(false)); let worker = MigrationWorker { vm, check_migration_evt, @@ -737,7 +722,6 @@ impl MigrationWorker { postponed_lifecycle_event, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor, - cancel: cancel.clone(), }; // Cumbersome but we need this to take a value from the worker when @@ -769,7 +753,6 @@ impl MigrationWorker { Ok(MigrationWorkerHandle { handle: Some(inner_handle), - cancel, }) } } @@ -1828,19 +1811,9 @@ impl Vmm { hypervisor: &dyn hypervisor::Hypervisor, send_data_migration: &VmSendMigrationData, postponed_lifecycle_event: &Mutex>, - cancel: Arc, ) -> result::Result<(), MigratableError> { // State machine that is updated with more context as we progress. let mut ctx = OngoingMigrationContext::new(); - let return_if_cancelled_cb = move |socket: &mut SocketStream| { - if cancel.load(Ordering::Acquire) { - info!("Cancelling migration now"); - Request::abandon().write_to(socket)?; - Err(MigratableError::Cancelled) - } else { - Ok(()) - } - }; // Set up the socket connection let mut socket = if send_data_migration.local { @@ -1969,10 +1942,6 @@ impl Vmm { mem_send.cleanup()?; } - // Very last cancellation check. After this, we release the disk locks and we can't cancel - // anymore. - return_if_cancelled_cb(&mut socket)?; - // Update migration progress snapshot { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -2246,30 +2215,9 @@ impl Vmm { } } } - Err(MigratableError::Cancelled) => { - error!("Migration cancelled"); - event!("vm", "migration-cancelled"); - try_resume_vm(vm); - - // Update migration progress snapshot - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_cancelled(); - } - } Err(e) => { error!("Migration failed: {e}"); try_resume_vm(vm); - - // Update migration progress snapshot - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_failed(&e); - } } } self.clear_postponed_lifecycle_event(); @@ -3410,14 +3358,7 @@ impl RequestHandler for Vmm { } } - let handle = self - .migration_thread_handle - .as_ref() - .expect("should have handle"); - // We just dispatch the cancellation. - handle.trigger_cancellation(); - - Ok(()) + todo!() } fn vm_migration_progress(&mut self) -> Option { From 749bed341bcb6970eaf7e7f4b05f1a3d08e5d993 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:13:06 +0200 Subject: [PATCH 19/79] vmm: Revert "add try_resume_vm() helper" This reverts commit c70dd779055e04be9b8fc9999abf49faf97adb48. The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 99 ++++++++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 7801139f83..c60ed8445d 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -2138,7 +2138,7 @@ impl Vmm { // At this point, the thread must be finished. // If we fail here, we have lost anyway. Just panic. let MigrationThreadOut { - vm, + mut vm, migration_res, migration_cfg, } = self @@ -2147,52 +2147,6 @@ impl Vmm { .expect("should have thread") .join(); - let mut try_resume_vm = |mut vm: Vm| { - // If the failure happened very late in the migration path, the VM might already be - // stopped. We resume it to ensure proper operation. - // - // Cloud Hypervisor only supports migration of running VMs, therefore it cannot - // happen that we resume a previously paused VM. - if vm.get_state() == VmState::Paused { - match vm.resume() { - Ok(_) => { - info!("Resumed VM successfully after failed migration"); - } - Err(e) => { - error!("Failed resuming VM after failed migration: {e}"); - self.exit_evt.write(1).unwrap(); - } - } - } - - // Ensure full VM performance. The operation is idempotent. - let _ = vm.stop_dirty_log().inspect_err(|e| { - warn!("Failed stopping dirty log after resuming VM: {e} - VM performance might be slower than usual"); - }); - - // Give VMM back control. - self.vm = MaybeVmOwnership::Vmm(vm); - - if let Some(event) = self.current_postponed_lifecycle_event() { - match event { - PostMigrationLifecycleEvent::VmReboot => { - self.reset_evt - .write(1) - .context("Failed replaying reset event after failed migration") - .inspect_err(|write_err| error!("{write_err}")) - .ok(); - } - PostMigrationLifecycleEvent::VmmShutdown => { - self.exit_evt - .write(1) - .context("Failed replaying shutdown event after failed migration") - .inspect_err(|write_err| error!("{write_err}")) - .ok(); - } - } - } - }; - match migration_res { Ok(()) => { self.vm = MaybeVmOwnership::None; @@ -2217,7 +2171,56 @@ impl Vmm { } Err(e) => { error!("Migration failed: {e}"); - try_resume_vm(vm); + // We don't fail the VMM here, it just continues running its VM. + // If the failure happened very late in the migration path, the VM might already be + // stopped. We resume it to ensure proper operation. + // + // Cloud Hypervisor only supports migration of running VMs, therefore it cannot + // happen that we resume a previously paused VM. + if vm.get_state() == VmState::Paused { + match vm.resume() { + Ok(_) => { + info!("Resumed VM successfully after failed migration"); + } + Err(e) => { + error!("Failed resuming VM after failed migration: {e}"); + self.exit_evt.write(1).unwrap(); + } + } + } + + // Ensure full VM performance. The operation is idempotent. + let _ = vm.stop_dirty_log().inspect_err(|e| { + warn!("Failed stopping dirty log after resuming VM: {e} - VM performance might be slower than usual"); + }); + + // Give VMM back control. + self.vm = MaybeVmOwnership::Vmm(vm); + if let Some(event) = self.current_postponed_lifecycle_event() { + match event { + PostMigrationLifecycleEvent::VmReboot => { + self.reset_evt + .write(1) + .context("Failed replaying reset event after failed migration") + .inspect_err(|write_err| error!("{write_err}")) + .ok(); + } + PostMigrationLifecycleEvent::VmmShutdown => { + self.exit_evt + .write(1) + .context("Failed replaying shutdown event after failed migration") + .inspect_err(|write_err| error!("{write_err}")) + .ok(); + } + } + } + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_failed(&e); + } } } self.clear_postponed_lifecycle_event(); From 24628b3303f03b767b1f534eee90e9e853995356 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:13:06 +0200 Subject: [PATCH 20/79] vmm: Revert "migration: add handle wrapper for MigrationWorker" This reverts commit 2a2f4e618d8ea1c3f13c62ff6f10b26b7e9aad0b. The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 127 ++++++++++--------------------------------------- 1 file changed, 24 insertions(+), 103 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index c60ed8445d..0151c0d077 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -634,34 +634,6 @@ impl VmmVersionInfo { } } -/// Handle for the [`MigrationWorker`] thread. -struct MigrationWorkerHandle { - // Option to take the inner handle - handle: Option>, -} - -impl MigrationWorkerHandle { - /// Joins the thread and returns the result. - fn join(mut self) -> MigrationThreadOut { - self.handle - .take() - .expect("should have thread") - .join() - .expect("should join migration thread gracefully") - } -} - -impl Drop for MigrationWorkerHandle { - fn drop(&mut self) { - if let Some(handle) = self.handle.take() { - warn!("Migration thread wasn't cleaned up explicitly via join()"); - handle - .join() - .expect("should join migration thread gracefully"); - } - } -} - /// Abstraction for the thread controlling and performing the live migration. /// /// The migration thread also takes ownership of the [`Vm`] from the [`Vmm`]. @@ -704,57 +676,6 @@ impl MigrationWorker { migration_cfg: self.config, } } - - #[expect(clippy::result_large_err)] - fn spawn( - vm: Vm, - check_migration_evt: EventFd, - config: VmSendMigrationData, - postponed_lifecycle_event: Arc>>, - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: Arc< - dyn hypervisor::Hypervisor, - >, - ) -> result::Result { - let worker = MigrationWorker { - vm, - check_migration_evt, - config, - postponed_lifecycle_event, - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - hypervisor, - }; - - // Cumbersome but we need this to take a value from the worker when - // thread spawning failed. Ownership of the worker is either by the - // thread or this function. - let worker = Arc::new(Mutex::new(Some(worker))); - let thread_worker = worker.clone(); - - let inner_handle = thread::Builder::new() - .name("migration".into()) - .spawn(move || { - thread_worker - .lock() - .unwrap() - .take() - .expect("migration worker should only be taken once") - .run() - }) - .context("should spawn migration thread") - .map_err(|e| { - // Get the VM back from the worker. - let worker = worker - .lock() - .unwrap() - .take() - .expect("migration worker should remain available on spawn failure"); - (worker.vm, MigratableError::MigrateSend(e)) - })?; - - Ok(MigrationWorkerHandle { - handle: Some(inner_handle), - }) - } } pub struct VmmThreadHandle { @@ -832,7 +753,9 @@ pub struct Vmm { postponed_lifecycle_event: Arc>>, received_postponed_lifecycle_event: Option, /// Handle to the [`MigrationWorker`] thread. - migration_thread_handle: Option, + /// + /// The handle will return the [`Vm`] back in any case. Further, the underlying error (if any) is returned. + migration_thread_handle: Option>, } /// Just a wrapper for the data that goes into @@ -2145,7 +2068,8 @@ impl Vmm { .migration_thread_handle .take() .expect("should have thread") - .join(); + .join() + .expect("should have joined"); match migration_res { Ok(()) => { @@ -3325,29 +3249,26 @@ impl RequestHandler for Vmm { )); } - // When spawning the thread fails, the VM keeps running normally. - let migration_worker = MigrationWorker::spawn( - vm, - self.check_migration_evt.try_clone().unwrap(), - send_data_migration, - self.postponed_lifecycle_event.clone(), - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - self.hypervisor.clone(), - ) - .map_err(|(vm, e)| { - self.vm = MaybeVmOwnership::Vmm(vm); - - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_failed(&e); - - e - })?; - let old = self.migration_thread_handle.replace(migration_worker); - // If this fails, we messed up the thread lifecycle management. - debug_assert!(old.is_none()); + // Start migration thread + { + let worker = MigrationWorker { + vm, + check_migration_evt: self.check_migration_evt.try_clone().unwrap(), + config: send_data_migration, + postponed_lifecycle_event: self.postponed_lifecycle_event.clone(), + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + hypervisor: self.hypervisor.clone(), + }; + self.migration_thread_handle = Some( + thread::Builder::new() + .name("migration".into()) + .spawn(move || worker.run()) + // For upstreaming, we should simply continue and return an + // error when this fails. For our PoC, this is fine. + .unwrap(), + ); + } Ok(()) } From 3cc95e1fdc2244cfacc2358bbe78a585112a5b10 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:14:02 +0200 Subject: [PATCH 21/79] vmm: Revert "http api: add VmCancelMigration action" This reverts commit 9fa49fd84607f3edadc7e062376401ee974eb545. The import lists also carry VmPostMigrationAnnounce, which stays. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 8 +++----- vmm/src/api/http/mod.rs | 12 ++++-------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index 57aa6c4469..b5d5e4ded9 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -48,10 +48,9 @@ use crate::api::http::{EndpointHandler, HttpError, error_response}; use crate::api::{ AddDisk, ApiAction, ApiError, ApiRequest, NetConfig, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, - VmCancelMigration, VmConfig, VmCounters, VmDelete, VmMigrationProgress, VmNmi, VmPause, - VmPostMigrationAnnounce, VmPowerButton, VmReboot, VmReceiveMigration, VmReceiveMigrationData, - VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, - VmShutdown, VmSnapshot, + VmConfig, VmCounters, VmDelete, VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, + VmPowerButton, VmReboot, VmReceiveMigration, VmReceiveMigrationData, VmRemoveDevice, VmResize, + VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, }; use crate::config::RestoreConfig; use crate::cpu::Error as CpuError; @@ -419,7 +418,6 @@ vm_action_put_handler!(VmResume); vm_action_put_handler!(VmPostMigrationAnnounce); vm_action_put_handler!(VmPowerButton); vm_action_put_handler!(VmNmi); -vm_action_put_handler!(VmCancelMigration); vm_action_put_handler_body!(VmAddDevice); vm_action_put_handler_body!(AddDisk); diff --git a/vmm/src/api/http/mod.rs b/vmm/src/api/http/mod.rs index 5464ca87ab..5ac3b35672 100644 --- a/vmm/src/api/http/mod.rs +++ b/vmm/src/api/http/mod.rs @@ -29,10 +29,10 @@ use self::http_endpoint::{VmActionHandler, VmCreate, VmInfo, VmmPing, VmmShutdow use crate::api::VmCoredump; use crate::api::{ AddDisk, ApiError, ApiRequest, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, - VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCancelMigration, VmCounters, - VmDelete, VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, - VmReboot, VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, - VmResume, VmSendMigration, VmShutdown, VmSnapshot, + VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, + VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, VmReboot, + VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, + VmSendMigration, VmShutdown, VmSnapshot, }; use crate::landlock::Landlock; use crate::seccomp_filters::{Thread, get_seccomp_filter}; @@ -282,10 +282,6 @@ pub static HTTP_ROUTES: LazyLock = LazyLock::new(|| { endpoint!("/vm.send-migration"), Box::new(VmActionHandler::new(&VmSendMigration)), ); - r.routes.insert( - endpoint!("/vm.cancel-migration"), - Box::new(VmActionHandler::new(&VmCancelMigration)), - ); r.routes.insert( endpoint!("/vm.shutdown"), Box::new(VmActionHandler::new(&VmShutdown)), From eb382e3b0e2dfbd15227b2447708d16a7bf8c9c2 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:14:31 +0200 Subject: [PATCH 22/79] vmm: Revert "api: add VmCancelMigration action" This reverts commit 1688e55c3acdb219a8109745230e09ede827d53c. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- fuzz/fuzz_targets/http_api.rs | 4 ---- vm-migration/src/lib.rs | 3 --- vmm/src/api/mod.rs | 44 ----------------------------------- vmm/src/lib.rs | 13 ----------- 4 files changed, 64 deletions(-) diff --git a/fuzz/fuzz_targets/http_api.rs b/fuzz/fuzz_targets/http_api.rs index aa3841243d..0273d5b455 100644 --- a/fuzz/fuzz_targets/http_api.rs +++ b/fuzz/fuzz_targets/http_api.rs @@ -310,10 +310,6 @@ impl RequestHandler for StubApiRequestHandler { None } - fn vm_cancel_migration(&mut self) -> Result<(), MigratableError> { - Ok(()) - } - fn vm_post_migration_announce(&mut self) -> Result<(), VmError> { Ok(()) } diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 89cbcecf87..3b5f25987c 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -86,9 +86,6 @@ pub enum MigratableError { #[error("Failed to retrieve dirty ranges for migratable component")] DirtyLog(#[source] anyhow::Error), - #[error("Failed to cancel migration")] - CancelMigration(#[source] anyhow::Error), - #[error("Failed to start migration for migratable component")] StartMigration(#[source] anyhow::Error), diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 280196157b..60172d5d56 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -209,10 +209,6 @@ pub enum ApiError { #[error("Error starting migration sender")] VmSendMigration(#[source] MigratableError), - /// Error cancelling migration - #[error("Error cancelling migration")] - VmCancelMigration(#[source] MigratableError), - /// Error triggering power button #[error("Error triggering power button")] VmPowerButton(#[source] VmError), @@ -788,18 +784,11 @@ pub trait RequestHandler { receive_data_migration: VmReceiveMigrationData, ) -> Result<(), MigratableError>; - /// Dispatches the migration. fn vm_send_migration( &mut self, send_data_migration: VmSendMigrationData, ) -> Result<(), MigratableError>; - /// Triggers a migration cancellation. - /// - /// The cancellation is not guaranteed to succeed, as the migration may have - /// succeeded already. - fn vm_cancel_migration(&mut self) -> Result<(), MigratableError>; - fn vm_nmi(&mut self) -> Result<(), VmError>; /// Returns the progress of the currently active migration or any previous @@ -1550,39 +1539,6 @@ impl ApiAction for VmReceiveMigration { } } -pub struct VmCancelMigration; - -impl ApiAction for VmCancelMigration { - type RequestBody = (); - type ResponseBody = Option; - - fn request(&self, data: Self::RequestBody, response_sender: Sender) -> ApiRequest { - Box::new(move |vmm| { - info!("API request event: VmCancelMigration {data:?}"); - - let response = vmm - .vm_cancel_migration() - .map_err(ApiError::VmCancelMigration) - .map(|_| ApiResponsePayload::Empty); - - response_sender - .send(response) - .map_err(VmmError::ApiResponseSend)?; - - Ok(false) - }) - } - - fn send( - &self, - api_evt: EventFd, - api_sender: Sender, - data: Self::RequestBody, - ) -> ApiResult { - get_response_body(self, api_evt, api_sender, data) - } -} - pub struct VmRemoveDevice; impl ApiAction for VmRemoveDevice { diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 0151c0d077..b9c9f6a118 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -3272,19 +3272,6 @@ impl RequestHandler for Vmm { Ok(()) } - fn vm_cancel_migration(&mut self) -> result::Result<(), MigratableError> { - match self.vm { - MaybeVmOwnership::Migration => (), - _ => { - return Err(MigratableError::CancelMigration(anyhow!( - "There is no ongoing migration" - ))); - } - } - - todo!() - } - fn vm_migration_progress(&mut self) -> Option { // We explicitly do not check here for `is VM running?` to always // enable querying the state of the last failed migration. From afc30faa5e9c3c7f63b20c1e6f09f6b4463e5794 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:14:38 +0200 Subject: [PATCH 23/79] vmm: Revert "streamline migration failure cleanup" This reverts commit 838ea2e71b500a2d7cc20c2dd59c6abbb79ce178. This commit cleaned up the fork's own migration worker. Upstream's worker, which this series cherry-picks, already has the resulting shape, so the commit is not re-applied. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 57 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index b9c9f6a118..c50c1a196d 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -648,23 +648,46 @@ struct MigrationWorker { } impl MigrationWorker { - /// Perform the migration and communicate with the [`Vmm`] thread. - fn run(mut self) -> MigrationThreadOut { - debug!("migration thread is starting"); - event!("vm", "migration-started"); + /// Performs any final cleanup after failed live migrations. + /// + /// Helper for [`Self::migrate`]. + fn migrate_error_cleanup(&mut self) -> result::Result<(), MigratableError> { + // Stop logging dirty pages only for non-local migrations + if !self.config.local { + self.vm.stop_dirty_log()?; + } - let res = Vmm::send_migration( + Ok(()) + } + + /// Migrate and cleanup. + fn migrate(&mut self) -> result::Result<(), MigratableError> { + debug!("start sending migration"); + event!("vm", "migration-started"); + Vmm::send_migration( &mut self.vm, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - self.hypervisor.as_ref(), + self.hypervisor.as_ref(), &self.config, self.postponed_lifecycle_event.as_ref(), ) - .inspect(|_| event!("vm", "migration-finished")) - .inspect_err(|e| { - event!("vm", "migration-failed"); - error!("migrate error: {e}"); - }); + .inspect(|_| event!("vm", "migration-finished")) + .inspect_err(|_| { + event!("vm", "migration-failed"); + let e = self.migrate_error_cleanup(); + if let Err(e) = e { + error!("Failed to clean up after a failed live migration. VM might keep running but in an odd or possibly slowed-down state: {e}"); + } + })?; + + Ok(()) + } + + /// Perform the migration and communicate with the [`Vmm`] thread. + fn run(mut self) -> MigrationThreadOut { + debug!("migration thread is starting"); + + let res = self.migrate().inspect_err(|e| error!("migrate error: {e}")); // Notify VMM thread to get migration result by joining this thread. self.check_migration_evt.write(1).unwrap(); @@ -2095,7 +2118,7 @@ impl Vmm { } Err(e) => { error!("Migration failed: {e}"); - // We don't fail the VMM here, it just continues running its VM. + // If the failure happened very late in the migration path, the VM might already be // stopped. We resume it to ensure proper operation. // @@ -2105,6 +2128,11 @@ impl Vmm { match vm.resume() { Ok(_) => { info!("Resumed VM successfully after failed migration"); + + // Ensure full VM performance. The operation is idempotent. + let _ = vm.stop_dirty_log().inspect_err(|e| { + warn!("Failed stopping dirty log after resuming VM: {e} - VM performance might be slower than usual"); + }); } Err(e) => { error!("Failed resuming VM after failed migration: {e}"); @@ -2113,11 +2141,6 @@ impl Vmm { } } - // Ensure full VM performance. The operation is idempotent. - let _ = vm.stop_dirty_log().inspect_err(|e| { - warn!("Failed stopping dirty log after resuming VM: {e} - VM performance might be slower than usual"); - }); - // Give VMM back control. self.vm = MaybeVmOwnership::Vmm(vm); if let Some(event) = self.current_postponed_lifecycle_event() { From 92a03cc9a50e45239967405c7ea4e19b81c92739 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:14:38 +0200 Subject: [PATCH 24/79] vmm: Revert "migration: switch downtime on postponed event" This reverts commit fe3709fd46ffed12d620b4a25281de3bd6411947. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index c50c1a196d..2685168d22 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1019,10 +1019,6 @@ impl Vmm { } } - fn current_postponed_lifecycle_event(&self) -> Option { - *self.postponed_lifecycle_event.lock().unwrap() - } - fn clear_postponed_lifecycle_event(&self) { let mut postponed_event = self.postponed_lifecycle_event.lock().unwrap(); *postponed_event = None; @@ -1495,7 +1491,6 @@ impl Vmm { ctx: &mut MemoryMigrationContext, is_converged: impl Fn(&MemoryMigrationContext) -> result::Result, mem_send: &mut SendAdditionalConnections, - postponed_lifecycle_event: &Mutex>, ) -> result::Result { let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -1581,16 +1576,6 @@ impl Vmm { // Increment iteration last: This way we ensure that the logging // above matches the actual iteration. ctx.iteration += 1; - - let event = *postponed_lifecycle_event.lock().unwrap(); - if let Some(event) = event { - info!( - "Lifecycle event postponed during migration ({event:?}), switching to downtime phase early" - ); - // The current iteration has already been sent, therefore no extra range - // needs to be carried into the final transfer batch. - break Ok(MemoryRangeTable::default()); - } } } @@ -1700,7 +1685,6 @@ impl Vmm { send_data_migration: &VmSendMigrationData, mem_send: &mut SendAdditionalConnections, ctx: &mut OngoingMigrationContext, - postponed_lifecycle_event: &Mutex>, ) -> result::Result<(), MigratableError> { let mut mem_ctx = MemoryMigrationContext::new(); @@ -1712,7 +1696,6 @@ impl Vmm { // We bind send_data_migration to the callback |ctx| Self::is_precopy_converged(ctx, send_data_migration), mem_send, - postponed_lifecycle_event, )?; let downtime_begin = Instant::now(); // End throttle thread @@ -1875,7 +1858,6 @@ impl Vmm { send_data_migration, &mut mem_send, &mut ctx, - postponed_lifecycle_event, ) .inspect_err(|_| { // Calling cleanup multiple times is fine, thus here we just make sure @@ -2143,24 +2125,7 @@ impl Vmm { // Give VMM back control. self.vm = MaybeVmOwnership::Vmm(vm); - if let Some(event) = self.current_postponed_lifecycle_event() { - match event { - PostMigrationLifecycleEvent::VmReboot => { - self.reset_evt - .write(1) - .context("Failed replaying reset event after failed migration") - .inspect_err(|write_err| error!("{write_err}")) - .ok(); - } - PostMigrationLifecycleEvent::VmmShutdown => { - self.exit_evt - .write(1) - .context("Failed replaying shutdown event after failed migration") - .inspect_err(|write_err| error!("{write_err}")) - .ok(); - } - } - } + // Update migration progress snapshot { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -2170,7 +2135,6 @@ impl Vmm { } } } - self.clear_postponed_lifecycle_event(); } fn control_loop( From 8fcc2281c6952314fcd7f36cd03b577a3402f017 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:15:12 +0200 Subject: [PATCH 25/79] vmm: Revert "postpone reset/exit during migration" This reverts commit 23414bb4b063f4b1e329556937b31e7b57fe9f61. The post-migration announce call added later by 3655c61c5 stays. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 87 ++++++-------------------------------------------- 1 file changed, 9 insertions(+), 78 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 2685168d22..b95f53aafe 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -72,12 +72,14 @@ use crate::coredump::GuestDebuggable; use crate::cpu::IS_IN_SHUTDOWN; use crate::landlock::Landlock; use crate::memory_manager::MemoryManager; -use crate::migration::{get_vm_snapshot, recv_vm_config, recv_vm_state}; +#[cfg(all(feature = "kvm", target_arch = "x86_64"))] +use crate::migration::get_vm_snapshot; +use crate::migration::{recv_vm_config, recv_vm_state}; use crate::migration_transport::{ ReceiveAdditionalConnections, ReceiveListener, SendAdditionalConnections, SocketStream, }; use crate::seccomp_filters::{Thread, get_seccomp_filter}; -use crate::vm::{Error as VmError, PostMigrationLifecycleEvent, Vm, VmState}; +use crate::vm::{Error as VmError, Vm, VmState}; use crate::vm_config::{ DeviceConfig, DiskConfig, FsConfig, GenericVhostUserConfig, MemoryZoneConfig, NetConfig, PmemConfig, UserDeviceConfig, VdpaConfig, VmConfig, VsockConfig, @@ -641,8 +643,6 @@ struct MigrationWorker { vm: Vm, check_migration_evt: EventFd, config: VmSendMigrationData, - // Shared with main VMM thread - postponed_lifecycle_event: Arc>>, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: Arc, } @@ -669,7 +669,6 @@ impl MigrationWorker { #[cfg(all(feature = "kvm", target_arch = "x86_64"))] self.hypervisor.as_ref(), &self.config, - self.postponed_lifecycle_event.as_ref(), ) .inspect(|_| event!("vm", "migration-finished")) .inspect_err(|_| { @@ -773,8 +772,6 @@ pub struct Vmm { console_info: Option, no_shutdown: bool, check_migration_evt: EventFd, - postponed_lifecycle_event: Arc>>, - received_postponed_lifecycle_event: Option, /// Handle to the [`MigrationWorker`] thread. /// /// The handle will return the [`Vm`] back in any case. Further, the underlying error (if any) is returned. @@ -1005,25 +1002,10 @@ impl Vmm { console_info: None, no_shutdown, check_migration_evt, - postponed_lifecycle_event: Arc::new(Mutex::new(None)), - received_postponed_lifecycle_event: None, migration_thread_handle: None, }) } - fn postpone_lifecycle_event_during_migration(&self, event: PostMigrationLifecycleEvent) { - let mut postponed_event = self.postponed_lifecycle_event.lock().unwrap(); - if postponed_event.is_none() { - *postponed_event = Some(event); - info!("Postponed post-migration lifecycle event: {event:?}"); - } - } - - fn clear_postponed_lifecycle_event(&self) { - let mut postponed_event = self.postponed_lifecycle_event.lock().unwrap(); - *postponed_event = None; - } - /// Try to receive a file descriptor from a socket. Returns the slot number and the file descriptor. fn vm_receive_memory_fd( socket: &mut SocketStream, @@ -1192,34 +1174,11 @@ impl Vmm { // The thread in background periodically sends multiple messages. vm.post_migration_announce(); - // We are on the control-loop thread handling an API request, so - // there is no concurrent access from other VMM or migration - // threads. The VM is in the Paused state , which permits both - // the Running transition (resume) and the Shutdown transition (reboot / exit) - // triggered via the eventfds below. - match self.received_postponed_lifecycle_event { - None => { - let (_, resume_duration) = measure_ok(|| vm.resume())?; - debug!( - "Migration (incoming): resume:{}ms", - resume_duration.as_millis() - ); - } - Some(PostMigrationLifecycleEvent::VmReboot) => { - self.reset_evt - .write(1) - .context("Failed writing reset eventfd after migration") - .map_err(MigratableError::MigrateReceive)?; - } - Some(PostMigrationLifecycleEvent::VmmShutdown) => { - self.exit_evt - .write(1) - .context("Failed writing exit eventfd after migration") - .map_err(MigratableError::MigrateReceive)?; - } - } - self.received_postponed_lifecycle_event = None; - + let (_, resume_duration) = measure_ok(|| vm.resume())?; + debug!( + "Migration (incoming): resume:{}ms", + resume_duration.as_millis() + ); // This logs the downtime without the final memory delta, so // it does not reflect the actual downtime. While we could // pass along the timestamp from when the VM was paused, @@ -1397,11 +1356,6 @@ impl Vmm { .map_err(MigratableError::MigrateReceive) })?; - let vm_snapshot = get_vm_snapshot(&snapshot) - .context("Failed extracting VM snapshot data") - .map_err(MigratableError::MigrateReceive)?; - self.received_postponed_lifecycle_event = vm_snapshot.post_migration_lifecycle_event; - let exit_evt = self .exit_evt .try_clone() @@ -1739,7 +1693,6 @@ impl Vmm { #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: &dyn hypervisor::Hypervisor, send_data_migration: &VmSendMigrationData, - postponed_lifecycle_event: &Mutex>, ) -> result::Result<(), MigratableError> { // State machine that is updated with more context as we progress. let mut ctx = OngoingMigrationContext::new(); @@ -1892,7 +1845,6 @@ impl Vmm { } // Capture snapshot and send it - vm.set_post_migration_lifecycle_event(*postponed_lifecycle_event.lock().unwrap()); let (vm_snapshot, snapshot_duration) = measure_ok(|| vm.snapshot())?; let (_, send_snapshot_duration) = measure_ok(|| migration_transport::send_state(&mut socket, &vm_snapshot))?; @@ -2176,13 +2128,6 @@ impl Vmm { info!("VM exit event"); // Consume the event. self.exit_evt.read().map_err(Error::EventFdRead)?; - // Workaround for guest-induced shutdown during a live-migration. - if matches!(self.vm, MaybeVmOwnership::Migration) { - self.postpone_lifecycle_event_during_migration( - PostMigrationLifecycleEvent::VmmShutdown, - ); - continue; - } self.vmm_shutdown().map_err(Error::VmmShutdown)?; break 'outer; @@ -2191,13 +2136,6 @@ impl Vmm { info!("VM reset event"); // Consume the event. self.reset_evt.read().map_err(Error::EventFdRead)?; - // Workaround for guest-induced shutdown during a live-migration. - if matches!(self.vm, MaybeVmOwnership::Migration) { - self.postpone_lifecycle_event_during_migration( - PostMigrationLifecycleEvent::VmReboot, - ); - continue; - } self.vm_reboot().map_err(Error::VmReboot)?; } EpollDispatch::GuestExit => { @@ -3066,9 +3004,6 @@ impl RequestHandler for Vmm { .context("Invalid receive migration configuration") .map_err(MigratableError::MigrateReceive)?; - // Prevent stale lifecycle intent from a previous failed receive attempt. - self.received_postponed_lifecycle_event = None; - info!( "Receiving migration: receiver_url={},tls={},net_fds={:?}, tcp_url={:?}, zones={:?}", receive_data_migration.receiver_url, @@ -3172,9 +3107,6 @@ impl RequestHandler for Vmm { send_data_migration.timeout_strategy ); - // New migration attempt: clear postponed lifecycle from any previous run. - self.clear_postponed_lifecycle_event(); - if !self .vm_config .as_ref() @@ -3242,7 +3174,6 @@ impl RequestHandler for Vmm { vm, check_migration_evt: self.check_migration_evt.try_clone().unwrap(), config: send_data_migration, - postponed_lifecycle_event: self.postponed_lifecycle_event.clone(), #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: self.hypervisor.clone(), }; From 17bac69b394332c852c32ffc9b6c99fcf4d40d11 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:15:22 +0200 Subject: [PATCH 26/79] vmm: Revert "add post-migration event to VmSnapshot" This reverts commit fbb64c8a46bb8899d9b8ab90088c0563afb736cc. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/vm.rs | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 18f9200755..0725c63268 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -108,9 +108,11 @@ use crate::landlock::LandlockError; use crate::memory_manager::{ Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData, }; +#[cfg(target_arch = "x86_64")] +use crate::migration::get_vm_snapshot; #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] use crate::migration::url_to_file; -use crate::migration::{SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE, get_vm_snapshot, url_to_path}; +use crate::migration::{SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE, url_to_path}; #[cfg(all( feature = "kvm", feature = "sev_snp", @@ -581,13 +583,6 @@ pub struct Vm { stop_on_boot: bool, load_payload_handle: Option>>, vcpu_throttler: ThrottleThreadHandle, - post_migration_lifecycle_event: Option, -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum PostMigrationLifecycleEvent { - VmReboot, - VmmShutdown, } impl Vm { @@ -746,15 +741,6 @@ impl Vm { } else { VmState::Created }; - let post_migration_lifecycle_event = snapshot - .as_ref() - .map(|snapshot| { - get_vm_snapshot(snapshot) - .map(|vm_snapshot| vm_snapshot.post_migration_lifecycle_event) - .map_err(Error::Restore) - }) - .transpose()? - .flatten(); // TODO we could also spawn the thread when a migration with auto-converge starts. // Probably this is the better design. @@ -780,7 +766,6 @@ impl Vm { stop_on_boot, load_payload_handle, vcpu_throttler, - post_migration_lifecycle_event, }) } @@ -1502,17 +1487,6 @@ impl Vm { self.vcpu_throttler.shutdown(); } - pub fn set_post_migration_lifecycle_event( - &mut self, - event: Option, - ) { - self.post_migration_lifecycle_event = event; - } - - pub fn post_migration_lifecycle_event(&self) -> Option { - self.post_migration_lifecycle_event - } - #[allow(clippy::too_many_arguments)] pub fn new( vm_config: Arc>, @@ -3489,8 +3463,6 @@ impl Pausable for Vm { #[derive(Serialize, Deserialize)] pub struct VmSnapshot { - #[serde(default)] - pub post_migration_lifecycle_event: Option, #[cfg(target_arch = "x86_64")] pub clock: Option, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -3553,7 +3525,6 @@ impl Snapshottable for Vm { }; let vm_snapshot_state = VmSnapshot { - post_migration_lifecycle_event: self.post_migration_lifecycle_event(), #[cfg(target_arch = "x86_64")] clock: self.saved_clock, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] From 362a302ffeda4bf6069ece374670984db42917fe Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:15:22 +0200 Subject: [PATCH 27/79] vmm: Revert "api: less verbose log" This reverts commit 98405450b273d3d239eb56556f88be53ba70f7a2. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 60172d5d56..b179468349 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -40,7 +40,7 @@ use std::str::FromStr; use std::sync::mpsc::{RecvError, SendError, Sender, channel}; use std::time::Duration; -use log::{debug, info}; +use log::info; use micro_http::Body; use option_parser::{OptionParser, OptionParserError, Toggle}; use serde::{Deserialize, Serialize}; @@ -1377,7 +1377,7 @@ impl ApiAction for VmInfo { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - debug!("API request event: VmInfo"); + info!("API request event: VmInfo"); let response = vmm .vm_info() @@ -2005,7 +2005,7 @@ impl ApiAction for VmMigrationProgress { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - debug!("API request event: VmMigrationProgress"); + info!("API request event: VmMigrationProgress"); let snapshot = Ok(vmm.vm_migration_progress()); let response = snapshot From a8f5c5a91468bb0146d9595abeab6b90be14f12a Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:15:40 +0200 Subject: [PATCH 28/79] ch-remote: Revert "wait for migration to finish via progress" This reverts commit 8bbea46a0cd9028479a62098863054865c08d0aa. The original subject was "ch-remote: wait for migration to finish by querying migration progress". This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- Cargo.lock | 1 - cloud-hypervisor/Cargo.toml | 1 - cloud-hypervisor/src/bin/ch-remote.rs | 83 ++------------------------- 3 files changed, 4 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5689e0e2f..201acf7fbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -508,7 +508,6 @@ dependencies = [ "tpm", "tracer", "vm-memory", - "vm-migration", "vmm", "vmm-sys-util", "wait-timeout", diff --git a/cloud-hypervisor/Cargo.toml b/cloud-hypervisor/Cargo.toml index 73a74be6f4..352a53b1bc 100644 --- a/cloud-hypervisor/Cargo.toml +++ b/cloud-hypervisor/Cargo.toml @@ -30,7 +30,6 @@ thiserror = { workspace = true } tpm = { path = "../tpm" } tracer = { path = "../tracer" } vm-memory = { workspace = true } -vm-migration = { path = "../vm-migration" } vmm = { path = "../vmm" } vmm-sys-util = { workspace = true } zbus = { version = "5.15.0", optional = true } diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index f8075de113..4a64c621cd 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -11,20 +11,17 @@ use std::io::Read; use std::marker::PhantomData; use std::os::unix::net::UnixStream; use std::process; -use std::thread::sleep; -use std::time::Duration; use api_client::{ - Error as ApiClientError, StatusCode, simple_api_command, simple_api_command_with_fds, - simple_api_full_command, simple_api_full_command_and_response, + Error as ApiClientError, simple_api_command, simple_api_command_with_fds, + simple_api_full_command, }; #[cfg(feature = "dbus_api")] use clap::ArgAction; use clap::{Arg, ArgMatches, Command}; -use log::{error, info}; +use log::error; use option_parser::{ByteSized, ByteSizedParseError}; use thiserror::Error; -use vm_migration::progress::{MigrationProgress, MigrationState}; use vmm::config::RestoreConfig; use vmm::vm_config::{ DeviceConfig, DiskConfig, FsConfig, GenericVhostUserConfig, NetConfig, PmemConfig, @@ -534,14 +531,6 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu .map_err(Error::HttpApiClient) } Some("send-migration") => { - let just_dispatch = matches - .subcommand_matches("send-migration") - .unwrap() - .get_one::("dispatch") - .cloned() - .unwrap_or(false); - let wait_for_migration = !just_dispatch; - let send_migration_data = send_migration_data( matches .subcommand_matches("send-migration") @@ -550,65 +539,7 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu .unwrap(), )?; simple_api_command(socket, "PUT", "send-migration", Some(&send_migration_data)) - .map_err(Error::HttpApiClient)?; - - if !wait_for_migration { - return Ok(()); - } - loop { - let response = simple_api_full_command_and_response( - socket, - "GET", - "vm.migration-progress", - None, - ) - .map_err(Error::HttpApiClient)? - // should have response - .ok_or(Error::HttpApiClient(ApiClientError::ServerResponse( - StatusCode::Ok, - None, - )))?; - - // This is guaranteed by the SendMigration call - assert_ne!( - response, "null", - "migration progress should be there immediately when the migration was dispatched" - ); - - let progress = serde_json::from_slice::(response.as_bytes()) - .map_err(|e| { - error!("failed to parse response as MigrationProgress: {e}"); - Error::HttpApiClient(ApiClientError::ServerResponse( - StatusCode::Ok, - Some(response), - )) - })?; - - match progress.state { - MigrationState::Cancelled { .. } => { - info!("Migration was cancelled"); - break; - } - MigrationState::Failed { - error_msg, - error_msg_debug, - } => { - error!("Migration failed! {error_msg}\n{error_msg_debug}"); - break; - } - MigrationState::Finished { .. } => { - info!("Migration finished successfully. Shutting down Cloud Hypervisor"); - simple_api_full_command(socket, "PUT", "vmm.shutdown", None) - .map_err(Error::HttpApiClient)?; - break; - } - MigrationState::Ongoing { .. } => { - sleep(Duration::from_millis(50)); - continue; - } - } - } - Ok(()) + .map_err(Error::HttpApiClient) } Some("receive-migration") => { let receive_migration_data = receive_migration_data( @@ -1216,12 +1147,6 @@ fn get_cli_commands_sorted() -> Box<[Command]> { Command::new("resume").about("Resume the VM"), Command::new("send-migration") .about("Initiate a VM migration") - .arg( - Arg::new("dispatch") - .long("dispatch") - .help("just dispatch the migration without waiting for it to finish") - .num_args(0), - ) .arg( Arg::new("send_migration_config") .index(1) From ea1d1397d37cb4e9d7bd7bd0a241e692fd21039d Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:15:40 +0200 Subject: [PATCH 29/79] ch-remote: Revert "add `migration-progress` command" This reverts commit d9a820606a888f1fd8768ab8314ea6eccdf1103c. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/src/bin/ch-remote.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index 4a64c621cd..f9675e3946 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -327,8 +327,6 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu Some("shutdown") => { simple_api_command(socket, "PUT", "shutdown", None).map_err(Error::HttpApiClient) } - Some("migration-progress") => simple_api_command(socket, "GET", "migration-progress", None) - .map_err(Error::HttpApiClient), Some("nmi") => simple_api_command(socket, "PUT", "nmi", None).map_err(Error::HttpApiClient), Some("resize") => { let resize = resize_config( @@ -1072,7 +1070,6 @@ fn get_cli_commands_sorted() -> Box<[Command]> { .arg(Arg::new("path").index(1).default_value("-")), Command::new("delete").about("Delete a VM"), Command::new("info").about("Info on the VM"), - Command::new("migration-progress"), Command::new("nmi").about("Trigger NMI"), Command::new("pause").about("Pause the VM"), Command::new("ping").about("Ping the VMM to check for API server availability"), From 06a8b76a701734eb863d8930f8645c34fdbfd1b4 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:15:40 +0200 Subject: [PATCH 30/79] vmm: Revert "migration: switch to non-blocking SendMigration call" This reverts commit cf187423374a7c6a911463f1d8fa9743ee37ceb0. The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 42 +++++++++++++++++++++++-------- vmm/src/api/mod.rs | 15 ++--------- vmm/src/lib.rs | 39 +++++++++++++++++----------- 3 files changed, 58 insertions(+), 38 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index b5d5e4ded9..456da87395 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -35,12 +35,23 @@ //! [special HTTP library]: https://github.com/firecracker-microvm/micro-http use std::fs::File; -use std::sync::mpsc::Sender; +use std::sync::mpsc::{Receiver, Sender, SyncSender}; +use std::sync::{LazyLock, Mutex}; use log::info; use micro_http::{Body, Method, Request, Response, StatusCode, Version}; use vmm_sys_util::eventfd::EventFd; +/// Helper to make the VmSendMigration call blocking as long as a migration is ongoing. +#[allow(clippy::type_complexity)] +pub static ONGOING_LIVEMIGRATION: LazyLock<( + SyncSender>, + Mutex>>, +)> = LazyLock::new(|| { + let (sender, receiver) = std::sync::mpsc::sync_channel(0); + (sender, Mutex::new(receiver)) +}); + #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] use crate::api::VmCoredump; use crate::api::http::http_endpoint::fds_helper::{attach_fds_to_cfg, attach_fds_to_cfgs}; @@ -499,15 +510,26 @@ impl PutHandler for VmSendMigration { _files: Vec, ) -> std::result::Result, HttpError> { if let Some(body) = body { - self.send( - api_notifier, - api_sender, - serde_json::from_slice(body.raw())?, - ) - .inspect(|_| { - info!("live migration started (in background)"); - }) - .map_err(HttpError::ApiError) + let res = self + .send( + api_notifier, + api_sender, + serde_json::from_slice(body.raw())?, + ) + .map_err(HttpError::ApiError)?; + + info!("live migration started"); + + let (_, receiver) = &*ONGOING_LIVEMIGRATION; + + info!("waiting for live migration result"); + let mig_res = receiver.lock().unwrap().recv().unwrap(); + info!("received live migration result"); + + // We forward the migration error here to the guest + mig_res + .map(|_| res) + .map_err(|e| HttpError::ApiError(ApiError::VmSendMigration(e))) } else { Err(HttpError::BadRequest) } diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index b179468349..557a0205b4 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -498,8 +498,6 @@ pub struct VmSendMigrationData { /// Path to the directory containing the TLS root CA certificate (ca-cert.pem), the TLS client certificate (client-cert.pem), and TLS client key (client-key.pem). #[serde(default)] pub tls_dir: Option, - /// Keep the VMM alive. - pub keep_alive: bool, } impl VmSendMigrationData { @@ -536,8 +534,7 @@ impl VmSendMigrationData { .add("timeout_s") .add("timeout_strategy") .add("connections") - .add("tls_dir") - .add("keep_alive"); + .add("tls_dir"); parser .parse(migration) .map_err(VmSendMigrationConfigError::ParseError)?; @@ -593,11 +590,6 @@ impl VmSendMigrationData { .convert::("tls_dir") .map_err(VmSendMigrationConfigError::ParseError)? .map(|path| PathBuf::from(&path)); - let keep_alive = parser - .convert::("keep_alive") - .map_err(VmSendMigrationConfigError::ParseError)? - .unwrap_or(Toggle(false)) - .0; let data = Self { destination_url, @@ -607,7 +599,6 @@ impl VmSendMigrationData { timeout_strategy, connections, tls_dir, - keep_alive, }; data.validate()?; @@ -2228,14 +2219,13 @@ mod unit_tests { timeout_strategy: Default::default(), connections: VmSendMigrationData::default_connections(), tls_dir: None, - keep_alive: false, } ); // Happy path, fully specified let tls_dir = std::env::temp_dir(); let data = - VmSendMigrationData::parse(&format!("destination_url=tcp:192.168.1.1:8080,downtime_ms=150,timeout_s=900,timeout_strategy=ignore,connections=4,tls_dir={},keep_alive=true", tls_dir.display())) + VmSendMigrationData::parse(&format!("destination_url=tcp:192.168.1.1:8080,downtime_ms=150,timeout_s=900,timeout_strategy=ignore,connections=4,tls_dir={}", tls_dir.display())) .unwrap(); assert_eq!( data, @@ -2247,7 +2237,6 @@ mod unit_tests { timeout_strategy: TimeoutStrategy::Ignore, connections: NonZeroU32::new(4).unwrap(), tls_dir: Some(tls_dir), - keep_alive: true } ); } diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index b95f53aafe..91357cc294 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -61,6 +61,7 @@ use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::unblock_signal; use vmm_sys_util::sock_ctrl_msg::ScmSocket; +use crate::api::http::http_endpoint::ONGOING_LIVEMIGRATION; use crate::api::{ ApiRequest, ApiResponse, RequestHandler, TimeoutStrategy, VmInfoResponse, VmReceiveMigrationData, VmSendMigrationData, VmmPingResponse, @@ -695,7 +696,6 @@ impl MigrationWorker { MigrationThreadOut { vm: self.vm, migration_res: res, - migration_cfg: self.config, } } } @@ -746,7 +746,6 @@ impl MaybeVmOwnership { struct MigrationThreadOut { vm: Vm, migration_res: result::Result<(), MigratableError>, - migration_cfg: VmSendMigrationData, } pub struct Vmm { @@ -1878,6 +1877,14 @@ impl Vmm { vm.stop_dirty_log()?; } + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_finished(); + } + // Let every Migratable object know about the migration being complete vm.complete_migration() } @@ -2020,7 +2027,6 @@ impl Vmm { let MigrationThreadOut { mut vm, migration_res, - migration_cfg, } = self .migration_thread_handle .take() @@ -2034,20 +2040,15 @@ impl Vmm { drop(vm); { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_finished(); + info!("Sending Receiver in HTTP thread that migration succeeded"); + let (sender, _) = &*ONGOING_LIVEMIGRATION; + // unblock API call; propagate migration result + sender.send(Ok(())).unwrap(); } - if migration_cfg.keep_alive { - // API users can still query live-migration statistics - info!("Keeping VMM alive as requested"); - } else { - // Shutdown the VM after the migration succeeded - if let Err(e) = self.exit_evt.write(1) { - error!("Failed shutting down the VM after migration: {e}"); - } + // Shutdown the VM after the migration succeeded + if let Err(e) = self.exit_evt.write(1) { + error!("Failed shutting down the VM after migration: {e}"); } } Err(e) => { @@ -2085,6 +2086,14 @@ impl Vmm { .expect("live migration should be ongoing") .mark_as_failed(&e); } + + { + info!("Sending Receiver in HTTP thread that migration failed"); + let (sender, _) = &*ONGOING_LIVEMIGRATION; + // unblock API call; propagate migration result + sender.send(Err(e)).unwrap(); + } + // we don't fail the VMM here, it just continues running its VM } } } From 8bd22d8aa81759c99f2509f1e5126d79372b9abc Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:15:40 +0200 Subject: [PATCH 31/79] vmm: Revert "actually populate migration progress" This reverts commit fe5387d44fa578d3bcd99d25f098dcdfb8c597f8. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/context.rs | 4 +- vmm/src/lib.rs | 138 +++++------------------------------- 2 files changed, 19 insertions(+), 123 deletions(-) diff --git a/vm-migration/src/context.rs b/vm-migration/src/context.rs index 8e4d28c4f0..21801c0290 100644 --- a/vm-migration/src/context.rs +++ b/vm-migration/src/context.rs @@ -225,13 +225,13 @@ pub struct MemoryMigrationContext { /// Current iteration: 0 initial total transmission, >0 delta transmission. pub iteration: usize, /// Total bytes sent across all iterations. - pub total_sent_bytes: u64, + total_sent_bytes: u64, /// Total bytes to send in the current iteration. pub current_iteration_total_bytes: u64, /// The currently measured bandwidth. /// /// This is updated (at least) after each completed iteration. - pub bandwidth_bytes_per_second: f64, + bandwidth_bytes_per_second: f64, /// Calculated downtime in milliseconds regarding the current bandwidth and /// the remaining memory. /// diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 91357cc294..66230914b8 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -31,7 +31,6 @@ use anyhow::{Context, anyhow}; #[cfg(feature = "dbus_api")] use api::dbus::{DBusApiOptions, DBusApiShutdownChannels}; use api::http::HttpApiHandle; -use arch::PAGE_SIZE; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] use arch::x86_64::MAX_SUPPORTED_CPUS_LEGACY; use console_devices::{ConsoleInfo, pre_create_console_devices}; @@ -48,10 +47,7 @@ use signal_hook::iterator::{Handle, Signals}; use thiserror::Error; use vm_memory::GuestMemoryAtomic; use vm_memory::bitmap::AtomicBitmap; -use vm_migration::progress::{ - MemoryTransmissionInfo, MigrationProgress, MigrationState, MigrationStateOngoingPhase, - TransportationMode, -}; +use vm_migration::progress::MigrationProgress; use vm_migration::protocol::*; use vm_migration::{ MemoryMigrationContext, Migratable, MigratableError, OngoingMigrationContext, Pausable, @@ -302,9 +298,6 @@ impl From for EpollDispatch { } } -// TODO make this a member of Vmm? -static MIGRATION_PROGRESS_SNAPSHOT: Mutex> = Mutex::new(None); - pub struct EpollContext { epoll_file: File, } @@ -1445,36 +1438,6 @@ impl Vmm { is_converged: impl Fn(&MemoryMigrationContext) -> result::Result, mem_send: &mut SendAdditionalConnections, ) -> result::Result { - let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .update( - MigrationStateOngoingPhase::MemoryPrecopy, - Some(MemoryTransmissionInfo { - memory_iteration: s.iteration as u64, - memory_transmission_bps: s.current_iteration_total_bytes, - memory_bytes_total: s.bandwidth_bytes_per_second as u64, - memory_bytes_transmitted: s.total_sent_bytes, - memory_pages_4k_transmitted: s.total_sent_bytes.div_ceil(PAGE_SIZE as u64), - memory_pages_4k_remaining_iteration: s - .current_iteration_total_bytes - .div_ceil(PAGE_SIZE as u64), - memory_bytes_remaining_iteration: s.current_iteration_total_bytes, - memory_dirty_rate_pps: { - let pages = s.current_iteration_total_bytes.div_ceil(PAGE_SIZE as u64); - s.iteration_duration - .filter(|d| !d.is_zero()) - .map(|d| (pages as f64 / d.as_secs_f64()).ceil()) - .map_or(0, |dirty_rate| dirty_rate as u64) - }, - memory_pages_constant_count: 0, /* TODO */ - }), - Some(vm.throttle_percent()), - s.estimated_downtime, - ); - }; - loop { // todo: check if auto-converge is enabled at all? if Self::can_increase_autoconverge_step(ctx) @@ -1499,16 +1462,11 @@ impl Vmm { }; ctx.update_metrics_before_transfer(iteration_begin, &iteration_table); - // Update before we might exit the loop. - update_migration_progress(ctx, vm); if is_converged(ctx)? { info!("Precopy converged: {ctx}"); break Ok(iteration_table); } - // Update with new metrics before transmission. - update_migration_progress(ctx, vm); - // Send the current dirty pages let transfer_begin = Instant::now(); mem_send.send_memory(iteration_table, socket)?; @@ -1757,11 +1715,6 @@ impl Vmm { if send_data_migration.local { match &mut socket { SocketStream::Unix(unix_socket) => { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .update(MigrationStateOngoingPhase::MemoryFds, None, None, None); - // Proceed with sending memory file descriptors over UNIX socket vm.send_memory_fds(unix_socket)?; } @@ -1822,14 +1775,6 @@ impl Vmm { mem_send.cleanup()?; } - // Update migration progress snapshot - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .update(MigrationStateOngoingPhase::Completing, None, None, None); - } - // We release the locks early to enable locking them on the destination host. // The VM is already stopped. vm.release_disk_locks() @@ -1877,14 +1822,6 @@ impl Vmm { vm.stop_dirty_log()?; } - // Update migration progress snapshot - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_finished(); - } - // Let every Migratable object know about the migration being complete vm.complete_migration() } @@ -2079,14 +2016,6 @@ impl Vmm { // Give VMM back control. self.vm = MaybeVmOwnership::Vmm(vm); - // Update migration progress snapshot - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_failed(&e); - } - { info!("Sending Receiver in HTTP thread that migration failed"); let (sender, _) = &*ONGOING_LIVEMIGRATION; @@ -3149,61 +3078,28 @@ impl RequestHandler for Vmm { ))); } - // Update migration progress snapshot early: - // We guarantee that migration statistics can be fetched as soon as SendMigration returns. - // - // If the migration fails, the state will later be updated accordingly. - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - if lock - .as_ref() - .map(|p| &p.state) - .is_some_and(|snapshot| matches!(snapshot, MigrationState::Ongoing { .. })) - { - // If this panic triggers, we made a programming error in our state handling. - panic!("migration already ongoing"); - } - let transportation_mode = if send_data_migration.local { - TransportationMode::Local - } else { - TransportationMode::Tcp { - connections: send_data_migration.connections, - tls: send_data_migration.tls_dir.is_some(), - } - }; - lock.replace(MigrationProgress::new( - transportation_mode, - send_data_migration.downtime(), - )); - } - // Start migration thread - { - let worker = MigrationWorker { - vm, - check_migration_evt: self.check_migration_evt.try_clone().unwrap(), - config: send_data_migration, - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - hypervisor: self.hypervisor.clone(), - }; + let worker = MigrationWorker { + vm, + check_migration_evt: self.check_migration_evt.try_clone().unwrap(), + config: send_data_migration, + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + hypervisor: self.hypervisor.clone(), + }; - self.migration_thread_handle = Some( - thread::Builder::new() - .name("migration".into()) - .spawn(move || worker.run()) - // For upstreaming, we should simply continue and return an - // error when this fails. For our PoC, this is fine. - .unwrap(), - ); - } + self.migration_thread_handle = Some( + thread::Builder::new() + .name("migration".into()) + .spawn(move || worker.run()) + // For upstreaming, we should simply continue and return an + // error when this fails. For our PoC, this is fine. + .unwrap(), + ); Ok(()) } fn vm_migration_progress(&mut self) -> Option { - // We explicitly do not check here for `is VM running?` to always - // enable querying the state of the last failed migration. - let lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.clone() + None } } From 6e93ca89756ccf62b7f16af23b98cd278eaba99a Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:16:01 +0200 Subject: [PATCH 32/79] vmm: Revert "add migration-progress HTTP endpoint" This reverts commit 2dd9618cfaa88f12cfd7cf6a6f52e070c18f4ea9. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 32 +++---------------------------- vmm/src/api/http/mod.rs | 12 ++++-------- 2 files changed, 7 insertions(+), 37 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index 456da87395..38d5f830dc 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -59,9 +59,9 @@ use crate::api::http::{EndpointHandler, HttpError, error_response}; use crate::api::{ AddDisk, ApiAction, ApiError, ApiRequest, NetConfig, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, - VmConfig, VmCounters, VmDelete, VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, - VmPowerButton, VmReboot, VmReceiveMigration, VmReceiveMigrationData, VmRemoveDevice, VmResize, - VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, + VmConfig, VmCounters, VmDelete, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, + VmReboot, VmReceiveMigration, VmReceiveMigrationData, VmRemoveDevice, VmResize, VmResizeDisk, + VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, }; use crate::config::RestoreConfig; use crate::cpu::Error as CpuError; @@ -711,32 +711,6 @@ impl EndpointHandler for VmmShutdown { } } -impl EndpointHandler for VmMigrationProgress { - fn handle_request( - &self, - req: &Request, - api_notifier: EventFd, - api_sender: Sender, - ) -> Response { - match req.method() { - Method::Get => match crate::api::VmMigrationProgress - .send(api_notifier, api_sender, ()) - .map_err(HttpError::ApiError) - { - Ok(info) => { - let mut response = Response::new(Version::Http11, StatusCode::OK); - let info_serialized = serde_json::to_string(&info).unwrap(); - - response.set_body(Body::new(info_serialized)); - response - } - Err(e) => error_response(e, StatusCode::InternalServerError), - }, - _ => error_response(HttpError::BadRequest, StatusCode::BadRequest), - } - } -} - #[cfg(test)] mod external_fds_tests { use super::*; diff --git a/vmm/src/api/http/mod.rs b/vmm/src/api/http/mod.rs index 5ac3b35672..7351406f1a 100644 --- a/vmm/src/api/http/mod.rs +++ b/vmm/src/api/http/mod.rs @@ -29,10 +29,10 @@ use self::http_endpoint::{VmActionHandler, VmCreate, VmInfo, VmmPing, VmmShutdow use crate::api::VmCoredump; use crate::api::{ AddDisk, ApiError, ApiRequest, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, - VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, - VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, VmReboot, - VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, - VmSendMigration, VmShutdown, VmSnapshot, + VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, VmNmi, + VmPause, VmPostMigrationAnnounce, VmPowerButton, VmReboot, VmReceiveMigration, VmRemoveDevice, + VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, + VmSnapshot, }; use crate::landlock::Landlock; use crate::seccomp_filters::{Thread, get_seccomp_filter}; @@ -286,10 +286,6 @@ pub static HTTP_ROUTES: LazyLock = LazyLock::new(|| { endpoint!("/vm.shutdown"), Box::new(VmActionHandler::new(&VmShutdown)), ); - r.routes.insert( - endpoint!("/vm.migration-progress"), - Box::new(VmMigrationProgress {}), - ); r.routes.insert( endpoint!("/vm.snapshot"), Box::new(VmActionHandler::new(&VmSnapshot)), From a38ab26302278ba9ac97e895b0edb5c72c8766e8 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:16:18 +0200 Subject: [PATCH 33/79] vmm: Revert "add migration-progress API endpoint" This reverts commit e27612d9a304d72009cb27d0f9e2f8a3356de61e. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- fuzz/Cargo.lock | 1 - fuzz/fuzz_targets/http_api.rs | 5 ---- vmm/src/api/mod.rs | 51 ----------------------------------- vmm/src/lib.rs | 5 ---- 4 files changed, 62 deletions(-) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 58ef2c4d99..52f5d4248f 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1601,7 +1601,6 @@ version = "0.1.0" dependencies = [ "arch", "libc", - "thiserror", "vm-memory", ] diff --git a/fuzz/fuzz_targets/http_api.rs b/fuzz/fuzz_targets/http_api.rs index 0273d5b455..d6dfbf2bf9 100644 --- a/fuzz/fuzz_targets/http_api.rs +++ b/fuzz/fuzz_targets/http_api.rs @@ -11,7 +11,6 @@ use std::thread; use libfuzzer_sys::{fuzz_target, Corpus}; use micro_http::Request; -use vm_migration::progress::MigrationProgress; use vm_migration::MigratableError; use vmm::api::http::*; use vmm::api::{ @@ -306,10 +305,6 @@ impl RequestHandler for StubApiRequestHandler { Ok(()) } - fn vm_migration_progress(&mut self) -> Option { - None - } - fn vm_post_migration_announce(&mut self) -> Result<(), VmError> { Ok(()) } diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 557a0205b4..f62f80aeb0 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -46,7 +46,6 @@ use option_parser::{OptionParser, OptionParserError, Toggle}; use serde::{Deserialize, Serialize}; use thiserror::Error; use vm_migration::MigratableError; -use vm_migration::progress::MigrationProgress; use vmm_sys_util::eventfd::EventFd; #[cfg(feature = "dbus_api")] @@ -216,10 +215,6 @@ pub enum ApiError { /// Error triggering NMI #[error("Error triggering NMI")] VmNmi(#[source] VmError), - - /// Error fetching the migration progress - #[error("Error fetching the migration progress")] - VmMigrationProgress(#[source] VmError), } pub type ApiResult = Result; @@ -686,9 +681,6 @@ pub enum ApiResponsePayload { /// Virtual machine information VmInfo(VmInfoResponse), - /// The progress of a possibly ongoing live migration. - VmMigrationProgress(Box>), - /// Vmm ping response VmmPing(VmmPingResponse), @@ -781,10 +773,6 @@ pub trait RequestHandler { ) -> Result<(), MigratableError>; fn vm_nmi(&mut self) -> Result<(), VmError>; - - /// Returns the progress of the currently active migration or any previous - /// failed or canceled migration. - fn vm_migration_progress(&mut self) -> Option; } /// It would be nice if we could pass around an object like this: @@ -1988,45 +1976,6 @@ impl ApiAction for VmNmi { } } -pub struct VmMigrationProgress; - -impl ApiAction for VmMigrationProgress { - type RequestBody = (); - type ResponseBody = Box>; - - fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { - Box::new(move |vmm| { - info!("API request event: VmMigrationProgress"); - - let snapshot = Ok(vmm.vm_migration_progress()); - let response = snapshot - .map(Box::new) - .map(ApiResponsePayload::VmMigrationProgress) - .map_err(ApiError::VmMigrationProgress); - - response_sender - .send(response) - .map_err(VmmError::ApiResponseSend)?; - - Ok(false) - }) - } - - fn send( - &self, - api_evt: EventFd, - api_sender: Sender, - data: Self::RequestBody, - ) -> ApiResult { - let info = get_response(self, api_evt, api_sender, data)?; - - match info { - ApiResponsePayload::VmMigrationProgress(info) => Ok(info), - _ => Err(ApiError::ResponsePayloadType), - } - } -} - #[cfg(test)] mod unit_tests { use super::*; diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 66230914b8..cbff0a885b 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -47,7 +47,6 @@ use signal_hook::iterator::{Handle, Signals}; use thiserror::Error; use vm_memory::GuestMemoryAtomic; use vm_memory::bitmap::AtomicBitmap; -use vm_migration::progress::MigrationProgress; use vm_migration::protocol::*; use vm_migration::{ MemoryMigrationContext, Migratable, MigratableError, OngoingMigrationContext, Pausable, @@ -3097,10 +3096,6 @@ impl RequestHandler for Vmm { ); Ok(()) } - - fn vm_migration_progress(&mut self) -> Option { - None - } } const CPU_MANAGER_SNAPSHOT_ID: &str = "cpu-manager"; From c67d4ef1662128b8ddebddce6f461569447c6279 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:16:32 +0200 Subject: [PATCH 34/79] vm-migration: Revert "prepare progress types for new API endpoint" This reverts commit 04dd58fe1c67be484791b79011a8ce2497ebd20f. This commit builds on the fork's own migration worker, which this series replaces with the upstream implementation. It is re-applied on top of the upstream worker later in this series. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/lib.rs | 1 - vm-migration/src/progress.rs | 564 ----------------------------------- 2 files changed, 565 deletions(-) delete mode 100644 vm-migration/src/progress.rs diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 3b5f25987c..2255bceeb7 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -16,7 +16,6 @@ use crate::protocol::MemoryRangeTable; mod bitpos_iterator; mod context; pub mod keep_alive_stream; -pub mod progress; pub mod protocol; pub mod tls; diff --git a/vm-migration/src/progress.rs b/vm-migration/src/progress.rs deleted file mode 100644 index 8a5083068d..0000000000 --- a/vm-migration/src/progress.rs +++ /dev/null @@ -1,564 +0,0 @@ -// Copyright © 2025 Cyberus Technology GmbH -// -// SPDX-License-Identifier: Apache-2.0 - -//! Module for reporting status and progress of live migrations. -//! -//! The main export is [`MigrationProgress`]. -//! -//! # Motivation -//! -//! Monitoring a live-migration is important for debugging of cloud deployments, -//! for cloud monitoring in general, and for network optimization, such as -//! verifying the throughput for the migration is as high as expected. -//! -//! It also helps to analyze the downtime of VMs and see how much pressure a -//! guest is putting on its memory (by writing), which is slowing down -//! migrations. - -use std::error::Error; -use std::fmt; -use std::fmt::Display; -use std::num::NonZeroU32; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -#[derive( - Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum TransportationMode { - Local, - Tcp { connections: NonZeroU32, tls: bool }, -} - -/// Carries information about the transmission of the VM's memory. -#[derive( - Clone, - Copy, - Debug, - Default, - PartialOrd, - Ord, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -pub struct MemoryTransmissionInfo { - /// The memory iteration (only in precopy mode). - pub memory_iteration: u64, - /// Memory bytes per second. - pub memory_transmission_bps: u64, - /// The total size of the VMs memory in bytes. - pub memory_bytes_total: u64, - /// The total size of transmitted bytes. - pub memory_bytes_transmitted: u64, - /// The amount of remaining bytes for this iteration. - pub memory_bytes_remaining_iteration: u64, - /// The amount of transmitted 4k pages. - pub memory_pages_4k_transmitted: u64, - /// The amount of remaining 4k pages for this iteration. - pub memory_pages_4k_remaining_iteration: u64, - /// The amount of constant pages for that we could take a shortcut. - /// Pages where all bits are either zero or one. - pub memory_pages_constant_count: u64, - /// Current memory dirty rate in pages per seconds (pps). - pub memory_dirty_rate_pps: u64, -} - -/// The different phases of an ongoing ([`MigrationState::Ongoing`]) migration -/// (good case). -/// -/// The states correspond to the [live-migration protocol]. -/// -/// [live-migration protocol]: super::protocol -#[derive( - Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum MigrationStateOngoingPhase { - /// The migration starts. Handshake and transfer of VM config. - Starting, - /// Transfer of memory FDs. - /// - /// Only used for local migrations. - MemoryFds, - /// Transfer of VM memory in precopy mode. - /// - /// Not used for local migrations. - MemoryPrecopy, - // TODO eventually add MemoryPostcopy here - /// The VM migration is completing. This means the last chunks of memory - /// are transmitted as well as the final VM state (vCPUs, devices). - Completing, -} - -impl Display for MigrationStateOngoingPhase { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Starting => write!(f, "starting"), - Self::MemoryFds => write!(f, "memory FDs"), - Self::MemoryPrecopy => write!(f, "memory (precopy)"), - Self::Completing => write!(f, "completing"), - } - } -} - -/// The different states of a migration, covering steady progress and failure. -#[derive( - Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum MigrationState { - /// The migration has been cancelled. - Cancelled {}, - /// The migration has failed. - Failed { - /// Stringified error. - error_msg: String, - /// Debug-stringified error. - error_msg_debug: String, - // TODO this is very tricky because I need clone() - // error: Box, - }, - /// The migration has finished successfully. - Finished {}, - /// The migration is ongoing. - Ongoing { - phase: MigrationStateOngoingPhase, - /// Percent in range `0..=100`. - vcpu_throttle_percent: u8, - }, -} - -impl Display for MigrationState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - MigrationState::Cancelled { .. } => write!(f, "{}", self.state_name()), - MigrationState::Failed { error_msg, .. } => { - write!(f, "{}: {error_msg}", self.state_name()) - } - MigrationState::Finished { .. } => write!(f, "{}", self.state_name()), - MigrationState::Ongoing { - phase, - vcpu_throttle_percent, - } => write!( - f, - "{}: phase={phase}, vcpu_throttle={vcpu_throttle_percent}", - self.state_name() - ), - } - } -} - -impl MigrationState { - fn state_name(&self) -> &'static str { - match self { - MigrationState::Cancelled { .. } => "cancelled", - MigrationState::Failed { .. } => "failed", - MigrationState::Finished { .. } => "finished", - MigrationState::Ongoing { .. } => "ongoing", - } - } -} - -/// Returns the current UNIX timestamp in ms. -fn current_unix_timestamp_ms() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("should be valid duration") - .as_millis() as u64 -} - -/// Holds a snapshot of progress and status information for an ongoing live -/// migration, or the last snapshot of a canceled or aborted migration. -/// -/// This type carries insightful information for every step of the -/// [live-migration protocol] in a way that makes it easy for API users to -/// parse the data with ease while retaining all important information. -/// -/// [live-migration protocol]: super::protocol -#[derive( - Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct MigrationProgress { - /// UNIX timestamp of the start of the live-migration process in ms. - pub timestamp_begin_ms: u64, - /// UNIX timestamp of the current snapshot in ms. - pub timestamp_snapshot_ms: u64, - /// Relative timestamp since the beginning of the migration in ms. - pub timestamp_snapshot_relative_ms: u64, - /// Configured target downtime. - pub downtime_configured_ms: u64, - /// Currently estimated (computed) downtime given the remaining - /// transmissions and the bandwidth. - /// - /// If this is `0`, the downtime could not yet be calculated. - pub downtime_estimated_ms: u64, - /// Requested transportation mode. - pub transportation_mode: TransportationMode, - /// Snapshot of the current phase. - pub state: MigrationState, - /// Latest [`MemoryTransmissionInfo`] info, if any. - /// - /// The most interesting phase is when current state is - /// [`MigrationState::Ongoing`] and [`MigrationStateOngoingPhase::MemoryPrecopy`] - /// as this value will be updated frequently. - pub memory_transmission_info: MemoryTransmissionInfo, -} - -impl MigrationProgress { - /// Creates new progress in a valid init state. - /// - /// This progress must be updated using any of: - /// - [`Self::update`] - /// - [`Self::mark_as_finished`] - /// - [`Self::mark_as_failed`] - /// - [`Self::mark_as_cancelled`] - pub fn new(transportation_mode: TransportationMode, target_downtime: Duration) -> Self { - let timestamp = current_unix_timestamp_ms(); - Self { - timestamp_begin_ms: timestamp, - timestamp_snapshot_ms: timestamp, - timestamp_snapshot_relative_ms: 0, - downtime_configured_ms: target_downtime.as_millis() as u64, - downtime_estimated_ms: 0, - transportation_mode, - state: MigrationState::Ongoing { - phase: MigrationStateOngoingPhase::Starting, - vcpu_throttle_percent: 0, - }, - memory_transmission_info: MemoryTransmissionInfo::default(), - } - } - - /// Updates the state of an ongoing migration. - /// - /// Only updates new values that are provided via `Some`. - /// - /// # Arguments - /// - /// - `new_phase`: The current [`MigrationStateOngoingPhase`]. - /// - `new_memory_transmission_info`: If `Some`, the current [`MemoryTransmissionInfo`]. - /// - `new_cpu_throttle_percent`: If `Some`, the current value of the vCPU throttle percentage. - /// Must be in range `0..=100`. - /// - `new_estimated_downtime`: If `Some`, the latest expected (calculated) downtime. - pub fn update( - &mut self, - new_phase: MigrationStateOngoingPhase, - new_memory_transmission_info: Option, - new_cpu_throttle_percent: Option, - new_estimated_downtime: Option, - ) { - if let Some(percent) = new_cpu_throttle_percent { - assert!(percent <= 100); - } - - if let Some(downtime) = new_estimated_downtime { - self.downtime_estimated_ms = u64::try_from(downtime.as_millis()).unwrap(); - } else { - // This is better than showing `0` and it is likely close to the final actual downtime. - self.downtime_estimated_ms = self.downtime_configured_ms; - } - - match &self.state { - MigrationState::Ongoing { - phase: _old_phase, - vcpu_throttle_percent: old_vcpu_throttle_percent, - } => { - self.timestamp_snapshot_ms = current_unix_timestamp_ms(); - self.timestamp_snapshot_relative_ms = - self.timestamp_snapshot_ms - self.timestamp_begin_ms; - - self.memory_transmission_info = - new_memory_transmission_info.unwrap_or(self.memory_transmission_info); - self.state = MigrationState::Ongoing { - phase: new_phase, - vcpu_throttle_percent: new_cpu_throttle_percent - .unwrap_or(*old_vcpu_throttle_percent), - }; - } - illegal => { - // panic is fine as we have a logic error here, nothing that was caused by a user. - panic!( - "illegal state transition: {} -> ongoing", - illegal.state_name(), - ); - } - } - } - - /// Sets the underlying state to [`MigrationState::Cancelled`] and - /// updates all corresponding metadata. - /// - /// After this state change, the object is supposed to be handled as immutable. - /// - /// # Panics - /// - /// If the current state is not [`MigrationState::Ongoing`], this function panics. - pub fn mark_as_cancelled(&mut self) { - if !matches!(self.state, MigrationState::Ongoing { .. }) { - panic!( - "illegal state transition: {} -> cancelled", - self.state.state_name() - ); - } - self.timestamp_snapshot_ms = current_unix_timestamp_ms(); - self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; - self.state = MigrationState::Cancelled {}; - } - - /// Sets the underlying state to [`MigrationState::Failed`] and - /// updates all corresponding metadata. - /// - /// After this state change, the object is supposed to be handled as immutable. - /// - /// # Panics - /// - /// If the current state is not [`MigrationState::Ongoing`], this function panics. - pub fn mark_as_failed(&mut self, error: &dyn Error) { - if !matches!(self.state, MigrationState::Ongoing { .. }) { - panic!( - "illegal state transition: {} -> failed", - self.state.state_name() - ); - } - self.timestamp_snapshot_ms = current_unix_timestamp_ms(); - self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; - self.state = MigrationState::Failed { - error_msg: format!("{error}",), - error_msg_debug: format!("{error:?}",), - }; - } - - /// Sets the underlying state to [`MigrationState::Finished`] and - /// updates all corresponding metadata. - /// - /// After this state change, the object is supposed to be handled as immutable. - /// - /// # Panics - /// - /// If the current state is not [`MigrationState::Ongoing`], this function panics. - pub fn mark_as_finished(&mut self) { - if !matches!(self.state, MigrationState::Ongoing { .. }) { - panic!( - "illegal state transition: {} -> finished", - self.state.state_name() - ); - } - self.timestamp_snapshot_ms = current_unix_timestamp_ms(); - self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; - self.state = MigrationState::Finished {}; - } -} - -#[cfg(test)] -mod unit_tests { - use std::thread; - - use super::*; - - fn tcp_mode() -> TransportationMode { - TransportationMode::Tcp { - connections: NonZeroU32::new(2).unwrap(), - tls: true, - } - } - - #[test] - fn new_initializes_valid_state() { - let target = Duration::from_millis(150); - let progress = MigrationProgress::new(tcp_mode(), target); - - assert_eq!(progress.timestamp_snapshot_ms, progress.timestamp_begin_ms); - assert_eq!(progress.timestamp_snapshot_relative_ms, 0); - assert_eq!(progress.downtime_configured_ms, 150); - assert_eq!(progress.downtime_estimated_ms, 0); - - match progress.state { - MigrationState::Ongoing { - phase, - vcpu_throttle_percent, - } => { - assert_eq!(phase, MigrationStateOngoingPhase::Starting); - assert_eq!(vcpu_throttle_percent, 0); - } - _ => panic!("expected Ongoing state"), - } - - assert_eq!( - progress.memory_transmission_info, - MemoryTransmissionInfo::default() - ); - } - - #[test] - fn update_changes_phase_and_preserves_previous_values() { - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(200)); - - let initial_timestamp = progress.timestamp_snapshot_ms; - - thread::sleep(Duration::from_millis(1)); - - progress.update(MigrationStateOngoingPhase::MemoryPrecopy, None, None, None); - - match progress.state { - MigrationState::Ongoing { - phase, - vcpu_throttle_percent, - } => { - assert_eq!(phase, MigrationStateOngoingPhase::MemoryPrecopy); - assert_eq!(vcpu_throttle_percent, 0); // unchanged - } - _ => panic!("expected Ongoing"), - } - - assert!(progress.timestamp_snapshot_ms >= initial_timestamp); - assert!(progress.timestamp_snapshot_relative_ms > 0); - - // If no estimated downtime provided, fallback to configured value - assert_eq!( - progress.downtime_estimated_ms, - progress.downtime_configured_ms - ); - } - - #[test] - fn update_replaces_memory_info_and_throttle() { - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(100)); - - let mem = MemoryTransmissionInfo { - memory_iteration: 3, - memory_transmission_bps: 10_000, - memory_bytes_total: 1_000_000, - memory_bytes_transmitted: 400_000, - memory_bytes_remaining_iteration: 100_000, - memory_pages_4k_transmitted: 100, - memory_pages_4k_remaining_iteration: 25, - memory_pages_constant_count: 10, - memory_dirty_rate_pps: 500, - }; - - progress.update( - MigrationStateOngoingPhase::MemoryPrecopy, - Some(mem), - Some(42), - Some(Duration::from_millis(55)), - ); - - assert_eq!(progress.memory_transmission_info, mem); - assert_eq!(progress.downtime_estimated_ms, 55); - - match progress.state { - MigrationState::Ongoing { - phase, - vcpu_throttle_percent, - } => { - assert_eq!(phase, MigrationStateOngoingPhase::MemoryPrecopy); - assert_eq!(vcpu_throttle_percent, 42); - } - _ => panic!("expected Ongoing"), - } - } - - #[test] - #[should_panic] - fn update_panics_if_not_ongoing() { - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); - progress.mark_as_finished(); - - progress.update(MigrationStateOngoingPhase::Completing, None, None, None); - } - - #[test] - #[should_panic] - fn throttle_above_100_panics() { - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); - - progress.update( - MigrationStateOngoingPhase::MemoryPrecopy, - None, - Some(101), - None, - ); - } - - #[test] - fn mark_as_finished_transitions_state() { - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); - - thread::sleep(Duration::from_millis(1)); - progress.mark_as_finished(); - - match progress.state { - MigrationState::Finished {} => {} - _ => panic!("expected Finished"), - } - - assert!(progress.timestamp_snapshot_relative_ms > 0); - } - - #[test] - #[should_panic] - fn mark_as_finished_twice_panics() { - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); - - progress.mark_as_finished(); - progress.mark_as_finished(); - } - - #[test] - fn mark_as_failed_sets_error_strings() { - #[derive(Debug)] - struct TestError; - - impl fmt::Display for TestError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "test error") - } - } - - impl Error for TestError {} - - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); - - progress.mark_as_failed(&TestError); - - match &progress.state { - MigrationState::Failed { - error_msg, - error_msg_debug, - } => { - assert_eq!(error_msg, "test error"); - assert!(error_msg_debug.contains("TestError")); - } - _ => panic!("expected Failed"), - } - } - - #[test] - fn display_formats_are_stable() { - let mut progress = - MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); - - progress.update( - MigrationStateOngoingPhase::MemoryPrecopy, - None, - Some(12), - None, - ); - - let s = format!("{}", progress.state); - assert!(s.contains("ongoing")); - assert!(s.contains("phase=memory (precopy)")); - assert!(s.contains("vcpu_throttle=12")); - - progress.mark_as_cancelled(); - assert_eq!(format!("{}", progress.state), "cancelled"); - } -} From 13cf42ed85852b24fa3754a8d94dc61cf6cace13 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:16:39 +0200 Subject: [PATCH 35/79] vmm: Revert "migration: limit to running VMs only" This reverts commit 27ba41b55651244078b9ff2fd136141d6738bdeb. Upstream's migration worker carries the initial VM state and resumes a failed migration only if the VM was running before, so this fork-only guard is not re-applied. Migrating a paused VM is possible again with that, so the precopy path must not pause the VM a second time: the transition Paused -> Paused is rejected. The local path already skips it; add the same guard to do_memory_migration(), as upstream does. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index cbff0a885b..999495fa08 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1612,9 +1612,12 @@ impl Vmm { info!("stopping vcpu thread"); vm.stop_vcpu_throttling(); info!("stopped vcpu thread"); - info!("pausing VM"); - vm.pause()?; - info!("paused VM"); + // Skip if already paused, e.g. when migrating a paused VM. + if vm.get_state() != VmState::Paused { + info!("pausing VM"); + vm.pause()?; + info!("paused VM"); + } // Send last batch of dirty pages: final iteration { @@ -3058,14 +3061,6 @@ impl RequestHandler for Vmm { ))); } - // Cloud Hypervisor only supports the migration of running VMs. - let current_state = self.vm.vm_mut().as_ref().unwrap().get_state(); - if current_state != VmState::Running { - return Err(MigratableError::MigrateSend(anyhow!(format!( - "Only running VMs can be migrated! state={current_state:?}" - )))); - } - // Take VM ownership. This also means that API events can no longer // change the VM (e.g. net device hotplug). let vm = self.vm.take_vm_for_migration(); From 0dd098a8159f59007926a065b74b51ab19e842bd Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:16:39 +0200 Subject: [PATCH 36/79] vmm: Revert "migration: fix missing resume() after failure" This reverts commit 3b3eda17bf0eb3de1103baa64ebaf8ed4043ad78. The original subject was "vmm: migration: fix missing resume() VM after failed live migration". The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 999495fa08..7ede5377c4 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1963,10 +1963,7 @@ impl Vmm { fn check_migration_result(&mut self) { // At this point, the thread must be finished. // If we fail here, we have lost anyway. Just panic. - let MigrationThreadOut { - mut vm, - migration_res, - } = self + let MigrationThreadOut { vm, migration_res } = self .migration_thread_handle .take() .expect("should have thread") @@ -1993,28 +1990,6 @@ impl Vmm { Err(e) => { error!("Migration failed: {e}"); - // If the failure happened very late in the migration path, the VM might already be - // stopped. We resume it to ensure proper operation. - // - // Cloud Hypervisor only supports migration of running VMs, therefore it cannot - // happen that we resume a previously paused VM. - if vm.get_state() == VmState::Paused { - match vm.resume() { - Ok(_) => { - info!("Resumed VM successfully after failed migration"); - - // Ensure full VM performance. The operation is idempotent. - let _ = vm.stop_dirty_log().inspect_err(|e| { - warn!("Failed stopping dirty log after resuming VM: {e} - VM performance might be slower than usual"); - }); - } - Err(e) => { - error!("Failed resuming VM after failed migration: {e}"); - self.exit_evt.write(1).unwrap(); - } - } - } - // Give VMM back control. self.vm = MaybeVmOwnership::Vmm(vm); From 244176be2b4430c02897da97cc11c16252dd8589 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:17:07 +0200 Subject: [PATCH 37/79] vmm: Revert "api: temporarily make VmSendMigration call blocking again" This reverts commit f7303f34a9e6edec9e1f642cb43cb3a03e328fbc. SYS_read stays: it was added later for the event-monitor and HTTP threads. The SYS_rt_sigprocmask and SYS_getcwd entries added here were duplicates of entries already present in the same list. The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 55 ++----------------------------- vmm/src/lib.rs | 26 +-------------- vmm/src/seccomp_filters.rs | 3 -- 3 files changed, 3 insertions(+), 81 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index 38d5f830dc..ead1a9de5e 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -35,23 +35,11 @@ //! [special HTTP library]: https://github.com/firecracker-microvm/micro-http use std::fs::File; -use std::sync::mpsc::{Receiver, Sender, SyncSender}; -use std::sync::{LazyLock, Mutex}; +use std::sync::mpsc::Sender; -use log::info; use micro_http::{Body, Method, Request, Response, StatusCode, Version}; use vmm_sys_util::eventfd::EventFd; -/// Helper to make the VmSendMigration call blocking as long as a migration is ongoing. -#[allow(clippy::type_complexity)] -pub static ONGOING_LIVEMIGRATION: LazyLock<( - SyncSender>, - Mutex>>, -)> = LazyLock::new(|| { - let (sender, receiver) = std::sync::mpsc::sync_channel(0); - (sender, Mutex::new(receiver)) -}); - #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] use crate::api::VmCoredump; use crate::api::http::http_endpoint::fds_helper::{attach_fds_to_cfg, attach_fds_to_cfgs}; @@ -442,6 +430,7 @@ vm_action_put_handler_body!(VmRemoveDevice); vm_action_put_handler_body!(VmResizeDisk); vm_action_put_handler_body!(VmResizeZone); vm_action_put_handler_body!(VmSnapshot); +vm_action_put_handler_body!(VmSendMigration); #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] vm_action_put_handler_body!(VmCoredump); @@ -498,46 +487,6 @@ impl PutHandler for VmReceiveMigration { impl GetHandler for VmReceiveMigration {} -// Special Handling for virtio-net Devices Backed by Network File Descriptors -// -// See above. -impl PutHandler for VmSendMigration { - fn handle_request( - &'static self, - api_notifier: EventFd, - api_sender: Sender, - body: &Option, - _files: Vec, - ) -> std::result::Result, HttpError> { - if let Some(body) = body { - let res = self - .send( - api_notifier, - api_sender, - serde_json::from_slice(body.raw())?, - ) - .map_err(HttpError::ApiError)?; - - info!("live migration started"); - - let (_, receiver) = &*ONGOING_LIVEMIGRATION; - - info!("waiting for live migration result"); - let mig_res = receiver.lock().unwrap().recv().unwrap(); - info!("received live migration result"); - - // We forward the migration error here to the guest - mig_res - .map(|_| res) - .map_err(|e| HttpError::ApiError(ApiError::VmSendMigration(e))) - } else { - Err(HttpError::BadRequest) - } - } -} - -impl GetHandler for VmSendMigration {} - impl PutHandler for VmResize { fn handle_request( &'static self, diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 7ede5377c4..ae00ff6cd7 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -56,7 +56,6 @@ use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::unblock_signal; use vmm_sys_util::sock_ctrl_msg::ScmSocket; -use crate::api::http::http_endpoint::ONGOING_LIVEMIGRATION; use crate::api::{ ApiRequest, ApiResponse, RequestHandler, TimeoutStrategy, VmInfoResponse, VmReceiveMigrationData, VmSendMigrationData, VmmPingResponse, @@ -1975,13 +1974,6 @@ impl Vmm { self.vm = MaybeVmOwnership::None; drop(vm); - { - info!("Sending Receiver in HTTP thread that migration succeeded"); - let (sender, _) = &*ONGOING_LIVEMIGRATION; - // unblock API call; propagate migration result - sender.send(Ok(())).unwrap(); - } - // Shutdown the VM after the migration succeeded if let Err(e) = self.exit_evt.write(1) { error!("Failed shutting down the VM after migration: {e}"); @@ -1992,14 +1984,6 @@ impl Vmm { // Give VMM back control. self.vm = MaybeVmOwnership::Vmm(vm); - - { - info!("Sending Receiver in HTTP thread that migration failed"); - let (sender, _) = &*ONGOING_LIVEMIGRATION; - // unblock API call; propagate migration result - sender.send(Err(e)).unwrap(); - } - // we don't fail the VMM here, it just continues running its VM } } } @@ -2558,18 +2542,10 @@ impl RequestHandler for Vmm { } fn vm_resize_disk(&mut self, id: String, desired_size: u64) -> result::Result<(), VmError> { - info!("request to resize disk: id={id}"); self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - if let Err(e) = vm.resize_disk(&id, desired_size) { - error!("Error when resizing disk: {e:?}"); - Err(e) - } else { - Ok(()) - } - } + MaybeVmOwnership::Vmm(ref mut vm) => vm.resize_disk(&id, desired_size), MaybeVmOwnership::Migration => Err(VmError::VmMigrating), MaybeVmOwnership::None => Err(VmError::ResizeDisk), } diff --git a/vmm/src/seccomp_filters.rs b/vmm/src/seccomp_filters.rs index 60c820358d..48781c402a 100644 --- a/vmm/src/seccomp_filters.rs +++ b/vmm/src/seccomp_filters.rs @@ -983,9 +983,6 @@ fn http_api_thread_rules() -> Result)>, BackendError> (libc::SYS_sendto, vec![]), (libc::SYS_sigaltstack, vec![]), (libc::SYS_write, vec![]), - (libc::SYS_rt_sigprocmask, vec![]), - (libc::SYS_getcwd, vec![]), - (libc::SYS_clock_nanosleep, vec![]), (libc::SYS_read, vec![]), ]) } From 3b96496c4b19e3b9ca4eefbb4437256f5a438baf Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:18:02 +0200 Subject: [PATCH 38/79] vmm: Revert "better VM ownership handling in context of live migration" This reverts commit e3977923b2e47fb6c3d86477729bc41b36b5ab7e. This also adapts vm_post_migration_announce(), which was added later by 0f8e07333 and matched on the ownership enum, back to Option. Without that adaptation the tree would not compile. The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 794 ++++++++++++++++++++++--------------------------- vmm/src/vm.rs | 3 - 2 files changed, 355 insertions(+), 442 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index ae00ff6cd7..ec26d90151 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -25,7 +25,7 @@ use std::thread::JoinHandle; use std::time::Duration; #[cfg(not(target_arch = "riscv64"))] use std::time::Instant; -use std::{io, mem, result, thread}; +use std::{io, result, thread}; use anyhow::{Context, anyhow}; #[cfg(feature = "dbus_api")] @@ -45,6 +45,7 @@ use serde::ser::{SerializeStruct, Serializer}; use serde::{Deserialize, Serialize}; use signal_hook::iterator::{Handle, Signals}; use thiserror::Error; +use tracer::trace_scoped; use vm_memory::GuestMemoryAtomic; use vm_memory::bitmap::AtomicBitmap; use vm_migration::protocol::*; @@ -698,41 +699,6 @@ pub struct VmmThreadHandle { pub http_api_handle: Option, } -/// Describes the current ownership of a running VM. -#[allow(clippy::large_enum_variant)] -pub enum MaybeVmOwnership { - /// The VMM holds the ownership of the VM. - Vmm(Vm), - /// The VM is temporarily blocked by the current ongoing migration. - Migration, - /// No VM is running. - None, -} - -impl MaybeVmOwnership { - /// Takes the VM and replaces it with [`Self::Migration`]. - /// - /// # Panics - /// This method panics if `self` is not [`Self::Vmm`]. - fn take_vm_for_migration(&mut self) -> Vm { - if !matches!(self, Self::Vmm(_)) { - panic!("should only be called when a migration can start"); - } - - match mem::replace(self, Self::Migration) { - MaybeVmOwnership::Vmm(vm) => vm, - _ => unreachable!(), - } - } - - fn vm_mut(&mut self) -> Option<&mut Vm> { - match self { - MaybeVmOwnership::Vmm(vm) => Some(vm), - _ => None, - } - } -} - /// Output value of [`MigrationWorker`]. struct MigrationThreadOut { vm: Vm, @@ -750,7 +716,12 @@ pub struct Vmm { #[cfg(feature = "guest_debug")] vm_debug_evt: EventFd, version: VmmVersionInfo, - vm: MaybeVmOwnership, + /// The currently running [`Vm`] instance, if any. + /// + /// This is `Some` from the boot to the shutdown of a VM. In the special + /// case of an ongoing live-migration, this is temporarily `None` and held + /// by a guard to prevent modifications to the VM. + vm: Option, vm_config: Option>>, seccomp_action: SeccompAction, hypervisor: Arc, @@ -980,7 +951,7 @@ impl Vmm { #[cfg(feature = "guest_debug")] vm_debug_evt, version: vmm_version, - vm: MaybeVmOwnership::None, + vm: None, vm_config: None, seccomp_action, hypervisor, @@ -1158,7 +1129,7 @@ impl Vmm { Command::Complete => { // The unwrap is safe, because the state machine makes sure we called // vm_receive_state before, which creates the VM. - let vm = self.vm.vm_mut().unwrap(); + let vm = self.vm.as_mut().unwrap(); // Advertise new VM location to network switches. // The thread in background periodically sends multiple messages. @@ -1409,7 +1380,7 @@ impl Vmm { Ok(vm) })?; - self.vm = MaybeVmOwnership::Vmm(vm); + self.vm = Some(vm); Ok((receive_duration, restore_duration)) } @@ -1882,10 +1853,6 @@ impl Vmm { prefault: bool, memory_restore_mode: MemoryRestoreMode, ) -> std::result::Result<(), VmError> { - if matches!(self.vm, MaybeVmOwnership::Migration) { - return Err(VmError::VmMigrating); - } - let snapshot = recv_vm_state(source_url).map_err(VmError::Restore)?; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] let vm_snapshot = get_vm_snapshot(&snapshot).map_err(VmError::Restore)?; @@ -1934,7 +1901,7 @@ impl Vmm { Some(prefault), Some(memory_restore_mode), )?; - self.vm = MaybeVmOwnership::Vmm(vm); + self.vm = Some(vm); if self .vm_config @@ -1949,8 +1916,11 @@ impl Vmm { } // Now we can restore the rest of the VM. - // PANIC: won't panic, we just checked that the VM is there. - self.vm.vm_mut().unwrap().restore() + if let Some(ref mut vm) = self.vm { + vm.restore() + } else { + Err(VmError::VmNotCreated) + } } /// Checks the migration result. @@ -1969,11 +1939,11 @@ impl Vmm { .join() .expect("should have joined"); + // Give VMM back control. + self.vm = Some(vm); + match migration_res { Ok(()) => { - self.vm = MaybeVmOwnership::None; - drop(vm); - // Shutdown the VM after the migration succeeded if let Err(e) = self.exit_evt.write(1) { error!("Failed shutting down the VM after migration: {e}"); @@ -1981,9 +1951,6 @@ impl Vmm { } Err(e) => { error!("Migration failed: {e}"); - - // Give VMM back control. - self.vm = MaybeVmOwnership::Vmm(vm); } } } @@ -2048,7 +2015,7 @@ impl Vmm { } } EpollDispatch::ActivateVirtioDevices => { - if let MaybeVmOwnership::Vmm(ref vm) = self.vm { + if let Some(ref vm) = self.vm { let count = self.activate_evt.read().map_err(Error::EventFdRead)?; info!("Trying to activate pending virtio devices: count = {count}"); vm.activate_virtio_devices() @@ -2073,7 +2040,7 @@ impl Vmm { // Read from the API receiver channel let gdb_request = gdb_receiver.recv().map_err(Error::GdbRequestRecv)?; - let response = if let MaybeVmOwnership::Vmm(ref mut vm) = self.vm { + let response = if let Some(ref mut vm) = self.vm { vm.debug_request(&gdb_request.payload, gdb_request.cpu_id) } else { Err(VmError::VmNotRunning) @@ -2146,125 +2113,121 @@ impl RequestHandler for Vmm { tracer::start(); info!("Booting VM"); event!("vm", "booting"); - - if matches!(self.vm, MaybeVmOwnership::Migration) { - return Err(VmError::VmMigrating); - } - - // Create a new VM if we don't have one yet. - if matches!(self.vm, MaybeVmOwnership::None) { - let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; - let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; - let guest_exit_evt = self - .guest_exit_evt - .try_clone() - .map_err(VmError::EventFdClone)?; - #[cfg(feature = "guest_debug")] - let vm_debug_evt = self - .vm_debug_evt - .try_clone() - .map_err(VmError::EventFdClone)?; - let activate_evt = self - .activate_evt - .try_clone() - .map_err(VmError::EventFdClone)?; - - if let Some(ref vm_config) = self.vm_config { - let vm = Vm::new( - Arc::clone(vm_config), - exit_evt, - reset_evt, - guest_exit_evt, - #[cfg(feature = "guest_debug")] - vm_debug_evt, - &self.seccomp_action, - self.hypervisor.clone(), - activate_evt, - self.console_info.clone(), - self.console_resize_pipe.clone(), - Arc::clone(&self.original_termios_opt), - None, - None, - None, - None, - )?; - - self.vm = MaybeVmOwnership::Vmm(vm); + let r = { + trace_scoped!("vm_boot"); + // If we don't have a config, we cannot boot a VM. + if self.vm_config.is_none() { + return Err(VmError::VmMissingConfig); } - } - // Now we can boot the VM. - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - vm.boot()?; - event!("vm", "booted"); + // console_info is set to None in vm_shutdown. re-populate here if empty + if self.console_info.is_none() { + self.console_info = + Some(pre_create_console_devices(self).map_err(VmError::CreateConsoleDevices)?); } - MaybeVmOwnership::None => { - return Err(VmError::VmNotCreated); + + // Create a new VM if we don't have one yet. + if self.vm.is_none() { + let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; + let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; + let guest_exit_evt = self + .guest_exit_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + #[cfg(feature = "guest_debug")] + let vm_debug_evt = self + .vm_debug_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + let activate_evt = self + .activate_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + + if let Some(ref vm_config) = self.vm_config { + let vm = Vm::new( + Arc::clone(vm_config), + exit_evt, + reset_evt, + guest_exit_evt, + #[cfg(feature = "guest_debug")] + vm_debug_evt, + &self.seccomp_action, + self.hypervisor.clone(), + activate_evt, + self.console_info.clone(), + self.console_resize_pipe.clone(), + Arc::clone(&self.original_termios_opt), + None, + None, + None, + None, + )?; + + self.vm = Some(vm); + } } - _ => unreachable!(), - } + // Now we can boot the VM. + if let Some(ref mut vm) = self.vm { + vm.boot() + } else { + Err(VmError::VmNotCreated) + } + }; tracer::end(); - Ok(()) + if r.is_ok() { + event!("vm", "booted"); + } + r } fn vm_pause(&mut self) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => vm.pause().map_err(VmError::Pause), - MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, - MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, + if let Some(ref mut vm) = self.vm { + vm.pause().map_err(VmError::Pause) + } else { + Err(VmError::VmNotRunning) } } fn vm_resume(&mut self) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => vm.resume().map_err(VmError::Resume), - MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, - MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, + if let Some(ref mut vm) = self.vm { + vm.resume().map_err(VmError::Resume) + } else { + Err(VmError::VmNotRunning) } } fn vm_post_migration_announce(&mut self) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref vm) => { - if vm.get_state() != VmState::Running { - return Err(VmError::VmNotRunning); - } - - vm.post_migration_announce(); - Ok(()) + if let Some(ref vm) = self.vm { + if vm.get_state() != VmState::Running { + return Err(VmError::VmNotRunning); } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, - MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, + + vm.post_migration_announce(); + Ok(()) + } else { + Err(VmError::VmNotRunning) } } fn vm_snapshot(&mut self, destination_url: &str) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - // Drain console_info so that FDs are not reused - let _ = self.console_info.take(); - vm.snapshot() - .map_err(VmError::Snapshot) - .and_then(|snapshot| { - vm.send(&snapshot, destination_url) - .map_err(VmError::SnapshotSend) - }) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, - MaybeVmOwnership::None => Err(VmError::VmNotRunning)?, + if let Some(ref mut vm) = self.vm { + // Drain console_info so that FDs are not reused + let _ = self.console_info.take(); + vm.snapshot() + .map_err(VmError::Snapshot) + .and_then(|snapshot| { + vm.send(&snapshot, destination_url) + .map_err(VmError::SnapshotSend) + }) + } else { + Err(VmError::VmNotRunning) } } fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> result::Result<(), VmError> { - match &self.vm { - MaybeVmOwnership::Vmm(_vm) => return Err(VmError::VmAlreadyCreated), - MaybeVmOwnership::Migration => return Err(VmError::VmMigrating), - MaybeVmOwnership::None => (), - } - - if self.vm_config.is_some() { + if self.vm.is_some() || self.vm_config.is_some() { return Err(VmError::VmAlreadyCreated); } @@ -2324,25 +2287,21 @@ impl RequestHandler for Vmm { #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] fn vm_coredump(&mut self, destination_url: &str) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - vm.coredump(destination_url).map_err(VmError::Coredump) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => Err(VmError::VmNotRunning), + if let Some(ref mut vm) = self.vm { + vm.coredump(destination_url).map_err(VmError::Coredump) + } else { + Err(VmError::VmNotRunning) } } fn vm_shutdown(&mut self) -> result::Result<(), VmError> { - let vm = match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => vm, - MaybeVmOwnership::Migration => return Err(VmError::VmMigrating), - MaybeVmOwnership::None => return Err(VmError::VmNotRunning), + let r = if let Some(ref mut vm) = self.vm.take() { + // Drain console_info so that the FDs are not reused + let _ = self.console_info.take(); + vm.shutdown() + } else { + Err(VmError::VmNotRunning) }; - // Drain console_info so that the FDs are not reused - let _ = self.console_info.take(); - let r = vm.shutdown(); - self.vm = MaybeVmOwnership::None; if r.is_ok() { event!("vm", "shutdown"); @@ -2355,14 +2314,13 @@ impl RequestHandler for Vmm { event!("vm", "rebooting"); // First we stop the current VM - let vm = match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => vm, - MaybeVmOwnership::Migration => return Err(VmError::VmMigrating), - MaybeVmOwnership::None => return Err(VmError::VmNotRunning), + let config = if let Some(mut vm) = self.vm.take() { + let config = vm.get_config(); + vm.shutdown()?; + config + } else { + return Err(VmError::VmNotCreated); }; - let config = vm.get_config(); - vm.shutdown()?; - self.vm = MaybeVmOwnership::None; // vm.shutdown() closes all the console devices, so set console_info to None // so that the closed FD #s are not reused. @@ -2417,7 +2375,7 @@ impl RequestHandler for Vmm { // And we boot it vm.boot()?; - self.vm = MaybeVmOwnership::Vmm(vm); + self.vm = Some(vm); event!("vm", "rebooted"); @@ -2425,40 +2383,35 @@ impl RequestHandler for Vmm { } fn vm_info(&self) -> result::Result { - let vm_config = self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; - let vm_config = vm_config.lock().unwrap().clone(); - - let state = match &self.vm { - MaybeVmOwnership::Vmm(vm) => vm.get_state(), - // TODO in theory one could live-migrate a non-running VM .. - MaybeVmOwnership::Migration => VmState::Running, - MaybeVmOwnership::None => VmState::Created, - }; + match &self.vm_config { + Some(vm_config) => { + let state = match &self.vm { + Some(vm) => vm.get_state(), + None => VmState::Created, + }; + let config = vm_config.lock().unwrap().clone(); + + let mut memory_actual_size = + config.memory.total_size() - config.memory.hotplugged_size(); + if let Some(vm) = &self.vm { + memory_actual_size = memory_actual_size.saturating_sub(vm.balloon_size()); + memory_actual_size += vm.virtio_mem_plugged_size(); + } + + let device_tree = self + .vm + .as_ref() + .map(|vm| vm.device_tree().lock().unwrap().clone()); - let mut memory_actual_size = - vm_config.memory.total_size() - vm_config.memory.hotplugged_size(); - match &self.vm { - MaybeVmOwnership::Vmm(vm) => { - memory_actual_size = memory_actual_size.saturating_sub(vm.balloon_size()); - memory_actual_size += vm.virtio_mem_plugged_size(); + Ok(VmInfoResponse { + config: Box::new(config), + state, + memory_actual_size, + device_tree, + }) } - MaybeVmOwnership::Migration => {} - MaybeVmOwnership::None => {} + None => Err(VmError::VmNotCreated), } - - let device_tree = match &self.vm { - MaybeVmOwnership::Vmm(vm) => Some(vm.device_tree().lock().unwrap().clone()), - // TODO we need to fix this - MaybeVmOwnership::Migration => None, - MaybeVmOwnership::None => None, - }; - - Ok(VmInfoResponse { - config: Box::new(vm_config), - state, - memory_actual_size, - device_tree, - }) } fn vmm_ping(&self) -> VmmPingResponse { @@ -2480,20 +2433,15 @@ impl RequestHandler for Vmm { return Ok(()); } - match &self.vm { - MaybeVmOwnership::Vmm(_vm) => { - event!("vm", "deleted"); - - // If a VM is booted, we first try to shut it down. - self.vm_shutdown()?; - self.vm_config = None; - } - MaybeVmOwnership::None => { - self.vm_config = None; - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?, + // If a VM is booted, we first try to shut it down. + if self.vm.is_some() { + self.vm_shutdown()?; } + self.vm_config = None; + + event!("vm", "deleted"); + Ok(()) } @@ -2515,68 +2463,59 @@ impl RequestHandler for Vmm { todo!("doesn't work currently with our thread-local KVM_RUN approach"); } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - vm.resize(desired_vcpus, desired_ram, desired_balloon) - .inspect_err(|e| error!("Error when resizing VM: {e:?}"))?; - Ok(()) + if let Some(ref mut vm) = self.vm { + vm.resize(desired_vcpus, desired_ram, desired_balloon) + .inspect_err(|e| error!("Error when resizing VM: {e:?}"))?; + Ok(()) + } else { + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + if let Some(desired_vcpus) = desired_vcpus { + config.cpus.boot_vcpus = desired_vcpus; } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - if let Some(desired_vcpus) = desired_vcpus { - config.cpus.boot_vcpus = desired_vcpus; - } - if let Some(desired_ram) = desired_ram { - config.memory.size = desired_ram; - } - if let Some(desired_balloon) = desired_balloon - && let Some(balloon_config) = &mut config.balloon - { - balloon_config.size = desired_balloon; - } - - Ok(()) + if let Some(desired_ram) = desired_ram { + config.memory.size = desired_ram; } + if let Some(desired_balloon) = desired_balloon + && let Some(balloon_config) = &mut config.balloon + { + balloon_config.size = desired_balloon; + } + Ok(()) } } fn vm_resize_disk(&mut self, id: String, desired_size: u64) -> result::Result<(), VmError> { self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => vm.resize_disk(&id, desired_size), - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => Err(VmError::ResizeDisk), + if let Some(ref mut vm) = self.vm { + return vm.resize_disk(&id, desired_size); } + + Err(VmError::ResizeDisk) } fn vm_resize_zone(&mut self, id: String, desired_ram: u64) -> result::Result<(), VmError> { self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - vm.resize_zone(&id, desired_ram) - .inspect_err(|e| error!("Error when resizing zone: {e:?}"))?; - Ok(()) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by setting the new desired ram. - let memory_config = &mut self.vm_config.as_ref().unwrap().lock().unwrap().memory; - - if let Some(zones) = &mut memory_config.zones { - for zone in zones.iter_mut() { - if zone.id == id { - zone.size = desired_ram; - return Ok(()); - } + if let Some(ref mut vm) = self.vm { + vm.resize_zone(&id, desired_ram) + .inspect_err(|e| error!("Error when resizing zone: {e:?}"))?; + Ok(()) + } else { + // Update VmConfig by setting the new desired ram. + let memory_config = &mut self.vm_config.as_ref().unwrap().lock().unwrap().memory; + + if let Some(zones) = &mut memory_config.zones { + for zone in zones.iter_mut() { + if zone.id == id { + zone.size = desired_ram; + return Ok(()); } } - - error!("Could not find the memory zone {id} for the resize"); - Err(VmError::ResizeZone) } + + error!("Could not find the memory zone {id} for the resize"); + Err(VmError::ResizeZone) } } @@ -2593,22 +2532,18 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_device(device_cfg).inspect_err(|e| { - error!("Error when adding new device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.devices, device_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_device(device_cfg).inspect_err(|e| { + error!("Error when adding new device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.devices, device_cfg); + Ok(None) } } @@ -2625,45 +2560,35 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_user_device(device_cfg).inspect_err(|e| { - error!("Error when adding new user device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.user_devices, device_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_user_device(device_cfg).inspect_err(|e| { + error!("Error when adding new user device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.user_devices, device_cfg); + Ok(None) } } fn vm_remove_device(&mut self, id: String) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - vm.remove_device(&id) - .inspect_err(|e| error!("Error when removing device from the VM: {e:?}"))?; + if let Some(ref mut vm) = self.vm { + vm.remove_device(&id) + .inspect_err(|e| error!("Error when removing device from the VM: {e:?}"))?; + Ok(()) + } else if let Some(ref config) = self.vm_config { + let mut config = config.lock().unwrap(); + if config.remove_device(&id) { Ok(()) + } else { + Err(VmError::NoDeviceToRemove(id)) } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - if let Some(ref config) = self.vm_config { - let mut config = config.lock().unwrap(); - if config.remove_device(&id) { - Ok(()) - } else { - Err(VmError::NoDeviceToRemove(id)) - } - } else { - Err(VmError::VmNotCreated) - } - } + } else { + Err(VmError::VmNotCreated) } } @@ -2677,22 +2602,18 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_disk(disk_cfg).inspect_err(|e| { - error!("Error when adding new disk to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.disks, disk_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_disk(disk_cfg).inspect_err(|e| { + error!("Error when adding new disk to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.disks, disk_cfg); + Ok(None) } } @@ -2706,32 +2627,52 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_fs(fs_cfg).inspect_err(|e| { - error!("Error when adding new fs to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.fs, fs_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_fs(fs_cfg).inspect_err(|e| { + error!("Error when adding new fs to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.fs, fs_cfg); + Ok(None) } } fn vm_add_generic_vhost_user( &mut self, - _generic_vhost_user_cfg: GenericVhostUserConfig, + generic_vhost_user_cfg: GenericVhostUserConfig, ) -> result::Result>, VmError> { self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; - unimplemented!("removed in our fork for simplicity"); + { + // Validate the configuration change in a cloned configuration + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap().clone(); + add_to_config( + &mut config.generic_vhost_user, + generic_vhost_user_cfg.clone(), + ); + config.validate().map_err(VmError::ConfigValidation)?; + } + + if let Some(ref mut vm) = self.vm { + let info = vm + .add_generic_vhost_user(generic_vhost_user_cfg) + .inspect_err(|e| { + error!("Error when adding new generic vhost-user device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.generic_vhost_user, generic_vhost_user_cfg); + Ok(None) + } } fn vm_add_pmem(&mut self, pmem_cfg: PmemConfig) -> result::Result>, VmError> { @@ -2744,22 +2685,18 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_pmem(pmem_cfg).inspect_err(|e| { - error!("Error when adding new pmem device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.pmem, pmem_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_pmem(pmem_cfg).inspect_err(|e| { + error!("Error when adding new pmem device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.pmem, pmem_cfg); + Ok(None) } } @@ -2773,22 +2710,18 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_net(net_cfg).inspect_err(|e| { - error!("Error when adding new network device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.net, net_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_net(net_cfg).inspect_err(|e| { + error!("Error when adding new network device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.net, net_cfg); + Ok(None) } } @@ -2802,22 +2735,18 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_vdpa(vdpa_cfg).inspect_err(|e| { - error!("Error when adding new vDPA device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.vdpa, vdpa_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_vdpa(vdpa_cfg).inspect_err(|e| { + error!("Error when adding new vDPA device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.vdpa, vdpa_cfg); + Ok(None) } } @@ -2836,53 +2765,47 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.add_vsock(vsock_cfg).inspect_err(|e| { - error!("Error when adding new vsock device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - config.vsock = Some(vsock_cfg); - Ok(None) - } + if let Some(ref mut vm) = self.vm { + let info = vm.add_vsock(vsock_cfg).inspect_err(|e| { + error!("Error when adding new vsock device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + config.vsock = Some(vsock_cfg); + Ok(None) } } fn vm_counters(&mut self) -> result::Result>, VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => { - let info = vm.counters().inspect_err(|e| { - error!("Error when getting counters from the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => Err(VmError::VmNotRunning), + if let Some(ref mut vm) = self.vm { + let info = vm.counters().inspect_err(|e| { + error!("Error when getting counters from the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + Err(VmError::VmNotRunning) } } fn vm_power_button(&mut self) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => vm.power_button(), - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => Err(VmError::VmNotRunning), + if let Some(ref mut vm) = self.vm { + vm.power_button() + } else { + Err(VmError::VmNotRunning) } } fn vm_nmi(&mut self) -> result::Result<(), VmError> { - match self.vm { - MaybeVmOwnership::Vmm(ref mut vm) => vm.nmi(), - MaybeVmOwnership::Migration => Err(VmError::VmMigrating), - MaybeVmOwnership::None => Err(VmError::VmNotRunning), + if let Some(ref mut vm) = self.vm { + vm.nmi() + } else { + Err(VmError::VmNotRunning) } } @@ -2954,7 +2877,7 @@ impl RequestHandler for Vmm { if matches!(res, Err(_) | Ok(ReceiveMigrationState::Aborted)) { event!("vm", "migration-receive-failed"); - self.vm = MaybeVmOwnership::None; + self.vm = None; self.vm_config = None; return match res { Ok(_) => Err(MigratableError::CompleteMigration(anyhow!( @@ -2976,18 +2899,6 @@ impl RequestHandler for Vmm { .context("Invalid send migration configuration") .map_err(MigratableError::MigrateSend)?; - match self.vm { - MaybeVmOwnership::Vmm(_) => (), - MaybeVmOwnership::Migration => { - return Err(MigratableError::MigrateSend(anyhow!( - "There is already an ongoing migration" - ))); - } - MaybeVmOwnership::None => { - return Err(MigratableError::MigrateSend(anyhow!("VM is not running"))); - } - } - info!( "Sending migration: destination_url={},local={},tls={},downtime={}ms,timeout={}s,timeout_strategy={:?}", send_data_migration.destination_url, @@ -2998,6 +2909,9 @@ impl RequestHandler for Vmm { send_data_migration.timeout_strategy ); + // TODO Check if there is already a migration in progress + // will be done in next commit + if !self .vm_config .as_ref() @@ -3014,7 +2928,10 @@ impl RequestHandler for Vmm { // Take VM ownership. This also means that API events can no longer // change the VM (e.g. net device hotplug). - let vm = self.vm.take_vm_for_migration(); + let vm = self + .vm + .take() + .ok_or(MigratableError::MigrateSend(anyhow!("VM is not running")))?; let initial_vm_state = vm.get_state(); if initial_vm_state != VmState::Running && initial_vm_state != VmState::Paused { @@ -3369,7 +3286,6 @@ mod unit_tests { ); } - #[ignore] // skipped in our fork for simplicity #[test] fn test_vmm_vm_cold_add_generic_vhost_user() { let mut vmm = create_dummy_vmm(); diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 0725c63268..47c57c8e90 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -207,9 +207,6 @@ pub enum Error { #[error("VM is not running")] VmNotRunning, - #[error("VM is currently migrating and can't be modified")] - VmMigrating, - #[error("Cannot clone EventFd")] EventFdClone(#[source] io::Error), From 17aac389c723042fa7e0c0b435371cc051e3db81 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:18:10 +0200 Subject: [PATCH 39/79] vmm: Revert "migration: handle in dedicated thread (make async)" This reverts commit 034935aed4d41c0a8a9c6035b7b6fa46c2d3c8d7. The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 205 ++++++++++++++----------------------------------- 1 file changed, 59 insertions(+), 146 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index ec26d90151..6153ec9029 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -21,7 +21,6 @@ use std::panic::AssertUnwindSafe; use std::path::PathBuf; use std::sync::mpsc::{Receiver, RecvError, SendError, Sender}; use std::sync::{Arc, Mutex}; -use std::thread::JoinHandle; use std::time::Duration; #[cfg(not(target_arch = "riscv64"))] use std::time::Instant; @@ -629,69 +628,6 @@ impl VmmVersionInfo { } } -/// Abstraction for the thread controlling and performing the live migration. -/// -/// The migration thread also takes ownership of the [`Vm`] from the [`Vmm`]. -struct MigrationWorker { - vm: Vm, - check_migration_evt: EventFd, - config: VmSendMigrationData, - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - hypervisor: Arc, -} - -impl MigrationWorker { - /// Performs any final cleanup after failed live migrations. - /// - /// Helper for [`Self::migrate`]. - fn migrate_error_cleanup(&mut self) -> result::Result<(), MigratableError> { - // Stop logging dirty pages only for non-local migrations - if !self.config.local { - self.vm.stop_dirty_log()?; - } - - Ok(()) - } - - /// Migrate and cleanup. - fn migrate(&mut self) -> result::Result<(), MigratableError> { - debug!("start sending migration"); - event!("vm", "migration-started"); - Vmm::send_migration( - &mut self.vm, - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - self.hypervisor.as_ref(), - &self.config, - ) - .inspect(|_| event!("vm", "migration-finished")) - .inspect_err(|_| { - event!("vm", "migration-failed"); - let e = self.migrate_error_cleanup(); - if let Err(e) = e { - error!("Failed to clean up after a failed live migration. VM might keep running but in an odd or possibly slowed-down state: {e}"); - } - })?; - - Ok(()) - } - - /// Perform the migration and communicate with the [`Vmm`] thread. - fn run(mut self) -> MigrationThreadOut { - debug!("migration thread is starting"); - - let res = self.migrate().inspect_err(|e| error!("migrate error: {e}")); - - // Notify VMM thread to get migration result by joining this thread. - self.check_migration_evt.write(1).unwrap(); - - debug!("migration thread is finished"); - MigrationThreadOut { - vm: self.vm, - migration_res: res, - } - } -} - pub struct VmmThreadHandle { pub thread_handle: thread::JoinHandle>, #[cfg(feature = "dbus_api")] @@ -699,12 +635,6 @@ pub struct VmmThreadHandle { pub http_api_handle: Option, } -/// Output value of [`MigrationWorker`]. -struct MigrationThreadOut { - vm: Vm, - migration_res: result::Result<(), MigratableError>, -} - pub struct Vmm { epoll: EpollContext, exit_evt: EventFd, @@ -716,11 +646,6 @@ pub struct Vmm { #[cfg(feature = "guest_debug")] vm_debug_evt: EventFd, version: VmmVersionInfo, - /// The currently running [`Vm`] instance, if any. - /// - /// This is `Some` from the boot to the shutdown of a VM. In the special - /// case of an ongoing live-migration, this is temporarily `None` and held - /// by a guard to prevent modifications to the VM. vm: Option, vm_config: Option>>, seccomp_action: SeccompAction, @@ -733,10 +658,6 @@ pub struct Vmm { console_info: Option, no_shutdown: bool, check_migration_evt: EventFd, - /// Handle to the [`MigrationWorker`] thread. - /// - /// The handle will return the [`Vm`] back in any case. Further, the underlying error (if any) is returned. - migration_thread_handle: Option>, } /// Just a wrapper for the data that goes into @@ -855,14 +776,14 @@ impl Vmm { .name("vmm_signal_handler".to_string()) .spawn(move || { if !signal_handler_seccomp_filter.is_empty() && let Err(e) = apply_filter(&signal_handler_seccomp_filter) - .map_err(Error::ApplySeccompFilter) - { - error!("Error applying seccomp filter: {e:?}"); - exit_evt.write(1).ok(); - return; - } + .map_err(Error::ApplySeccompFilter) + { + error!("Error applying seccomp filter: {e:?}"); + exit_evt.write(1).ok(); + return; + } - if landlock_enable { + if landlock_enable{ match Landlock::new() { Ok(landlock) => { let _ = landlock.restrict_self().map_err(Error::ApplyLandlock).map_err(|e| { @@ -880,11 +801,11 @@ impl Vmm { std::panic::catch_unwind(AssertUnwindSafe(|| { Vmm::signal_handler(signals, original_termios_opt.as_ref(), &exit_evt); })) - .map_err(|_| { - error!("vmm signal_handler thread panicked"); - exit_evt.write(1).ok() - }) - .ok(); + .map_err(|_| { + error!("vmm signal_handler thread panicked"); + exit_evt.write(1).ok() + }) + .ok(); }) .map_err(Error::SignalHandlerSpawn)?, ); @@ -963,7 +884,6 @@ impl Vmm { console_info: None, no_shutdown, check_migration_evt, - migration_thread_handle: None, }) } @@ -1611,17 +1531,13 @@ impl Vmm { Ok(()) } - /// Performs a live-migration. - /// - /// This function performs necessary after-migration cleanup only in the - /// good case. Callers are responsible for properly handling failed - /// migrations. - #[allow(unused_assignments)] // TODO remove + /// Performs a migration including all its phases. fn send_migration( vm: &mut Vm, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: &dyn hypervisor::Hypervisor, send_data_migration: &VmSendMigrationData, + initial_vm_state: VmState, ) -> result::Result<(), MigratableError> { // State machine that is updated with more context as we progress. let mut ctx = OngoingMigrationContext::new(); @@ -1769,10 +1685,15 @@ impl Vmm { // When this returns, we know the VM was resumed (if it was running // before the migration) and that the receiving VMM acquired disk // locks again. + let complete_req = if initial_vm_state == VmState::Running { + Request::complete() + } else { + Request::complete_paused() + }; let (_, complete_duration) = measure_ok(|| { migration_transport::send_request_expect_ok( &mut socket, - Request::complete(), + complete_req, MigratableError::MigrateSend(anyhow!("Error completing migration")), ) })?; @@ -1929,31 +1850,7 @@ impl Vmm { /// change (and therefore, its termination). The function checks the result /// of that thread and either shuts down the VMM on success or keeps the VM /// and the VMM running on migration failure. - fn check_migration_result(&mut self) { - // At this point, the thread must be finished. - // If we fail here, we have lost anyway. Just panic. - let MigrationThreadOut { vm, migration_res } = self - .migration_thread_handle - .take() - .expect("should have thread") - .join() - .expect("should have joined"); - - // Give VMM back control. - self.vm = Some(vm); - - match migration_res { - Ok(()) => { - // Shutdown the VM after the migration succeeded - if let Err(e) = self.exit_evt.write(1) { - error!("Failed shutting down the VM after migration: {e}"); - } - } - Err(e) => { - error!("Migration failed: {e}"); - } - } - } + fn check_migration_result(&mut self) {} fn control_loop( &mut self, @@ -2909,9 +2806,6 @@ impl RequestHandler for Vmm { send_data_migration.timeout_strategy ); - // TODO Check if there is already a migration in progress - // will be done in next commit - if !self .vm_config .as_ref() @@ -2926,12 +2820,10 @@ impl RequestHandler for Vmm { ))); } - // Take VM ownership. This also means that API events can no longer - // change the VM (e.g. net device hotplug). let vm = self .vm - .take() - .ok_or(MigratableError::MigrateSend(anyhow!("VM is not running")))?; + .as_mut() + .ok_or_else(|| MigratableError::MigrateSend(anyhow!("VM is not running")))?; let initial_vm_state = vm.get_state(); if initial_vm_state != VmState::Running && initial_vm_state != VmState::Paused { @@ -2940,24 +2832,45 @@ impl RequestHandler for Vmm { ))); } - // Start migration thread - let worker = MigrationWorker { + event!("vm", "migration-started"); + Self::send_migration( vm, - check_migration_evt: self.check_migration_evt.try_clone().unwrap(), - config: send_data_migration, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - hypervisor: self.hypervisor.clone(), - }; + self.hypervisor.as_ref(), + &send_data_migration, + initial_vm_state, + ) + .map_err(|migration_err| { + error!("Migration failed: {migration_err:?}"); + event!("vm", "migration-failed"); - self.migration_thread_handle = Some( - thread::Builder::new() - .name("migration".into()) - .spawn(move || worker.run()) - // For upstreaming, we should simply continue and return an - // error when this fails. For our PoC, this is fine. - .unwrap(), - ); - Ok(()) + // Stop logging dirty pages only for non-local migrations + if !send_data_migration.local + && let Err(e) = vm.stop_dirty_log() + { + return e; + } + + // Only resume if the VM was originally running; a VM that was already + // paused before migration should remain paused after failure. + if initial_vm_state == VmState::Running + && vm.get_state() == VmState::Paused + && let Err(e) = vm.resume() + { + return e; + } + + migration_err + })?; + + event!("vm", "migration-finished"); + + // Shutdown the VM after the migration succeeded + self.exit_evt.write(1).map_err(|e| { + MigratableError::MigrateSend(anyhow!( + "Failed shutting down the VM after migration: {e:?}" + )) + }) } } From 17469c45d76221921bc53b04e2a95623c83fb188 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:18:10 +0200 Subject: [PATCH 40/79] vmm: Revert "migration: prepare EventFd for async migration events" This reverts commit a904e9b9d9019e8f0ebcb5bbd206d157a998586d. The upstream asynchronization series implements the same functionality and is cherry-picked in this series instead. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 6153ec9029..b4c94ac7df 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -276,7 +276,6 @@ pub enum EpollDispatch { ActivateVirtioDevices = 3, Debug = 4, GuestExit = 5, - CheckMigration = 6, Unknown, } @@ -290,7 +289,6 @@ impl From for EpollDispatch { 3 => ActivateVirtioDevices, 4 => Debug, 5 => GuestExit, - 6 => CheckMigration, _ => Unknown, } } @@ -657,7 +655,6 @@ pub struct Vmm { console_resize_pipe: Option>, console_info: Option, no_shutdown: bool, - check_migration_evt: EventFd, } /// Just a wrapper for the data that goes into @@ -830,7 +827,6 @@ impl Vmm { let reset_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; let guest_exit_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; let activate_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; - let check_migration_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; epoll .add_event(&exit_evt, EpollDispatch::Exit) @@ -857,10 +853,6 @@ impl Vmm { .add_event(&debug_evt, EpollDispatch::Debug) .map_err(Error::Epoll)?; - epoll - .add_event(&check_migration_evt, EpollDispatch::CheckMigration) - .map_err(Error::Epoll)?; - Ok(Vmm { epoll, exit_evt, @@ -883,7 +875,6 @@ impl Vmm { console_resize_pipe: None, console_info: None, no_shutdown, - check_migration_evt, }) } @@ -1844,14 +1835,6 @@ impl Vmm { } } - /// Checks the migration result. - /// - /// This should be called when the migration thread indicated a state - /// change (and therefore, its termination). The function checks the result - /// of that thread and either shuts down the VMM on success or keeps the VM - /// and the VMM running on migration failure. - fn check_migration_result(&mut self) {} - fn control_loop( &mut self, api_receiver: &Receiver, @@ -1952,14 +1935,6 @@ impl Vmm { } #[cfg(not(feature = "guest_debug"))] EpollDispatch::Debug => {} - EpollDispatch::CheckMigration => { - info!("VM migration check event"); - // Consume the event. - self.check_migration_evt - .read() - .map_err(Error::EventFdRead)?; - self.check_migration_result(); - } } } } From 5232436f0cf447bc27f649234f2c1c65a19c3333 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 30 Oct 2025 12:10:51 +0100 Subject: [PATCH 41/79] vmm: improve VM ownership handling Cherry-picked from upstream commit 15cab7ee6 ("vmm: improve VM ownership handling"). Introducing a new enum that models the various states of VM ownership from the perspective of the VMM. This is an important prerequisite for the asynchronization of the migration, where the ownership of the Vm struct is transferred to the migration thread. Specifically, this allows to introduces a new "Migration(ThreadHandle)" variant and all existing match statements can be easily extended to react accordingly. The fork-only handlers vm_post_migration_announce() and vm_resize() were adapted to the new enum as well. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 561 ++++++++++++++++++++++++++++--------------------- 1 file changed, 316 insertions(+), 245 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index b4c94ac7df..cbb2d30f8a 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -24,7 +24,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; #[cfg(not(target_arch = "riscv64"))] use std::time::Instant; -use std::{io, result, thread}; +use std::{io, mem, result, thread}; use anyhow::{Context, anyhow}; #[cfg(feature = "dbus_api")] @@ -633,6 +633,43 @@ pub struct VmmThreadHandle { pub http_api_handle: Option, } +/// Models the current ownership and associated state of the VM from the +/// perspective of the VMM. +#[cfg_attr(feature = "tdx", expect(clippy::large_enum_variant))] +pub enum VmOwnership { + Owned(Vm), + None, +} + +impl VmOwnership { + /// Returns a shared reference to the underlying VM, if available. + fn as_ref(&self) -> Option<&Vm> { + match self { + VmOwnership::Owned(vm) => Some(vm), + _ => None, + } + } + + /// Returns a mutable reference to the underlying VM, if available. + fn as_mut(&mut self) -> Option<&mut Vm> { + match self { + VmOwnership::Owned(vm) => Some(vm), + _ => None, + } + } + + /// Takes the inner VM if it is currently owned. + fn take_owned(&mut self) -> Option { + match mem::replace(self, VmOwnership::None) { + VmOwnership::Owned(vm) => Some(vm), + old => { + *self = old; + None + } + } + } +} + pub struct Vmm { epoll: EpollContext, exit_evt: EventFd, @@ -644,7 +681,7 @@ pub struct Vmm { #[cfg(feature = "guest_debug")] vm_debug_evt: EventFd, version: VmmVersionInfo, - vm: Option, + vm: VmOwnership, vm_config: Option>>, seccomp_action: SeccompAction, hypervisor: Arc, @@ -864,7 +901,7 @@ impl Vmm { #[cfg(feature = "guest_debug")] vm_debug_evt, version: vmm_version, - vm: None, + vm: VmOwnership::None, vm_config: None, seccomp_action, hypervisor, @@ -1038,9 +1075,10 @@ impl Vmm { Ok(Completed) } Command::Complete => { - // The unwrap is safe, because the state machine makes sure we called - // vm_receive_state before, which creates the VM. - let vm = self.vm.as_mut().unwrap(); + let vm = self + .vm + .as_mut() + .expect("VM should have been created by now"); // Advertise new VM location to network switches. // The thread in background periodically sends multiple messages. @@ -1291,7 +1329,7 @@ impl Vmm { Ok(vm) })?; - self.vm = Some(vm); + self.vm = VmOwnership::Owned(vm); Ok((receive_duration, restore_duration)) } @@ -1813,7 +1851,7 @@ impl Vmm { Some(prefault), Some(memory_restore_mode), )?; - self.vm = Some(vm); + self.vm = VmOwnership::Owned(vm); if self .vm_config @@ -1828,11 +1866,8 @@ impl Vmm { } // Now we can restore the rest of the VM. - if let Some(ref mut vm) = self.vm { - vm.restore() - } else { - Err(VmError::VmNotCreated) - } + // PANIC: won't panic, we just checked that the VM is there. + self.vm.as_mut().unwrap().restore() } fn control_loop( @@ -1895,7 +1930,7 @@ impl Vmm { } } EpollDispatch::ActivateVirtioDevices => { - if let Some(ref vm) = self.vm { + if let VmOwnership::Owned(ref vm) = self.vm { let count = self.activate_evt.read().map_err(Error::EventFdRead)?; info!("Trying to activate pending virtio devices: count = {count}"); vm.activate_virtio_devices() @@ -1920,7 +1955,7 @@ impl Vmm { // Read from the API receiver channel let gdb_request = gdb_receiver.recv().map_err(Error::GdbRequestRecv)?; - let response = if let Some(ref mut vm) = self.vm { + let response = if let VmOwnership::Owned(ref mut vm) = self.vm { vm.debug_request(&gdb_request.payload, gdb_request.cpu_id) } else { Err(VmError::VmNotRunning) @@ -1999,7 +2034,7 @@ impl RequestHandler for Vmm { } // Create a new VM if we don't have one yet. - if self.vm.is_none() { + if matches!(&self.vm, VmOwnership::None) { let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; let guest_exit_evt = self @@ -2036,12 +2071,12 @@ impl RequestHandler for Vmm { None, )?; - self.vm = Some(vm); + self.vm = VmOwnership::Owned(vm); } } // Now we can boot the VM. - if let Some(ref mut vm) = self.vm { + if let VmOwnership::Owned(vm) = &mut self.vm { vm.boot() } else { Err(VmError::VmNotCreated) @@ -2055,51 +2090,51 @@ impl RequestHandler for Vmm { } fn vm_pause(&mut self) -> result::Result<(), VmError> { - if let Some(ref mut vm) = self.vm { - vm.pause().map_err(VmError::Pause) - } else { - Err(VmError::VmNotRunning) + match self.vm { + VmOwnership::Owned(ref mut vm) => vm.pause().map_err(VmError::Pause), + VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_resume(&mut self) -> result::Result<(), VmError> { - if let Some(ref mut vm) = self.vm { - vm.resume().map_err(VmError::Resume) - } else { - Err(VmError::VmNotRunning) + match self.vm { + VmOwnership::Owned(ref mut vm) => vm.resume().map_err(VmError::Resume), + VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_post_migration_announce(&mut self) -> result::Result<(), VmError> { - if let Some(ref vm) = self.vm { - if vm.get_state() != VmState::Running { - return Err(VmError::VmNotRunning); - } + match self.vm { + VmOwnership::Owned(ref vm) => { + if vm.get_state() != VmState::Running { + return Err(VmError::VmNotRunning); + } - vm.post_migration_announce(); - Ok(()) - } else { - Err(VmError::VmNotRunning) + vm.post_migration_announce(); + Ok(()) + } + VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_snapshot(&mut self, destination_url: &str) -> result::Result<(), VmError> { - if let Some(ref mut vm) = self.vm { - // Drain console_info so that FDs are not reused - let _ = self.console_info.take(); - vm.snapshot() - .map_err(VmError::Snapshot) - .and_then(|snapshot| { - vm.send(&snapshot, destination_url) - .map_err(VmError::SnapshotSend) - }) - } else { - Err(VmError::VmNotRunning) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + // Drain console_info so that FDs are not reused + let _ = self.console_info.take(); + vm.snapshot() + .map_err(VmError::Snapshot) + .and_then(|snapshot| { + vm.send(&snapshot, destination_url) + .map_err(VmError::SnapshotSend) + }) + } + VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> result::Result<(), VmError> { - if self.vm.is_some() || self.vm_config.is_some() { + if self.vm_config.is_some() || matches!(self.vm, VmOwnership::Owned(_)) { return Err(VmError::VmAlreadyCreated); } @@ -2159,21 +2194,19 @@ impl RequestHandler for Vmm { #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] fn vm_coredump(&mut self, destination_url: &str) -> result::Result<(), VmError> { - if let Some(ref mut vm) = self.vm { - vm.coredump(destination_url).map_err(VmError::Coredump) - } else { - Err(VmError::VmNotRunning) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + vm.coredump(destination_url).map_err(VmError::Coredump) + } + VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_shutdown(&mut self) -> result::Result<(), VmError> { - let r = if let Some(ref mut vm) = self.vm.take() { - // Drain console_info so that the FDs are not reused - let _ = self.console_info.take(); - vm.shutdown() - } else { - Err(VmError::VmNotRunning) - }; + let mut vm = self.vm.take_owned().ok_or(VmError::VmNotRunning)?; + // Drain console_info so that the FDs are not reused + let _ = self.console_info.take(); + let r = vm.shutdown(); if r.is_ok() { event!("vm", "shutdown"); @@ -2185,13 +2218,14 @@ impl RequestHandler for Vmm { fn vm_reboot(&mut self) -> result::Result<(), VmError> { event!("vm", "rebooting"); - // First we stop the current VM - let config = if let Some(mut vm) = self.vm.take() { + // Drop VM early to release disk locks and free other resources before + // we reboot. + let config = { + let mut vm = self.vm.take_owned().ok_or(VmError::VmNotCreated)?; let config = vm.get_config(); + // First we stop the current VM vm.shutdown()?; config - } else { - return Err(VmError::VmNotCreated); }; // vm.shutdown() closes all the console devices, so set console_info to None @@ -2247,7 +2281,7 @@ impl RequestHandler for Vmm { // And we boot it vm.boot()?; - self.vm = Some(vm); + self.vm = VmOwnership::Owned(vm); event!("vm", "rebooted"); @@ -2258,14 +2292,14 @@ impl RequestHandler for Vmm { match &self.vm_config { Some(vm_config) => { let state = match &self.vm { - Some(vm) => vm.get_state(), - None => VmState::Created, + VmOwnership::Owned(vm) => vm.get_state(), + VmOwnership::None => VmState::Created, }; let config = vm_config.lock().unwrap().clone(); let mut memory_actual_size = config.memory.total_size() - config.memory.hotplugged_size(); - if let Some(vm) = &self.vm { + if let VmOwnership::Owned(vm) = &self.vm { memory_actual_size = memory_actual_size.saturating_sub(vm.balloon_size()); memory_actual_size += vm.virtio_mem_plugged_size(); } @@ -2305,13 +2339,15 @@ impl RequestHandler for Vmm { return Ok(()); } - // If a VM is booted, we first try to shut it down. - if self.vm.is_some() { - self.vm_shutdown()?; + match &self.vm { + VmOwnership::Owned(_vm) => { + // If a VM is booted, we first try to shut it down. + self.vm_shutdown()?; + } + VmOwnership::None => {} } self.vm_config = None; - event!("vm", "deleted"); Ok(()) @@ -2335,59 +2371,63 @@ impl RequestHandler for Vmm { todo!("doesn't work currently with our thread-local KVM_RUN approach"); } - if let Some(ref mut vm) = self.vm { - vm.resize(desired_vcpus, desired_ram, desired_balloon) - .inspect_err(|e| error!("Error when resizing VM: {e:?}"))?; - Ok(()) - } else { - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - if let Some(desired_vcpus) = desired_vcpus { - config.cpus.boot_vcpus = desired_vcpus; - } - if let Some(desired_ram) = desired_ram { - config.memory.size = desired_ram; - } - if let Some(desired_balloon) = desired_balloon - && let Some(balloon_config) = &mut config.balloon - { - balloon_config.size = desired_balloon; + match self.vm { + VmOwnership::Owned(ref mut vm) => vm + .resize(desired_vcpus, desired_ram, desired_balloon) + .inspect_err(|e| error!("Error when resizing VM: {e:?}")), + VmOwnership::None => { + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + if let Some(desired_vcpus) = desired_vcpus { + config.cpus.boot_vcpus = desired_vcpus; + } + if let Some(desired_ram) = desired_ram { + config.memory.size = desired_ram; + } + if let Some(desired_balloon) = desired_balloon + && let Some(balloon_config) = &mut config.balloon + { + balloon_config.size = desired_balloon; + } + + Ok(()) } - Ok(()) } } fn vm_resize_disk(&mut self, id: String, desired_size: u64) -> result::Result<(), VmError> { self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; - if let Some(ref mut vm) = self.vm { - return vm.resize_disk(&id, desired_size); + match self.vm { + VmOwnership::Owned(ref mut vm) => vm.resize_disk(&id, desired_size), + VmOwnership::None => Err(VmError::ResizeDisk), } - - Err(VmError::ResizeDisk) } fn vm_resize_zone(&mut self, id: String, desired_ram: u64) -> result::Result<(), VmError> { self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; - if let Some(ref mut vm) = self.vm { - vm.resize_zone(&id, desired_ram) - .inspect_err(|e| error!("Error when resizing zone: {e:?}"))?; - Ok(()) - } else { - // Update VmConfig by setting the new desired ram. - let memory_config = &mut self.vm_config.as_ref().unwrap().lock().unwrap().memory; - - if let Some(zones) = &mut memory_config.zones { - for zone in zones.iter_mut() { - if zone.id == id { - zone.size = desired_ram; - return Ok(()); + match self.vm { + VmOwnership::Owned(ref mut vm) => { + vm.resize_zone(&id, desired_ram) + .inspect_err(|e| error!("Error when resizing zone: {e:?}"))?; + Ok(()) + } + VmOwnership::None => { + // Update VmConfig by setting the new desired ram. + let memory_config = &mut self.vm_config.as_ref().unwrap().lock().unwrap().memory; + + if let Some(zones) = &mut memory_config.zones { + for zone in zones.iter_mut() { + if zone.id == id { + zone.size = desired_ram; + return Ok(()); + } } } - } - error!("Could not find the memory zone {id} for the resize"); - Err(VmError::ResizeZone) + error!("Could not find the memory zone {id} for the resize"); + Err(VmError::ResizeZone) + } } } @@ -2404,18 +2444,21 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_device(device_cfg).inspect_err(|e| { - error!("Error when adding new device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.devices, device_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_device(device_cfg).inspect_err(|e| { + error!("Error when adding new device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.devices, device_cfg); + Ok(None) + } } } @@ -2432,35 +2475,43 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_user_device(device_cfg).inspect_err(|e| { - error!("Error when adding new user device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.user_devices, device_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_user_device(device_cfg).inspect_err(|e| { + error!("Error when adding new user device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.user_devices, device_cfg); + Ok(None) + } } } fn vm_remove_device(&mut self, id: String) -> result::Result<(), VmError> { - if let Some(ref mut vm) = self.vm { - vm.remove_device(&id) - .inspect_err(|e| error!("Error when removing device from the VM: {e:?}"))?; - Ok(()) - } else if let Some(ref config) = self.vm_config { - let mut config = config.lock().unwrap(); - if config.remove_device(&id) { + match self.vm { + VmOwnership::Owned(ref mut vm) => { + vm.remove_device(&id) + .inspect_err(|e| error!("Error when removing device from the VM: {e:?}"))?; Ok(()) - } else { - Err(VmError::NoDeviceToRemove(id)) } - } else { - Err(VmError::VmNotCreated) + VmOwnership::None => { + if let Some(ref config) = self.vm_config { + let mut config = config.lock().unwrap(); + if config.remove_device(&id) { + Ok(()) + } else { + Err(VmError::NoDeviceToRemove(id)) + } + } else { + Err(VmError::VmNotCreated) + } + } } } @@ -2474,18 +2525,21 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_disk(disk_cfg).inspect_err(|e| { - error!("Error when adding new disk to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.disks, disk_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_disk(disk_cfg).inspect_err(|e| { + error!("Error when adding new disk to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.disks, disk_cfg); + Ok(None) + } } } @@ -2499,18 +2553,21 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_fs(fs_cfg).inspect_err(|e| { - error!("Error when adding new fs to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.fs, fs_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_fs(fs_cfg).inspect_err(|e| { + error!("Error when adding new fs to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.fs, fs_cfg); + Ok(None) + } } } @@ -2530,20 +2587,23 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm - .add_generic_vhost_user(generic_vhost_user_cfg) - .inspect_err(|e| { - error!("Error when adding new generic vhost-user device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.generic_vhost_user, generic_vhost_user_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm + .add_generic_vhost_user(generic_vhost_user_cfg) + .inspect_err(|e| { + error!("Error when adding new generic vhost-user device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.generic_vhost_user, generic_vhost_user_cfg); + Ok(None) + } } } @@ -2557,18 +2617,21 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_pmem(pmem_cfg).inspect_err(|e| { - error!("Error when adding new pmem device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.pmem, pmem_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_pmem(pmem_cfg).inspect_err(|e| { + error!("Error when adding new pmem device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.pmem, pmem_cfg); + Ok(None) + } } } @@ -2582,18 +2645,21 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_net(net_cfg).inspect_err(|e| { - error!("Error when adding new network device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.net, net_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_net(net_cfg).inspect_err(|e| { + error!("Error when adding new network device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.net, net_cfg); + Ok(None) + } } } @@ -2607,18 +2673,21 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_vdpa(vdpa_cfg).inspect_err(|e| { - error!("Error when adding new vDPA device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - add_to_config(&mut config.vdpa, vdpa_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_vdpa(vdpa_cfg).inspect_err(|e| { + error!("Error when adding new vDPA device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.vdpa, vdpa_cfg); + Ok(None) + } } } @@ -2637,47 +2706,49 @@ impl RequestHandler for Vmm { config.validate().map_err(VmError::ConfigValidation)?; } - if let Some(ref mut vm) = self.vm { - let info = vm.add_vsock(vsock_cfg).inspect_err(|e| { - error!("Error when adding new vsock device to the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - // Update VmConfig by adding the new device. - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - config.vsock = Some(vsock_cfg); - Ok(None) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.add_vsock(vsock_cfg).inspect_err(|e| { + error!("Error when adding new vsock device to the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + config.vsock = Some(vsock_cfg); + Ok(None) + } } } fn vm_counters(&mut self) -> result::Result>, VmError> { - if let Some(ref mut vm) = self.vm { - let info = vm.counters().inspect_err(|e| { - error!("Error when getting counters from the VM: {e:?}"); - })?; - serde_json::to_vec(&info) - .map(Some) - .map_err(VmError::SerializeJson) - } else { - Err(VmError::VmNotRunning) + match self.vm { + VmOwnership::Owned(ref mut vm) => { + let info = vm.counters().inspect_err(|e| { + error!("Error when getting counters from the VM: {e:?}"); + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } + VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_power_button(&mut self) -> result::Result<(), VmError> { - if let Some(ref mut vm) = self.vm { - vm.power_button() - } else { - Err(VmError::VmNotRunning) + match self.vm { + VmOwnership::Owned(ref mut vm) => vm.power_button(), + VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_nmi(&mut self) -> result::Result<(), VmError> { - if let Some(ref mut vm) = self.vm { - vm.nmi() - } else { - Err(VmError::VmNotRunning) + match self.vm { + VmOwnership::Owned(ref mut vm) => vm.nmi(), + VmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2749,7 +2820,7 @@ impl RequestHandler for Vmm { if matches!(res, Err(_) | Ok(ReceiveMigrationState::Aborted)) { event!("vm", "migration-receive-failed"); - self.vm = None; + self.vm = VmOwnership::None; self.vm_config = None; return match res { Ok(_) => Err(MigratableError::CompleteMigration(anyhow!( From 80cebd5be952897b6e2bcf3ffd52b346cdbce33a Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 1 Jun 2026 20:36:05 +0200 Subject: [PATCH 42/79] vmm: init migration worker module Cherry-picked from upstream commit e03456769 ("vmm: init migration worker module"). This initializes the module and the thread that will handle (control) the migration. This introduces the new types without the necessary wiring. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 2 + vmm/src/migration_worker.rs | 162 ++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 vmm/src/migration_worker.rs diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index cbb2d30f8a..1c0e9ba36b 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -99,6 +99,8 @@ pub mod landlock; pub mod memory_manager; pub mod migration; pub mod migration_transport; +#[expect(unused)] +mod migration_worker; mod pci_segment; pub mod seccomp_filters; mod serial_manager; diff --git a/vmm/src/migration_worker.rs b/vmm/src/migration_worker.rs new file mode 100644 index 0000000000..986615aeed --- /dev/null +++ b/vmm/src/migration_worker.rs @@ -0,0 +1,162 @@ +// Copyright © 2026 Cyberus Technology GmbH +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Asynchronous migration worker. +//! +//! The migration worker owns the [`Vm`] while migration is in progress, so the +//! VMM cannot run VM lifecycle operations concurrently. To keep the VM +//! recoverable when thread creation fails, [`MigrationWorker::spawn`] creates +//! the thread before transferring the VM through a zero-capacity +//! (rendezvous-channel). If spawning fails, the VM is returned to the caller in +//! [`MigrationWorkerSpawnError`]. + +use std::fmt::{Debug, Formatter}; +#[cfg(all(feature = "kvm", target_arch = "x86_64"))] +use std::sync::Arc; +use std::sync::mpsc::Receiver; +use std::thread; +use std::thread::JoinHandle; + +use event_monitor::event; +use log::warn; +use vm_migration::MigratableError; +use vmm_sys_util::eventfd::EventFd; + +use crate::Vmm; +use crate::api::VmSendMigrationData; +use crate::vm::{Vm, VmState}; + +#[derive(thiserror::Error)] +#[error("Migration worker could not be spawned: {spawn_error}")] +pub struct MigrationWorkerSpawnError { + pub spawn_error: std::io::Error, + pub vm: Vm, +} + +impl Debug for MigrationWorkerSpawnError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MigrationWorkerSpawnError") + .field("spawn_error", &self.spawn_error) + .field("vm", &"") + .finish() + } +} + +pub struct MigrationWorkerHandle { + handle: Option>, +} + +impl MigrationWorkerHandle { + pub fn join(mut self) -> MigrationWorkerResult { + self.handle + .take() + .expect("should have thread") + .join() + .expect("should join migration worker gracefully") + } +} + +impl Drop for MigrationWorkerHandle { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + warn!("Migration worker wasn't cleaned up explicitly via join()"); + handle.join().expect("should not be joined already"); + } + } +} + +pub struct MigrationWorker { + // Keep the VM out of the thread closure until spawning succeeds. + vm_receiver: Receiver, + check_migration_evt: EventFd, + config: VmSendMigrationData, + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + hypervisor: Arc, + initial_vm_state: VmState, +} + +impl MigrationWorker { + /// Drives the migration from its start to its end (success, cancellation, + /// failure) + fn run(self) -> MigrationWorkerResult { + let mut vm = self.vm_receiver.recv().expect("VMM should send VM"); + + event!("vm", "migration-started"); + let res = Vmm::send_migration( + &mut vm, + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + self.hypervisor.as_ref(), + &self.config, + self.initial_vm_state, + ) + .inspect(|_| event!("vm", "migration-finished")) + .inspect_err(|_| event!("vm", "migration-failed")); + + // Notify VMM thread to check migration result. + self.check_migration_evt.write(1).unwrap(); + + MigrationWorkerResult { + vm, + migration_result: res, + initial_vm_state: self.initial_vm_state, + } + } + + /// Spawns a worker to coordinate the migration. + // All code paths need special care to prevent any panic and thus losing the + // VM in case of failure. + #[expect(clippy::result_large_err)] + pub fn spawn( + vm: Vm, + check_migration_evt: EventFd, + config: VmSendMigrationData, + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: Arc< + dyn hypervisor::Hypervisor, + >, + initial_vm_state: VmState, + ) -> Result { + let (vm_sender, vm_receiver) = std::sync::mpsc::sync_channel(0); + let worker = MigrationWorker { + vm_receiver, + check_migration_evt, + config, + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + hypervisor, + initial_vm_state, + }; + + let inner_handle = match thread::Builder::new() + .name("migration-worker".into()) + .spawn(move || worker.run()) + { + Ok(inner_handle) => { + // The zero-capacity (rendezvous-channel) confirms the worker + // has taken VM ownership. + vm_sender + .send(vm) + .expect("thread should be waiting to receive VM"); + inner_handle + } + Err(e) => return Err(MigrationWorkerSpawnError { spawn_error: e, vm }), + }; + + Ok(MigrationWorkerHandle { + handle: Some(inner_handle), + }) + } +} + +/// Return value of [`MigrationWorker`]. +pub struct MigrationWorkerResult { + /// The VM that was migrated. + /// + /// If `migration_result` is `Ok`, the VM is paused and can be deleted. + /// If `migration_result` is `Err`, the VM can be resumed and given back to + /// the VMM. + pub vm: Vm, + /// The result of [`Vmm::send_migration`]. + pub migration_result: Result<(), MigratableError>, + pub initial_vm_state: VmState, +} From d036af37ce4f0704f8e48b291e46a9a8a306dee0 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Fri, 21 Nov 2025 10:48:15 +0100 Subject: [PATCH 43/79] vmm: migration: prepare EventFd for async migration events Cherry-picked from upstream commit 5d835bdff ("vmm: migration: prepare EventFd for async migration events"). This is a pre-requisite for the following commit which puts the migration into a dedicated thread. It allows the VMM to react to migration events (success/failure). The commit series was inspired by @ljcore [0] but was changed quite significantly. [0] https://github.com/cloud-hypervisor/cloud-hypervisor/pull/7038 On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 1c0e9ba36b..002a691715 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -278,6 +278,7 @@ pub enum EpollDispatch { ActivateVirtioDevices = 3, Debug = 4, GuestExit = 5, + CheckMigration = 6, Unknown, } @@ -291,6 +292,7 @@ impl From for EpollDispatch { 3 => ActivateVirtioDevices, 4 => Debug, 5 => GuestExit, + 6 => CheckMigration, _ => Unknown, } } @@ -694,6 +696,7 @@ pub struct Vmm { console_resize_pipe: Option>, console_info: Option, no_shutdown: bool, + check_migration_evt: EventFd, } /// Just a wrapper for the data that goes into @@ -866,6 +869,7 @@ impl Vmm { let reset_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; let guest_exit_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; let activate_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; + let check_migration_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?; epoll .add_event(&exit_evt, EpollDispatch::Exit) @@ -892,6 +896,10 @@ impl Vmm { .add_event(&debug_evt, EpollDispatch::Debug) .map_err(Error::Epoll)?; + epoll + .add_event(&check_migration_evt, EpollDispatch::CheckMigration) + .map_err(Error::Epoll)?; + Ok(Vmm { epoll, exit_evt, @@ -914,6 +922,7 @@ impl Vmm { console_resize_pipe: None, console_info: None, no_shutdown, + check_migration_evt, }) } @@ -1872,6 +1881,9 @@ impl Vmm { self.vm.as_mut().unwrap().restore() } + /// Handles the outcome of the migration thread. + fn check_migration(&mut self) {} + fn control_loop( &mut self, api_receiver: &Receiver, @@ -1972,6 +1984,14 @@ impl Vmm { } #[cfg(not(feature = "guest_debug"))] EpollDispatch::Debug => {} + EpollDispatch::CheckMigration => { + info!("VM check migration event"); + // Consume the event. + self.check_migration_evt + .read() + .map_err(Error::EventFdRead)?; + self.check_migration(); + } } } } From e19adcd6507818f93c6fb9ce854c5090820e9c87 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 1 Jun 2026 20:36:26 +0200 Subject: [PATCH 44/79] vmm: migration: handle in dedicated thread (make async) Cherry-picked from upstream commit 796fc055b ("vmm: migration: handle in dedicated thread (make async)"). This puts the send-migration action into a dedicated thread, laying the groundwork for many follow-ups towards first-class live-migration in CH. This means: 1. The send-migration call will exit sooner (just trigger the migration - dispatch semantics) 2. Other API calls can be triggered while a migration is ongoing but will not be able to alter the VM as the VM's ownership is transferred from the VMM to the migration thread. Example: hotplugging won't work (which is good). 3. This is the basis for migration statistics via a dedicated endpoint (future work). The whole change was done with a special focus on graceful recover and cleanup: even if anything on the migration paths go wrong, the proper cleanups are already executed and the VMM can take back the ownership of the VM. The receive-migration API call remains blocking. To observe any status changes about the migration on the sender side, one can observe the event-monitor output and look for `vm.migration-{failed,finished}`. These changes are inspired by [0] but differ significantly in details. [0] https://github.com/cloud-hypervisor/cloud-hypervisor/pull/7038 Adapted to the fork: only the new "ongoing migration" guard is added to vm_receive_migration(), as the fork does not carry upstream's guard against receiving into an existing VM, and the fork-only vm_post_migration_announce() handler gets the new match arm. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 300 +++++++++++++++++++++++++++++++++++-------------- vmm/src/vm.rs | 3 + 2 files changed, 219 insertions(+), 84 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 002a691715..8da5db70b0 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -73,6 +73,7 @@ use crate::migration::{recv_vm_config, recv_vm_state}; use crate::migration_transport::{ ReceiveAdditionalConnections, ReceiveListener, SendAdditionalConnections, SocketStream, }; +use crate::migration_worker::{MigrationWorker, MigrationWorkerHandle, MigrationWorkerResult}; use crate::seccomp_filters::{Thread, get_seccomp_filter}; use crate::vm::{Error as VmError, Vm, VmState}; use crate::vm_config::{ @@ -99,7 +100,6 @@ pub mod landlock; pub mod memory_manager; pub mod migration; pub mod migration_transport; -#[expect(unused)] mod migration_worker; mod pci_segment; pub mod seccomp_filters; @@ -639,21 +639,18 @@ pub struct VmmThreadHandle { /// Models the current ownership and associated state of the VM from the /// perspective of the VMM. -#[cfg_attr(feature = "tdx", expect(clippy::large_enum_variant))] pub enum VmOwnership { Owned(Vm), + /// The VM is temporarily owned by an ongoing migration worker. + Migration { + migration_worker_handle: MigrationWorkerHandle, + /// Snapshot returned while the VMM cannot inspect the worker-owned VM. + vm_info_response: VmInfoResponse, + }, None, } impl VmOwnership { - /// Returns a shared reference to the underlying VM, if available. - fn as_ref(&self) -> Option<&Vm> { - match self { - VmOwnership::Owned(vm) => Some(vm), - _ => None, - } - } - /// Returns a mutable reference to the underlying VM, if available. fn as_mut(&mut self) -> Option<&mut Vm> { match self { @@ -663,13 +660,22 @@ impl VmOwnership { } /// Takes the inner VM if it is currently owned. - fn take_owned(&mut self) -> Option { + fn take_owned_or(&mut self, none_error: VmError) -> result::Result { match mem::replace(self, VmOwnership::None) { - VmOwnership::Owned(vm) => Some(vm), - old => { + VmOwnership::Owned(vm) => Ok(vm), + old @ VmOwnership::Migration { .. } => { *self = old; - None + Err(VmError::VmMigrating) } + VmOwnership::None => Err(none_error), + } + } + + /// Returns an error if the VM is currently migrated. + fn ok_or_migrating(&self) -> result::Result<(), VmError> { + match self { + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), + _ => Ok(()), } } } @@ -1571,7 +1577,10 @@ impl Vmm { Ok(()) } - /// Performs a migration including all its phases. + /// Performs a migration. + /// + /// Runs after-migration cleanup only on success. Callers must handle failed + /// migrations. fn send_migration( vm: &mut Vm, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -1814,6 +1823,8 @@ impl Vmm { prefault: bool, memory_restore_mode: MemoryRestoreMode, ) -> std::result::Result<(), VmError> { + self.vm.ok_or_migrating()?; + let snapshot = recv_vm_state(source_url).map_err(VmError::Restore)?; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] let vm_snapshot = get_vm_snapshot(&snapshot).map_err(VmError::Restore)?; @@ -1881,8 +1892,64 @@ impl Vmm { self.vm.as_mut().unwrap().restore() } - /// Handles the outcome of the migration thread. - fn check_migration(&mut self) {} + /// Handles the outcome of the migration worker thread. + fn check_migration(&mut self) { + let VmOwnership::Migration { + migration_worker_handle, + .. + } = mem::replace(&mut self.vm, VmOwnership::None) + else { + panic!("Should only be called after a migration was started"); + }; + let MigrationWorkerResult { + vm, + migration_result: migration_res, + initial_vm_state, + } = migration_worker_handle.join(); + + let mut try_resume_vm_after_failed_migration = |mut vm: Vm| { + // A late failure may leave the VM paused. + if initial_vm_state == VmState::Running && vm.get_state() == VmState::Paused { + match vm.resume() { + Ok(_) => { + info!("Resumed VM successfully after failed migration"); + } + Err(e) => { + error!("Failed resuming VM after failed migration: {e}"); + self.exit_evt.write(1).unwrap(); + } + } + } + + // Ensure full VM performance. The operation is idempotent. + let _ = vm.stop_dirty_log().inspect_err(|e| { + warn!("Failed stopping dirty log after resuming VM: {e} - VM performance might be slower than usual"); + }); + + self.vm = VmOwnership::Owned(vm); + }; + + match migration_res { + Ok(()) => { + self.vm = VmOwnership::None; + let mut vm = vm; + + // Since the VMM explicitly no longer owns the VM, the exit + // event won't call the shutdown path automatically. + if let Err(e) = vm.shutdown() { + error!("Failed shutting down the VM after migration: {e}"); + } + + if let Err(e) = self.exit_evt.write(1) { + error!("Failed exiting the VMM after migration: {e}"); + } + } + Err(e) => { + error!("Migration failed: {e}"); + try_resume_vm_after_failed_migration(vm); + } + } + } fn control_loop( &mut self, @@ -1923,6 +1990,7 @@ impl Vmm { info!("VM exit event"); // Consume the event. self.exit_evt.read().map_err(Error::EventFdRead)?; + // TODO: Future follow-up must resolve lifecycle handling while migrating. self.vmm_shutdown().map_err(Error::VmmShutdown)?; break 'outer; @@ -1931,11 +1999,13 @@ impl Vmm { info!("VM reset event"); // Consume the event. self.reset_evt.read().map_err(Error::EventFdRead)?; + // TODO: Future follow-up must resolve lifecycle handling while migrating. self.vm_reboot().map_err(Error::VmReboot)?; } EpollDispatch::GuestExit => { info!("VM guest exit event"); self.guest_exit_evt.read().map_err(Error::EventFdRead)?; + // TODO: Future follow-up must resolve lifecycle handling while migrating. if self.no_shutdown { self.vm_shutdown().map_err(Error::VmShutdown)?; } else { @@ -1944,8 +2014,9 @@ impl Vmm { } } EpollDispatch::ActivateVirtioDevices => { + // TODO: Future follow-up must resolve virtio activation handling while migrating. + let count = self.activate_evt.read().map_err(Error::EventFdRead)?; if let VmOwnership::Owned(ref vm) = self.vm { - let count = self.activate_evt.read().map_err(Error::EventFdRead)?; info!("Trying to activate pending virtio devices: count = {count}"); vm.activate_virtio_devices() .map_err(Error::ActivateVirtioDevices)?; @@ -1969,10 +2040,12 @@ impl Vmm { // Read from the API receiver channel let gdb_request = gdb_receiver.recv().map_err(Error::GdbRequestRecv)?; - let response = if let VmOwnership::Owned(ref mut vm) = self.vm { - vm.debug_request(&gdb_request.payload, gdb_request.cpu_id) - } else { - Err(VmError::VmNotRunning) + let response = match self.vm { + VmOwnership::Owned(ref mut vm) => { + vm.debug_request(&gdb_request.payload, gdb_request.cpu_id) + } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), + VmOwnership::None => Err(VmError::VmNotRunning), } .map_err(gdb::Error::Vm); @@ -2017,6 +2090,8 @@ fn apply_landlock(vm_config: &mut VmConfig) -> result::Result<(), LandlockError> impl RequestHandler for Vmm { fn vm_create(&mut self, config: Box) -> result::Result<(), VmError> { + self.vm.ok_or_migrating()?; + // We only store the passed VM config. // The VM will be created when being asked to boot it. if self.vm_config.is_some() { @@ -2039,6 +2114,8 @@ impl RequestHandler for Vmm { } fn vm_boot(&mut self) -> result::Result<(), VmError> { + self.vm.ok_or_migrating()?; + tracer::start(); info!("Booting VM"); event!("vm", "booting"); @@ -2114,6 +2191,7 @@ impl RequestHandler for Vmm { fn vm_pause(&mut self) -> result::Result<(), VmError> { match self.vm { VmOwnership::Owned(ref mut vm) => vm.pause().map_err(VmError::Pause), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2121,6 +2199,7 @@ impl RequestHandler for Vmm { fn vm_resume(&mut self) -> result::Result<(), VmError> { match self.vm { VmOwnership::Owned(ref mut vm) => vm.resume().map_err(VmError::Resume), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2135,6 +2214,7 @@ impl RequestHandler for Vmm { vm.post_migration_announce(); Ok(()) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2151,11 +2231,14 @@ impl RequestHandler for Vmm { .map_err(VmError::SnapshotSend) }) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> result::Result<(), VmError> { + self.vm.ok_or_migrating()?; + if self.vm_config.is_some() || matches!(self.vm, VmOwnership::Owned(_)) { return Err(VmError::VmAlreadyCreated); } @@ -2220,12 +2303,13 @@ impl RequestHandler for Vmm { VmOwnership::Owned(ref mut vm) => { vm.coredump(destination_url).map_err(VmError::Coredump) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } fn vm_shutdown(&mut self) -> result::Result<(), VmError> { - let mut vm = self.vm.take_owned().ok_or(VmError::VmNotRunning)?; + let mut vm = self.vm.take_owned_or(VmError::VmNotRunning)?; // Drain console_info so that the FDs are not reused let _ = self.console_info.take(); let r = vm.shutdown(); @@ -2243,7 +2327,7 @@ impl RequestHandler for Vmm { // Drop VM early to release disk locks and free other resources before // we reboot. let config = { - let mut vm = self.vm.take_owned().ok_or(VmError::VmNotCreated)?; + let mut vm = self.vm.take_owned_or(VmError::VmNotCreated)?; let config = vm.get_config(); // First we stop the current VM vm.shutdown()?; @@ -2311,35 +2395,42 @@ impl RequestHandler for Vmm { } fn vm_info(&self) -> result::Result { - match &self.vm_config { - Some(vm_config) => { - let state = match &self.vm { - VmOwnership::Owned(vm) => vm.get_state(), - VmOwnership::None => VmState::Created, - }; - let config = vm_config.lock().unwrap().clone(); + // In case of a migration, we emit the old VM info, as the VM is + // immutable during a migration. + if let VmOwnership::Migration { + vm_info_response, .. + } = &self.vm + { + return Ok(vm_info_response.clone()); + } - let mut memory_actual_size = - config.memory.total_size() - config.memory.hotplugged_size(); - if let VmOwnership::Owned(vm) = &self.vm { - memory_actual_size = memory_actual_size.saturating_sub(vm.balloon_size()); - memory_actual_size += vm.virtio_mem_plugged_size(); - } + let vm_config = self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; + let vm_config = vm_config.lock().unwrap().clone(); - let device_tree = self - .vm - .as_ref() - .map(|vm| vm.device_tree().lock().unwrap().clone()); + let state = match &self.vm { + VmOwnership::Owned(vm) => vm.get_state(), + VmOwnership::None => VmState::Created, + VmOwnership::Migration { .. } => unreachable!("migration path is handled above"), + }; - Ok(VmInfoResponse { - config: Box::new(config), - state, - memory_actual_size, - device_tree, - }) - } - None => Err(VmError::VmNotCreated), - } + let base_memory_actual_size = + vm_config.memory.total_size() - vm_config.memory.hotplugged_size(); + let (memory_actual_size, device_tree) = match &self.vm { + VmOwnership::Owned(vm) => ( + base_memory_actual_size.saturating_sub(vm.balloon_size()) + + vm.virtio_mem_plugged_size(), + Some(vm.device_tree().lock().unwrap().clone()), + ), + VmOwnership::None => (base_memory_actual_size, None), + VmOwnership::Migration { .. } => unreachable!("migration path is handled above"), + }; + + Ok(VmInfoResponse { + config: Box::new(vm_config), + state, + memory_actual_size, + device_tree, + }) } fn vmm_ping(&self) -> VmmPingResponse { @@ -2366,6 +2457,7 @@ impl RequestHandler for Vmm { // If a VM is booted, we first try to shut it down. self.vm_shutdown()?; } + VmOwnership::Migration { .. } => return Err(VmError::VmMigrating), VmOwnership::None => {} } @@ -2397,6 +2489,7 @@ impl RequestHandler for Vmm { VmOwnership::Owned(ref mut vm) => vm .resize(desired_vcpus, desired_ram, desired_balloon) .inspect_err(|e| error!("Error when resizing VM: {e:?}")), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); if let Some(desired_vcpus) = desired_vcpus { @@ -2421,6 +2514,7 @@ impl RequestHandler for Vmm { match self.vm { VmOwnership::Owned(ref mut vm) => vm.resize_disk(&id, desired_size), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::ResizeDisk), } } @@ -2434,6 +2528,7 @@ impl RequestHandler for Vmm { .inspect_err(|e| error!("Error when resizing zone: {e:?}"))?; Ok(()) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by setting the new desired ram. let memory_config = &mut self.vm_config.as_ref().unwrap().lock().unwrap().memory; @@ -2475,6 +2570,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2506,6 +2602,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2522,6 +2619,7 @@ impl RequestHandler for Vmm { .inspect_err(|e| error!("Error when removing device from the VM: {e:?}"))?; Ok(()) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { if let Some(ref config) = self.vm_config { let mut config = config.lock().unwrap(); @@ -2556,6 +2654,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2584,6 +2683,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2620,6 +2720,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2648,6 +2749,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2676,6 +2778,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2704,6 +2807,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2737,6 +2841,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => { // Update VmConfig by adding the new device. let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); @@ -2756,6 +2861,7 @@ impl RequestHandler for Vmm { .map(Some) .map_err(VmError::SerializeJson) } + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2763,6 +2869,7 @@ impl RequestHandler for Vmm { fn vm_power_button(&mut self) -> result::Result<(), VmError> { match self.vm { VmOwnership::Owned(ref mut vm) => vm.power_button(), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2770,6 +2877,7 @@ impl RequestHandler for Vmm { fn vm_nmi(&mut self) -> result::Result<(), VmError> { match self.vm { VmOwnership::Owned(ref mut vm) => vm.nmi(), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), VmOwnership::None => Err(VmError::VmNotRunning), } } @@ -2778,6 +2886,15 @@ impl RequestHandler for Vmm { &mut self, receive_data_migration: VmReceiveMigrationData, ) -> result::Result<(), MigratableError> { + match &self.vm { + VmOwnership::Migration { .. } => { + return Err(MigratableError::MigrateReceive(anyhow!( + "There is already an ongoing migration" + ))); + } + VmOwnership::Owned(_) | VmOwnership::None => {} + } + receive_data_migration .validate() .context("Invalid receive migration configuration") @@ -2855,10 +2972,27 @@ impl RequestHandler for Vmm { Ok(()) } + /// Dispatches a migration. + /// + /// Returns an error if the migration worker cannot be spawned. Once + /// spawned, [`Vmm::check_migration`] will be called after the thread exits + /// (on success, cancellation, or failure). fn vm_send_migration( &mut self, send_data_migration: VmSendMigrationData, ) -> result::Result<(), MigratableError> { + match self.vm { + VmOwnership::Owned(_) => (), + VmOwnership::Migration { .. } => { + return Err(MigratableError::MigrateSend(anyhow!( + "There is already an ongoing migration" + ))); + } + VmOwnership::None => { + return Err(MigratableError::MigrateSend(anyhow!("VM is not running"))); + } + } + send_data_migration .validate() .context("Invalid send migration configuration") @@ -2900,45 +3034,43 @@ impl RequestHandler for Vmm { ))); } - event!("vm", "migration-started"); - Self::send_migration( + let vm_info_snapshot = self.vm_info().map_err(|e| { + MigratableError::MigrateSend(anyhow!("Failed to query VM info snapshot: {e}")) + })?; + + let check_migration_evt = self + .check_migration_evt + .try_clone() + .with_context(|| "Failed to clone check_migration_evt FD") + .map_err(MigratableError::MigrateSend)?; + + // Take VM ownership. This also means that API events can no longer + // change the VM (e.g. net device hotplug). + let vm = self + .vm + .take_owned_or(VmError::VmNotRunning) + .expect("should have VM ownership as we just checked it"); + + match MigrationWorker::spawn( vm, + check_migration_evt, + send_data_migration, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - self.hypervisor.as_ref(), - &send_data_migration, + self.hypervisor.clone(), initial_vm_state, - ) - .map_err(|migration_err| { - error!("Migration failed: {migration_err:?}"); - event!("vm", "migration-failed"); - - // Stop logging dirty pages only for non-local migrations - if !send_data_migration.local - && let Err(e) = vm.stop_dirty_log() - { - return e; + ) { + Ok(handle) => { + self.vm = VmOwnership::Migration { + migration_worker_handle: handle, + vm_info_response: vm_info_snapshot, + }; + Ok(()) } - - // Only resume if the VM was originally running; a VM that was already - // paused before migration should remain paused after failure. - if initial_vm_state == VmState::Running - && vm.get_state() == VmState::Paused - && let Err(e) = vm.resume() - { - return e; + Err(e) => { + self.vm = VmOwnership::Owned(e.vm); + Err(MigratableError::MigrateSend(e.spawn_error.into())) } - - migration_err - })?; - - event!("vm", "migration-finished"); - - // Shutdown the VM after the migration succeeded - self.exit_evt.write(1).map_err(|e| { - MigratableError::MigrateSend(anyhow!( - "Failed shutting down the VM after migration: {e:?}" - )) - }) + } } } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 47c57c8e90..0725c63268 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -207,6 +207,9 @@ pub enum Error { #[error("VM is not running")] VmNotRunning, + #[error("VM is currently migrating and can't be modified")] + VmMigrating, + #[error("Cannot clone EventFd")] EventFdClone(#[source] io::Error), From 649a4385bfbd6e62f16b23d9182560dc4c967510 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Wed, 17 Jun 2026 07:29:03 +0200 Subject: [PATCH 45/79] vmm: streamline request handlers to use match{} on vm Cherry-picked from upstream commit 7c7a827de ("vmm: streamline request handlers to use match{} on vm"). This streamlines the behavior with the other request handlers so that now almost every request handler uses a match on self.vm. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 379 +++++++++++++++++++++++++------------------------ 1 file changed, 193 insertions(+), 186 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 8da5db70b0..396d9a81c8 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -670,14 +670,6 @@ impl VmOwnership { VmOwnership::None => Err(none_error), } } - - /// Returns an error if the VM is currently migrated. - fn ok_or_migrating(&self) -> result::Result<(), VmError> { - match self { - VmOwnership::Migration { .. } => Err(VmError::VmMigrating), - _ => Ok(()), - } - } } pub struct Vmm { @@ -1823,73 +1815,78 @@ impl Vmm { prefault: bool, memory_restore_mode: MemoryRestoreMode, ) -> std::result::Result<(), VmError> { - self.vm.ok_or_migrating()?; + match &self.vm { + VmOwnership::Owned(_) => Err(VmError::VmAlreadyCreated), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), + VmOwnership::None => { + let snapshot = recv_vm_state(source_url).map_err(VmError::Restore)?; + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + let vm_snapshot = get_vm_snapshot(&snapshot).map_err(VmError::Restore)?; - let snapshot = recv_vm_state(source_url).map_err(VmError::Restore)?; - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - let vm_snapshot = get_vm_snapshot(&snapshot).map_err(VmError::Restore)?; + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + self.vm_check_cpuid_compatibility(&vm_config, &vm_snapshot.common_cpuid) + .map_err(VmError::Restore)?; - #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - self.vm_check_cpuid_compatibility(&vm_config, &vm_snapshot.common_cpuid) - .map_err(VmError::Restore)?; + self.vm_config = Some(Arc::clone(&vm_config)); - self.vm_config = Some(Arc::clone(&vm_config)); + // Always re-populate the 'console_info' based on the new 'vm_config' + self.console_info = + Some(pre_create_console_devices(self).map_err(VmError::CreateConsoleDevices)?); - // Always re-populate the 'console_info' based on the new 'vm_config' - self.console_info = - Some(pre_create_console_devices(self).map_err(VmError::CreateConsoleDevices)?); + let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; + let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; + let guest_exit_evt = self + .guest_exit_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + #[cfg(feature = "guest_debug")] + let debug_evt = self + .vm_debug_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + let activate_evt = self + .activate_evt + .try_clone() + .map_err(VmError::EventFdClone)?; - let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; - let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; - let guest_exit_evt = self - .guest_exit_evt - .try_clone() - .map_err(VmError::EventFdClone)?; - #[cfg(feature = "guest_debug")] - let debug_evt = self - .vm_debug_evt - .try_clone() - .map_err(VmError::EventFdClone)?; - let activate_evt = self - .activate_evt - .try_clone() - .map_err(VmError::EventFdClone)?; + let mut vm = Vm::new( + vm_config, + exit_evt, + reset_evt, + guest_exit_evt, + #[cfg(feature = "guest_debug")] + debug_evt, + &self.seccomp_action, + self.hypervisor.clone(), + activate_evt, + self.console_info.clone(), + self.console_resize_pipe.clone(), + Arc::clone(&self.original_termios_opt), + Some(&snapshot), + Some(source_url), + Some(prefault), + Some(memory_restore_mode), + )?; - let vm = Vm::new( - vm_config, - exit_evt, - reset_evt, - guest_exit_evt, - #[cfg(feature = "guest_debug")] - debug_evt, - &self.seccomp_action, - self.hypervisor.clone(), - activate_evt, - self.console_info.clone(), - self.console_resize_pipe.clone(), - Arc::clone(&self.original_termios_opt), - Some(&snapshot), - Some(source_url), - Some(prefault), - Some(memory_restore_mode), - )?; - self.vm = VmOwnership::Owned(vm); + if self + .vm_config + .as_ref() + .unwrap() + .lock() + .unwrap() + .landlock_enable + { + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + apply_landlock(&mut config).map_err(VmError::ApplyLandlock)?; + } - if self - .vm_config - .as_ref() - .unwrap() - .lock() - .unwrap() - .landlock_enable - { - let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); - apply_landlock(&mut config).map_err(VmError::ApplyLandlock)?; + // Now we can restore the rest of the VM. + // PANIC: won't panic, we just checked that the VM is there. + vm.restore()?; + self.vm = VmOwnership::Owned(vm); + Ok(()) + } } - - // Now we can restore the rest of the VM. - // PANIC: won't panic, we just checked that the VM is there. - self.vm.as_mut().unwrap().restore() } /// Handles the outcome of the migration worker thread. @@ -2090,7 +2087,10 @@ fn apply_landlock(vm_config: &mut VmConfig) -> result::Result<(), LandlockError> impl RequestHandler for Vmm { fn vm_create(&mut self, config: Box) -> result::Result<(), VmError> { - self.vm.ok_or_migrating()?; + match &self.vm { + VmOwnership::Migration { .. } => return Err(VmError::VmMigrating), + VmOwnership::Owned(_) | VmOwnership::None => {} + } // We only store the passed VM config. // The VM will be created when being asked to boot it. @@ -2114,78 +2114,81 @@ impl RequestHandler for Vmm { } fn vm_boot(&mut self) -> result::Result<(), VmError> { - self.vm.ok_or_migrating()?; - - tracer::start(); - info!("Booting VM"); - event!("vm", "booting"); - let r = { - trace_scoped!("vm_boot"); - // If we don't have a config, we cannot boot a VM. - if self.vm_config.is_none() { - return Err(VmError::VmMissingConfig); - } + match &self.vm { + VmOwnership::Owned(_) => Err(VmError::VmAlreadyCreated), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), + VmOwnership::None => { + tracer::start(); + info!("Booting VM"); + event!("vm", "booting"); + + let r = (|| { + trace_scoped!("vm_boot"); + // If we don't have a config, we cannot boot a VM. + if self.vm_config.is_none() { + return Err(VmError::VmMissingConfig); + } - // console_info is set to None in vm_shutdown. re-populate here if empty - if self.console_info.is_none() { - self.console_info = - Some(pre_create_console_devices(self).map_err(VmError::CreateConsoleDevices)?); - } + // console_info is set to None in vm_shutdown. re-populate here if empty + if self.console_info.is_none() { + self.console_info = Some( + pre_create_console_devices(self) + .map_err(VmError::CreateConsoleDevices)?, + ); + } - // Create a new VM if we don't have one yet. - if matches!(&self.vm, VmOwnership::None) { - let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; - let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; - let guest_exit_evt = self - .guest_exit_evt - .try_clone() - .map_err(VmError::EventFdClone)?; - #[cfg(feature = "guest_debug")] - let vm_debug_evt = self - .vm_debug_evt - .try_clone() - .map_err(VmError::EventFdClone)?; - let activate_evt = self - .activate_evt - .try_clone() - .map_err(VmError::EventFdClone)?; + // Create a new VM if we don't have one yet. + let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; + let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; + let guest_exit_evt = self + .guest_exit_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + #[cfg(feature = "guest_debug")] + let vm_debug_evt = self + .vm_debug_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + let activate_evt = self + .activate_evt + .try_clone() + .map_err(VmError::EventFdClone)?; + + if let Some(ref vm_config) = self.vm_config { + let mut vm = Vm::new( + Arc::clone(vm_config), + exit_evt, + reset_evt, + guest_exit_evt, + #[cfg(feature = "guest_debug")] + vm_debug_evt, + &self.seccomp_action, + self.hypervisor.clone(), + activate_evt, + self.console_info.clone(), + self.console_resize_pipe.clone(), + Arc::clone(&self.original_termios_opt), + None, + None, + None, + None, + )?; + + let r = vm.boot(); + self.vm = VmOwnership::Owned(vm); + r + } else { + Err(VmError::VmNotCreated) + } + })(); - if let Some(ref vm_config) = self.vm_config { - let vm = Vm::new( - Arc::clone(vm_config), - exit_evt, - reset_evt, - guest_exit_evt, - #[cfg(feature = "guest_debug")] - vm_debug_evt, - &self.seccomp_action, - self.hypervisor.clone(), - activate_evt, - self.console_info.clone(), - self.console_resize_pipe.clone(), - Arc::clone(&self.original_termios_opt), - None, - None, - None, - None, - )?; - - self.vm = VmOwnership::Owned(vm); + tracer::end(); + if r.is_ok() { + event!("vm", "booted"); } + r } - - // Now we can boot the VM. - if let VmOwnership::Owned(vm) = &mut self.vm { - vm.boot() - } else { - Err(VmError::VmNotCreated) - } - }; - tracer::end(); - if r.is_ok() { - event!("vm", "booted"); } - r } fn vm_pause(&mut self) -> result::Result<(), VmError> { @@ -2237,64 +2240,68 @@ impl RequestHandler for Vmm { } fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> result::Result<(), VmError> { - self.vm.ok_or_migrating()?; - - if self.vm_config.is_some() || matches!(self.vm, VmOwnership::Owned(_)) { - return Err(VmError::VmAlreadyCreated); - } - - let source_url = restore_cfg.source_url.as_path().to_str(); - if source_url.is_none() { - return Err(VmError::InvalidRestoreSourceUrl); - } - // Safe to unwrap as we checked it was Some(&str). - let source_url = source_url.unwrap(); - - let vm_config = Arc::new(Mutex::new( - recv_vm_config(source_url).map_err(VmError::Restore)?, - )); - restore_cfg - .validate(&vm_config.lock().unwrap().clone()) - .map_err(VmError::ConfigValidation)?; + match &self.vm { + VmOwnership::Owned(_) => Err(VmError::VmAlreadyCreated), + VmOwnership::Migration { .. } => Err(VmError::VmMigrating), + VmOwnership::None => { + if self.vm_config.is_some() { + return Err(VmError::VmAlreadyCreated); + } - // Update VM's net configurations with new fds received for restore operation - if let (Some(restored_nets), Some(vm_net_configs)) = - (restore_cfg.net_fds, &mut vm_config.lock().unwrap().net) - { - for net in restored_nets.iter() { - for net_config in vm_net_configs.iter_mut() { - // update only if the net dev is backed by FDs - if net_config.pci_common.id.as_ref() == Some(&net.id) - && net_config.fds.is_some() - { - net_config.fds.clone_from(&net.fds); + let source_url = restore_cfg.source_url.as_path().to_str(); + if source_url.is_none() { + return Err(VmError::InvalidRestoreSourceUrl); + } + // Safe to unwrap as we checked it was Some(&str). + let source_url = source_url.unwrap(); + + let vm_config = Arc::new(Mutex::new( + recv_vm_config(source_url).map_err(VmError::Restore)?, + )); + restore_cfg + .validate(&vm_config.lock().unwrap().clone()) + .map_err(VmError::ConfigValidation)?; + + // Update VM's net configurations with new fds received for restore operation + if let (Some(restored_nets), Some(vm_net_configs)) = + (restore_cfg.net_fds, &mut vm_config.lock().unwrap().net) + { + for net in restored_nets.iter() { + for net_config in vm_net_configs.iter_mut() { + // update only if the net dev is backed by FDs + if net_config.pci_common.id.as_ref() == Some(&net.id) + && net_config.fds.is_some() + { + net_config.fds.clone_from(&net.fds); + } + } } } - } - } - self.vm_restore( - source_url, - vm_config, - restore_cfg.prefault, - restore_cfg.memory_restore_mode, - ) - .and_then(|()| { - if restore_cfg.resume { - self.vm_resume() - } else { + self.vm_restore( + source_url, + vm_config, + restore_cfg.prefault, + restore_cfg.memory_restore_mode, + ) + .and_then(|()| { + if restore_cfg.resume { + self.vm_resume() + } else { + Ok(()) + } + }) + .map_err(|e| { + error!("VM Restore failed: {e:?}"); + if let Err(e) = self.vm_delete() { + return e; + } + e + })?; + Ok(()) } - }) - .map_err(|e| { - error!("VM Restore failed: {e:?}"); - if let Err(e) = self.vm_delete() { - return e; - } - e - })?; - - Ok(()) + } } #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] From 519cd2f11b3230fe3eb1fbd53c00ad2b3f337625 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 12 May 2026 17:42:20 +0200 Subject: [PATCH 46/79] vmm: move migration modules into folder Cherry-picked from upstream commit a56594324 ("vmm: move migration modules into folder"). The hunk around the `vm.migration-receive-ready` event is dropped, as the fork does not carry that event yet. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 2 +- vmm/src/lib.rs | 32 ++++++++----------- vmm/src/{migration.rs => migration/mod.rs} | 3 ++ .../transport.rs} | 0 .../worker.rs} | 0 5 files changed, 17 insertions(+), 20 deletions(-) rename vmm/src/{migration.rs => migration/mod.rs} (98%) rename vmm/src/{migration_transport.rs => migration/transport.rs} (100%) rename vmm/src/{migration_worker.rs => migration/worker.rs} (100%) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index f62f80aeb0..844e961035 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -54,7 +54,7 @@ pub use self::http::{start_http_fd_thread, start_http_path_thread}; use crate::Error as VmmError; use crate::config::{RestoreConfig, RestoredNetConfig}; use crate::device_tree::DeviceTree; -use crate::migration_transport::MAX_MIGRATION_CONNECTIONS; +use crate::migration::transport::MAX_MIGRATION_CONNECTIONS; use crate::vm::{Error as VmError, VmState}; use crate::vm_config::{ DeviceConfig, DiskConfig, FsConfig, GenericVhostUserConfig, MemoryZoneConfig, NetConfig, diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 396d9a81c8..322bc61420 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -69,11 +69,11 @@ use crate::landlock::Landlock; use crate::memory_manager::MemoryManager; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] use crate::migration::get_vm_snapshot; -use crate::migration::{recv_vm_config, recv_vm_state}; -use crate::migration_transport::{ - ReceiveAdditionalConnections, ReceiveListener, SendAdditionalConnections, SocketStream, +use crate::migration::transport::{ + self, ReceiveAdditionalConnections, ReceiveListener, SendAdditionalConnections, SocketStream, }; -use crate::migration_worker::{MigrationWorker, MigrationWorkerHandle, MigrationWorkerResult}; +use crate::migration::worker::{MigrationWorker, MigrationWorkerHandle, MigrationWorkerResult}; +use crate::migration::{recv_vm_config, recv_vm_state}; use crate::seccomp_filters::{Thread, get_seccomp_filter}; use crate::vm::{Error as VmError, Vm, VmState}; use crate::vm_config::{ @@ -99,8 +99,6 @@ pub mod interrupt; pub mod landlock; pub mod memory_manager; pub mod migration; -pub mod migration_transport; -mod migration_worker; mod pci_segment; pub mod seccomp_filters; mod serial_manager; @@ -1042,11 +1040,7 @@ impl Vmm { // When multiple TCP connections are configured, the worker connections carry // all memory commands and the main connection is used only for control traffic. Command::Memory => { - migration_transport::receive_memory_ranges( - &config_data.guest_memory, - req, - socket, - ) + transport::receive_memory_ranges(&config_data.guest_memory, req, socket) .inspect_err(|_| { // connections.cleanup() already logs all errors that occurred in one of the // threads. Furthermore, this path is only taken in the single-connection case, @@ -1585,19 +1579,19 @@ impl Vmm { // Set up the socket connection let mut socket = if send_data_migration.local { - migration_transport::send_migration_socket( + transport::send_migration_socket( &send_data_migration.destination_url, send_data_migration.tls_dir.as_deref(), )? } else { - migration_transport::send_migration_socket_with_keep_alive( + transport::send_migration_socket_with_keep_alive( &send_data_migration.destination_url, send_data_migration.tls_dir.as_deref(), )? }; // Start the migration - migration_transport::send_request_expect_ok( + transport::send_request_expect_ok( &mut socket, Request::start(), MigratableError::MigrateSend(anyhow!("Error starting migration (got bad response)")), @@ -1661,7 +1655,7 @@ impl Vmm { common_cpuid, memory_manager_data: vm.memory_manager_data(), }; - migration_transport::send_config(&mut socket, &vm_migration_config)?; + transport::send_config(&mut socket, &vm_migration_config)?; // Let every Migratable object know about the migration being started. vm.start_migration()?; @@ -1679,7 +1673,7 @@ impl Vmm { ) .expect("migration context should transition to VmPaused for local migration"); } else { - let mut mem_send = migration_transport::SendAdditionalConnections::new( + let mut mem_send = transport::SendAdditionalConnections::new( &send_data_migration.destination_url, send_data_migration.connections, send_data_migration.tls_dir.as_deref(), @@ -1720,7 +1714,7 @@ impl Vmm { // Capture snapshot and send it let (vm_snapshot, snapshot_duration) = measure_ok(|| vm.snapshot())?; let (_, send_snapshot_duration) = - measure_ok(|| migration_transport::send_state(&mut socket, &vm_snapshot))?; + measure_ok(|| transport::send_state(&mut socket, &vm_snapshot))?; // Complete the migration. // When this returns, we know the VM was resumed (if it was running @@ -1732,7 +1726,7 @@ impl Vmm { Request::complete_paused() }; let (_, complete_duration) = measure_ok(|| { - migration_transport::send_request_expect_ok( + transport::send_request_expect_ok( &mut socket, complete_req, MigratableError::MigrateSend(anyhow!("Error completing migration")), @@ -2916,7 +2910,7 @@ impl RequestHandler for Vmm { receive_data_migration.zones, ); - let mut listener = migration_transport::receive_migration_listener( + let mut listener = transport::receive_migration_listener( &receive_data_migration.receiver_url, receive_data_migration.tls_dir.as_deref(), )?; diff --git a/vmm/src/migration.rs b/vmm/src/migration/mod.rs similarity index 98% rename from vmm/src/migration.rs rename to vmm/src/migration/mod.rs index 7046d838e3..05b64ceada 100644 --- a/vmm/src/migration.rs +++ b/vmm/src/migration/mod.rs @@ -14,6 +14,9 @@ use crate::coredump::GuestDebuggableError; use crate::vm::VmSnapshot; use crate::vm_config::VmConfig; +pub(crate) mod transport; +pub(crate) mod worker; + pub const SNAPSHOT_STATE_FILE: &str = "state.json"; pub const SNAPSHOT_CONFIG_FILE: &str = "config.json"; diff --git a/vmm/src/migration_transport.rs b/vmm/src/migration/transport.rs similarity index 100% rename from vmm/src/migration_transport.rs rename to vmm/src/migration/transport.rs diff --git a/vmm/src/migration_worker.rs b/vmm/src/migration/worker.rs similarity index 100% rename from vmm/src/migration_worker.rs rename to vmm/src/migration/worker.rs From e9ea9b2bd46ff306aada2dd692295f3711121121 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Fri, 29 May 2026 07:52:49 +0200 Subject: [PATCH 47/79] tests: adjust to new dispatch semantics of `ch-remote send-migration` Cherry-picked from upstream commit 6a16b65ea ("tests: adjust to new dispatch semantics of `ch-remote send-migration`"). The destination event monitor path is dropped, as the fork's variant of the test does not use it. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/tests/integration.rs | 71 +++++++++++++++++---------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index 087418b843..e9c4066196 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -6675,6 +6675,8 @@ mod common_parallel { let src_vm_path = clh_command("cloud-hypervisor"); let src_api_socket = temp_api_path(&guest.tmp_dir); + let event_path = temp_event_monitor_path(&guest.tmp_dir); + let src_event_path = format!("{event_path}.src"); let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); src_vm_cmd .args(["--cpus", format!("boot={boot_vcpus}").as_str()]) @@ -6684,6 +6686,7 @@ mod common_parallel { .default_disks() .args(["--net", net_params.as_str()]) .args(["--api-socket", &src_api_socket]) + .args(["--event-monitor", format!("path={src_event_path}").as_str()]) .capture_output(); let mut src_child = src_vm_cmd.spawn().unwrap(); @@ -6748,6 +6751,18 @@ mod common_parallel { .wait_timeout(Duration::from_secs(60)) .unwrap(); + let send_dispatched = match send_status { + Some(status) => status.success(), + None => { + let _ = send_migration.kill(); + false + } + }; + assert!( + send_dispatched, + "send-migration should have dispatched successfully" + ); + // Clean up receive-migration regardless of its outcome if receive_status.is_none() { let _ = receive_migration.kill(); @@ -6759,19 +6774,21 @@ mod common_parallel { match timeout_strategy { TimeoutStrategy::Cancel => { - // With cancel strategy the send must fail and the source VM - // must keep running. - let send_failed = match send_status { - Some(status) => !status.success(), - None => { - let _ = send_migration.kill(); - false - } - }; - assert!( - send_failed, - "send-migration should have failed due to 1s timeout with cancel strategy" - ); + let expected_events = [ + &MetaEvent { + event: "migration-started".to_string(), + device_id: None, + }, + &MetaEvent { + event: "migration-failed".to_string(), + device_id: None, + }, + ]; + assert!(wait_for_sequential_events( + Duration::from_secs(30), + &expected_events, + &src_event_path + )); thread::sleep(Duration::from_secs(2)); assert!( @@ -6783,19 +6800,21 @@ mod common_parallel { assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); } TimeoutStrategy::Ignore => { - // With Ignore strategy the send must succeed despite the timeout - // being reached, and the source VM must have terminated. - let send_succeeded = match send_status { - Some(status) => status.success(), - None => { - let _ = send_migration.kill(); - false - } - }; - assert!( - send_succeeded, - "send-migration should have succeeded with timeout_strategy=ignore" - ); + let expected_events = [ + &MetaEvent { + event: "migration-started".to_string(), + device_id: None, + }, + &MetaEvent { + event: "migration-finished".to_string(), + device_id: None, + }, + ]; + assert!(wait_for_sequential_events( + Duration::from_secs(30), + &expected_events, + &src_event_path + )); thread::sleep(Duration::from_secs(3)); assert!( From fff719946c57c83a27c7ca01b2fac6cfa59aa9f3 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 13 Jan 2026 11:26:55 +0100 Subject: [PATCH 48/79] vm-migration: prepare progress types for new API endpoint This is the first commit in a series of commits to introduce a new API endpoint in Cloud Hypervisor to report progress and live-insights about an ongoing live migration. Having live and frequently refreshing statistics/metrics about an ongoing live migration is especially interesting for debugging and monitoring, such as checking the actual network throughput. With the proposed changes, for the first time, we will be able to see how live migrations behave and create benchmarking infrastructure around it. The ch driver in libvirt will use these information to populate its `virsh domjobinfo` information. We will add a new API endpoint to query information for ongoing live migrations. The new endpoint will also serve to query information about any previously failed or canceled migrations. The SendMigration call will no longer be blocking (wait until the migration is done) but instead just dispatch the migration. This streamlines the behavior with QEMU and simplifies management software. When one queries the endpoint, a frequently refreshed snapshot of the migration statistics and progress will be returned. The data will not be assembled on the fly. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/lib.rs | 1 + vm-migration/src/progress.rs | 564 +++++++++++++++++++++++++++++++++++ 2 files changed, 565 insertions(+) create mode 100644 vm-migration/src/progress.rs diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 2255bceeb7..3b5f25987c 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -16,6 +16,7 @@ use crate::protocol::MemoryRangeTable; mod bitpos_iterator; mod context; pub mod keep_alive_stream; +pub mod progress; pub mod protocol; pub mod tls; diff --git a/vm-migration/src/progress.rs b/vm-migration/src/progress.rs new file mode 100644 index 0000000000..8a5083068d --- /dev/null +++ b/vm-migration/src/progress.rs @@ -0,0 +1,564 @@ +// Copyright © 2025 Cyberus Technology GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +//! Module for reporting status and progress of live migrations. +//! +//! The main export is [`MigrationProgress`]. +//! +//! # Motivation +//! +//! Monitoring a live-migration is important for debugging of cloud deployments, +//! for cloud monitoring in general, and for network optimization, such as +//! verifying the throughput for the migration is as high as expected. +//! +//! It also helps to analyze the downtime of VMs and see how much pressure a +//! guest is putting on its memory (by writing), which is slowing down +//! migrations. + +use std::error::Error; +use std::fmt; +use std::fmt::Display; +use std::num::NonZeroU32; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum TransportationMode { + Local, + Tcp { connections: NonZeroU32, tls: bool }, +} + +/// Carries information about the transmission of the VM's memory. +#[derive( + Clone, + Copy, + Debug, + Default, + PartialOrd, + Ord, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +pub struct MemoryTransmissionInfo { + /// The memory iteration (only in precopy mode). + pub memory_iteration: u64, + /// Memory bytes per second. + pub memory_transmission_bps: u64, + /// The total size of the VMs memory in bytes. + pub memory_bytes_total: u64, + /// The total size of transmitted bytes. + pub memory_bytes_transmitted: u64, + /// The amount of remaining bytes for this iteration. + pub memory_bytes_remaining_iteration: u64, + /// The amount of transmitted 4k pages. + pub memory_pages_4k_transmitted: u64, + /// The amount of remaining 4k pages for this iteration. + pub memory_pages_4k_remaining_iteration: u64, + /// The amount of constant pages for that we could take a shortcut. + /// Pages where all bits are either zero or one. + pub memory_pages_constant_count: u64, + /// Current memory dirty rate in pages per seconds (pps). + pub memory_dirty_rate_pps: u64, +} + +/// The different phases of an ongoing ([`MigrationState::Ongoing`]) migration +/// (good case). +/// +/// The states correspond to the [live-migration protocol]. +/// +/// [live-migration protocol]: super::protocol +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum MigrationStateOngoingPhase { + /// The migration starts. Handshake and transfer of VM config. + Starting, + /// Transfer of memory FDs. + /// + /// Only used for local migrations. + MemoryFds, + /// Transfer of VM memory in precopy mode. + /// + /// Not used for local migrations. + MemoryPrecopy, + // TODO eventually add MemoryPostcopy here + /// The VM migration is completing. This means the last chunks of memory + /// are transmitted as well as the final VM state (vCPUs, devices). + Completing, +} + +impl Display for MigrationStateOngoingPhase { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Starting => write!(f, "starting"), + Self::MemoryFds => write!(f, "memory FDs"), + Self::MemoryPrecopy => write!(f, "memory (precopy)"), + Self::Completing => write!(f, "completing"), + } + } +} + +/// The different states of a migration, covering steady progress and failure. +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum MigrationState { + /// The migration has been cancelled. + Cancelled {}, + /// The migration has failed. + Failed { + /// Stringified error. + error_msg: String, + /// Debug-stringified error. + error_msg_debug: String, + // TODO this is very tricky because I need clone() + // error: Box, + }, + /// The migration has finished successfully. + Finished {}, + /// The migration is ongoing. + Ongoing { + phase: MigrationStateOngoingPhase, + /// Percent in range `0..=100`. + vcpu_throttle_percent: u8, + }, +} + +impl Display for MigrationState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MigrationState::Cancelled { .. } => write!(f, "{}", self.state_name()), + MigrationState::Failed { error_msg, .. } => { + write!(f, "{}: {error_msg}", self.state_name()) + } + MigrationState::Finished { .. } => write!(f, "{}", self.state_name()), + MigrationState::Ongoing { + phase, + vcpu_throttle_percent, + } => write!( + f, + "{}: phase={phase}, vcpu_throttle={vcpu_throttle_percent}", + self.state_name() + ), + } + } +} + +impl MigrationState { + fn state_name(&self) -> &'static str { + match self { + MigrationState::Cancelled { .. } => "cancelled", + MigrationState::Failed { .. } => "failed", + MigrationState::Finished { .. } => "finished", + MigrationState::Ongoing { .. } => "ongoing", + } + } +} + +/// Returns the current UNIX timestamp in ms. +fn current_unix_timestamp_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should be valid duration") + .as_millis() as u64 +} + +/// Holds a snapshot of progress and status information for an ongoing live +/// migration, or the last snapshot of a canceled or aborted migration. +/// +/// This type carries insightful information for every step of the +/// [live-migration protocol] in a way that makes it easy for API users to +/// parse the data with ease while retaining all important information. +/// +/// [live-migration protocol]: super::protocol +#[derive( + Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct MigrationProgress { + /// UNIX timestamp of the start of the live-migration process in ms. + pub timestamp_begin_ms: u64, + /// UNIX timestamp of the current snapshot in ms. + pub timestamp_snapshot_ms: u64, + /// Relative timestamp since the beginning of the migration in ms. + pub timestamp_snapshot_relative_ms: u64, + /// Configured target downtime. + pub downtime_configured_ms: u64, + /// Currently estimated (computed) downtime given the remaining + /// transmissions and the bandwidth. + /// + /// If this is `0`, the downtime could not yet be calculated. + pub downtime_estimated_ms: u64, + /// Requested transportation mode. + pub transportation_mode: TransportationMode, + /// Snapshot of the current phase. + pub state: MigrationState, + /// Latest [`MemoryTransmissionInfo`] info, if any. + /// + /// The most interesting phase is when current state is + /// [`MigrationState::Ongoing`] and [`MigrationStateOngoingPhase::MemoryPrecopy`] + /// as this value will be updated frequently. + pub memory_transmission_info: MemoryTransmissionInfo, +} + +impl MigrationProgress { + /// Creates new progress in a valid init state. + /// + /// This progress must be updated using any of: + /// - [`Self::update`] + /// - [`Self::mark_as_finished`] + /// - [`Self::mark_as_failed`] + /// - [`Self::mark_as_cancelled`] + pub fn new(transportation_mode: TransportationMode, target_downtime: Duration) -> Self { + let timestamp = current_unix_timestamp_ms(); + Self { + timestamp_begin_ms: timestamp, + timestamp_snapshot_ms: timestamp, + timestamp_snapshot_relative_ms: 0, + downtime_configured_ms: target_downtime.as_millis() as u64, + downtime_estimated_ms: 0, + transportation_mode, + state: MigrationState::Ongoing { + phase: MigrationStateOngoingPhase::Starting, + vcpu_throttle_percent: 0, + }, + memory_transmission_info: MemoryTransmissionInfo::default(), + } + } + + /// Updates the state of an ongoing migration. + /// + /// Only updates new values that are provided via `Some`. + /// + /// # Arguments + /// + /// - `new_phase`: The current [`MigrationStateOngoingPhase`]. + /// - `new_memory_transmission_info`: If `Some`, the current [`MemoryTransmissionInfo`]. + /// - `new_cpu_throttle_percent`: If `Some`, the current value of the vCPU throttle percentage. + /// Must be in range `0..=100`. + /// - `new_estimated_downtime`: If `Some`, the latest expected (calculated) downtime. + pub fn update( + &mut self, + new_phase: MigrationStateOngoingPhase, + new_memory_transmission_info: Option, + new_cpu_throttle_percent: Option, + new_estimated_downtime: Option, + ) { + if let Some(percent) = new_cpu_throttle_percent { + assert!(percent <= 100); + } + + if let Some(downtime) = new_estimated_downtime { + self.downtime_estimated_ms = u64::try_from(downtime.as_millis()).unwrap(); + } else { + // This is better than showing `0` and it is likely close to the final actual downtime. + self.downtime_estimated_ms = self.downtime_configured_ms; + } + + match &self.state { + MigrationState::Ongoing { + phase: _old_phase, + vcpu_throttle_percent: old_vcpu_throttle_percent, + } => { + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = + self.timestamp_snapshot_ms - self.timestamp_begin_ms; + + self.memory_transmission_info = + new_memory_transmission_info.unwrap_or(self.memory_transmission_info); + self.state = MigrationState::Ongoing { + phase: new_phase, + vcpu_throttle_percent: new_cpu_throttle_percent + .unwrap_or(*old_vcpu_throttle_percent), + }; + } + illegal => { + // panic is fine as we have a logic error here, nothing that was caused by a user. + panic!( + "illegal state transition: {} -> ongoing", + illegal.state_name(), + ); + } + } + } + + /// Sets the underlying state to [`MigrationState::Cancelled`] and + /// updates all corresponding metadata. + /// + /// After this state change, the object is supposed to be handled as immutable. + /// + /// # Panics + /// + /// If the current state is not [`MigrationState::Ongoing`], this function panics. + pub fn mark_as_cancelled(&mut self) { + if !matches!(self.state, MigrationState::Ongoing { .. }) { + panic!( + "illegal state transition: {} -> cancelled", + self.state.state_name() + ); + } + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; + self.state = MigrationState::Cancelled {}; + } + + /// Sets the underlying state to [`MigrationState::Failed`] and + /// updates all corresponding metadata. + /// + /// After this state change, the object is supposed to be handled as immutable. + /// + /// # Panics + /// + /// If the current state is not [`MigrationState::Ongoing`], this function panics. + pub fn mark_as_failed(&mut self, error: &dyn Error) { + if !matches!(self.state, MigrationState::Ongoing { .. }) { + panic!( + "illegal state transition: {} -> failed", + self.state.state_name() + ); + } + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; + self.state = MigrationState::Failed { + error_msg: format!("{error}",), + error_msg_debug: format!("{error:?}",), + }; + } + + /// Sets the underlying state to [`MigrationState::Finished`] and + /// updates all corresponding metadata. + /// + /// After this state change, the object is supposed to be handled as immutable. + /// + /// # Panics + /// + /// If the current state is not [`MigrationState::Ongoing`], this function panics. + pub fn mark_as_finished(&mut self) { + if !matches!(self.state, MigrationState::Ongoing { .. }) { + panic!( + "illegal state transition: {} -> finished", + self.state.state_name() + ); + } + self.timestamp_snapshot_ms = current_unix_timestamp_ms(); + self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; + self.state = MigrationState::Finished {}; + } +} + +#[cfg(test)] +mod unit_tests { + use std::thread; + + use super::*; + + fn tcp_mode() -> TransportationMode { + TransportationMode::Tcp { + connections: NonZeroU32::new(2).unwrap(), + tls: true, + } + } + + #[test] + fn new_initializes_valid_state() { + let target = Duration::from_millis(150); + let progress = MigrationProgress::new(tcp_mode(), target); + + assert_eq!(progress.timestamp_snapshot_ms, progress.timestamp_begin_ms); + assert_eq!(progress.timestamp_snapshot_relative_ms, 0); + assert_eq!(progress.downtime_configured_ms, 150); + assert_eq!(progress.downtime_estimated_ms, 0); + + match progress.state { + MigrationState::Ongoing { + phase, + vcpu_throttle_percent, + } => { + assert_eq!(phase, MigrationStateOngoingPhase::Starting); + assert_eq!(vcpu_throttle_percent, 0); + } + _ => panic!("expected Ongoing state"), + } + + assert_eq!( + progress.memory_transmission_info, + MemoryTransmissionInfo::default() + ); + } + + #[test] + fn update_changes_phase_and_preserves_previous_values() { + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(200)); + + let initial_timestamp = progress.timestamp_snapshot_ms; + + thread::sleep(Duration::from_millis(1)); + + progress.update(MigrationStateOngoingPhase::MemoryPrecopy, None, None, None); + + match progress.state { + MigrationState::Ongoing { + phase, + vcpu_throttle_percent, + } => { + assert_eq!(phase, MigrationStateOngoingPhase::MemoryPrecopy); + assert_eq!(vcpu_throttle_percent, 0); // unchanged + } + _ => panic!("expected Ongoing"), + } + + assert!(progress.timestamp_snapshot_ms >= initial_timestamp); + assert!(progress.timestamp_snapshot_relative_ms > 0); + + // If no estimated downtime provided, fallback to configured value + assert_eq!( + progress.downtime_estimated_ms, + progress.downtime_configured_ms + ); + } + + #[test] + fn update_replaces_memory_info_and_throttle() { + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(100)); + + let mem = MemoryTransmissionInfo { + memory_iteration: 3, + memory_transmission_bps: 10_000, + memory_bytes_total: 1_000_000, + memory_bytes_transmitted: 400_000, + memory_bytes_remaining_iteration: 100_000, + memory_pages_4k_transmitted: 100, + memory_pages_4k_remaining_iteration: 25, + memory_pages_constant_count: 10, + memory_dirty_rate_pps: 500, + }; + + progress.update( + MigrationStateOngoingPhase::MemoryPrecopy, + Some(mem), + Some(42), + Some(Duration::from_millis(55)), + ); + + assert_eq!(progress.memory_transmission_info, mem); + assert_eq!(progress.downtime_estimated_ms, 55); + + match progress.state { + MigrationState::Ongoing { + phase, + vcpu_throttle_percent, + } => { + assert_eq!(phase, MigrationStateOngoingPhase::MemoryPrecopy); + assert_eq!(vcpu_throttle_percent, 42); + } + _ => panic!("expected Ongoing"), + } + } + + #[test] + #[should_panic] + fn update_panics_if_not_ongoing() { + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); + progress.mark_as_finished(); + + progress.update(MigrationStateOngoingPhase::Completing, None, None, None); + } + + #[test] + #[should_panic] + fn throttle_above_100_panics() { + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); + + progress.update( + MigrationStateOngoingPhase::MemoryPrecopy, + None, + Some(101), + None, + ); + } + + #[test] + fn mark_as_finished_transitions_state() { + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); + + thread::sleep(Duration::from_millis(1)); + progress.mark_as_finished(); + + match progress.state { + MigrationState::Finished {} => {} + _ => panic!("expected Finished"), + } + + assert!(progress.timestamp_snapshot_relative_ms > 0); + } + + #[test] + #[should_panic] + fn mark_as_finished_twice_panics() { + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); + + progress.mark_as_finished(); + progress.mark_as_finished(); + } + + #[test] + fn mark_as_failed_sets_error_strings() { + #[derive(Debug)] + struct TestError; + + impl fmt::Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "test error") + } + } + + impl Error for TestError {} + + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); + + progress.mark_as_failed(&TestError); + + match &progress.state { + MigrationState::Failed { + error_msg, + error_msg_debug, + } => { + assert_eq!(error_msg, "test error"); + assert!(error_msg_debug.contains("TestError")); + } + _ => panic!("expected Failed"), + } + } + + #[test] + fn display_formats_are_stable() { + let mut progress = + MigrationProgress::new(TransportationMode::Local, Duration::from_millis(10)); + + progress.update( + MigrationStateOngoingPhase::MemoryPrecopy, + None, + Some(12), + None, + ); + + let s = format!("{}", progress.state); + assert!(s.contains("ongoing")); + assert!(s.contains("phase=memory (precopy)")); + assert!(s.contains("vcpu_throttle=12")); + + progress.mark_as_cancelled(); + assert_eq!(format!("{}", progress.state), "cancelled"); + } +} From d4e846f9043bb66d3fcd6f5b2856cb725b504101 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 13 Jan 2026 11:27:01 +0100 Subject: [PATCH 49/79] vmm: add migration-progress API endpoint This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. We decided to use an Option<> rather than a Result<> as there isn't really an error that can happen when we query this endpoint. A previous snapshot may either be there or not. It also doesn't make sense here to check if the current VM is running, as users should always be able to query information about the past (failed or canceled) live migration. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- fuzz/Cargo.lock | 1 + fuzz/fuzz_targets/http_api.rs | 5 ++++ vmm/src/api/mod.rs | 51 +++++++++++++++++++++++++++++++++++ vmm/src/lib.rs | 5 ++++ 4 files changed, 62 insertions(+) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 52f5d4248f..58ef2c4d99 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1601,6 +1601,7 @@ version = "0.1.0" dependencies = [ "arch", "libc", + "thiserror", "vm-memory", ] diff --git a/fuzz/fuzz_targets/http_api.rs b/fuzz/fuzz_targets/http_api.rs index d6dfbf2bf9..0b6c03c88f 100644 --- a/fuzz/fuzz_targets/http_api.rs +++ b/fuzz/fuzz_targets/http_api.rs @@ -11,6 +11,7 @@ use std::thread; use libfuzzer_sys::{fuzz_target, Corpus}; use micro_http::Request; +use vm_migration::progress::MigrationProgress; use vm_migration::MigratableError; use vmm::api::http::*; use vmm::api::{ @@ -308,6 +309,10 @@ impl RequestHandler for StubApiRequestHandler { fn vm_post_migration_announce(&mut self) -> Result<(), VmError> { Ok(()) } + + fn vm_migration_progress(&mut self) -> Option { + None + } } fn http_receiver_stub(exit_evt: EventFd, api_evt: EventFd, api_receiver: Receiver) { diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 844e961035..b7c8e448c1 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -46,6 +46,7 @@ use option_parser::{OptionParser, OptionParserError, Toggle}; use serde::{Deserialize, Serialize}; use thiserror::Error; use vm_migration::MigratableError; +use vm_migration::progress::MigrationProgress; use vmm_sys_util::eventfd::EventFd; #[cfg(feature = "dbus_api")] @@ -215,6 +216,10 @@ pub enum ApiError { /// Error triggering NMI #[error("Error triggering NMI")] VmNmi(#[source] VmError), + + /// Error fetching the migration progress + #[error("Error fetching the migration progress")] + VmMigrationProgress(#[source] VmError), } pub type ApiResult = Result; @@ -681,6 +686,9 @@ pub enum ApiResponsePayload { /// Virtual machine information VmInfo(VmInfoResponse), + /// The progress of a possibly ongoing live migration. + VmMigrationProgress(Box>), + /// Vmm ping response VmmPing(VmmPingResponse), @@ -773,6 +781,10 @@ pub trait RequestHandler { ) -> Result<(), MigratableError>; fn vm_nmi(&mut self) -> Result<(), VmError>; + + /// Returns the progress of the currently active migration or any previous + /// failed or canceled migration. + fn vm_migration_progress(&mut self) -> Option; } /// It would be nice if we could pass around an object like this: @@ -1976,6 +1988,45 @@ impl ApiAction for VmNmi { } } +pub struct VmMigrationProgress; + +impl ApiAction for VmMigrationProgress { + type RequestBody = (); + type ResponseBody = Box>; + + fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { + Box::new(move |vmm| { + info!("API request event: VmMigrationProgress"); + + let snapshot = Ok(vmm.vm_migration_progress()); + let response = snapshot + .map(Box::new) + .map(ApiResponsePayload::VmMigrationProgress) + .map_err(ApiError::VmMigrationProgress); + + response_sender + .send(response) + .map_err(VmmError::ApiResponseSend)?; + + Ok(false) + }) + } + + fn send( + &self, + api_evt: EventFd, + api_sender: Sender, + data: Self::RequestBody, + ) -> ApiResult { + let info = get_response(self, api_evt, api_sender, data)?; + + match info { + ApiResponsePayload::VmMigrationProgress(info) => Ok(info), + _ => Err(ApiError::ResponsePayloadType), + } + } +} + #[cfg(test)] mod unit_tests { use super::*; diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 322bc61420..0026abc949 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -47,6 +47,7 @@ use thiserror::Error; use tracer::trace_scoped; use vm_memory::GuestMemoryAtomic; use vm_memory::bitmap::AtomicBitmap; +use vm_migration::progress::MigrationProgress; use vm_migration::protocol::*; use vm_migration::{ MemoryMigrationContext, Migratable, MigratableError, OngoingMigrationContext, Pausable, @@ -3073,6 +3074,10 @@ impl RequestHandler for Vmm { } } } + + fn vm_migration_progress(&mut self) -> Option { + None + } } const CPU_MANAGER_SNAPSHOT_ID: &str = "cpu-manager"; From fbd0dd67068996632b527c3f66222d2e5512d60c Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 12 Jan 2026 17:39:49 +0100 Subject: [PATCH 50/79] vmm: add migration-progress HTTP endpoint This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. In this commit, we add the HTTP endpoint to export ongoing VM live-migration progress. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 32 ++++++++++++++++++++++++++++--- vmm/src/api/http/mod.rs | 12 ++++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index ead1a9de5e..7cac9f5b2a 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -47,9 +47,9 @@ use crate::api::http::{EndpointHandler, HttpError, error_response}; use crate::api::{ AddDisk, ApiAction, ApiError, ApiRequest, NetConfig, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, - VmConfig, VmCounters, VmDelete, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, - VmReboot, VmReceiveMigration, VmReceiveMigrationData, VmRemoveDevice, VmResize, VmResizeDisk, - VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, + VmConfig, VmCounters, VmDelete, VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, + VmPowerButton, VmReboot, VmReceiveMigration, VmReceiveMigrationData, VmRemoveDevice, VmResize, + VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, }; use crate::config::RestoreConfig; use crate::cpu::Error as CpuError; @@ -660,6 +660,32 @@ impl EndpointHandler for VmmShutdown { } } +impl EndpointHandler for VmMigrationProgress { + fn handle_request( + &self, + req: &Request, + api_notifier: EventFd, + api_sender: Sender, + ) -> Response { + match req.method() { + Method::Get => match crate::api::VmMigrationProgress + .send(api_notifier, api_sender, ()) + .map_err(HttpError::ApiError) + { + Ok(info) => { + let mut response = Response::new(Version::Http11, StatusCode::OK); + let info_serialized = serde_json::to_string(&info).unwrap(); + + response.set_body(Body::new(info_serialized)); + response + } + Err(e) => error_response(e, StatusCode::InternalServerError), + }, + _ => error_response(HttpError::BadRequest, StatusCode::BadRequest), + } + } +} + #[cfg(test)] mod external_fds_tests { use super::*; diff --git a/vmm/src/api/http/mod.rs b/vmm/src/api/http/mod.rs index 7351406f1a..5ac3b35672 100644 --- a/vmm/src/api/http/mod.rs +++ b/vmm/src/api/http/mod.rs @@ -29,10 +29,10 @@ use self::http_endpoint::{VmActionHandler, VmCreate, VmInfo, VmmPing, VmmShutdow use crate::api::VmCoredump; use crate::api::{ AddDisk, ApiError, ApiRequest, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, - VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, VmNmi, - VmPause, VmPostMigrationAnnounce, VmPowerButton, VmReboot, VmReceiveMigration, VmRemoveDevice, - VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, - VmSnapshot, + VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, + VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, VmReboot, + VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, + VmSendMigration, VmShutdown, VmSnapshot, }; use crate::landlock::Landlock; use crate::seccomp_filters::{Thread, get_seccomp_filter}; @@ -286,6 +286,10 @@ pub static HTTP_ROUTES: LazyLock = LazyLock::new(|| { endpoint!("/vm.shutdown"), Box::new(VmActionHandler::new(&VmShutdown)), ); + r.routes.insert( + endpoint!("/vm.migration-progress"), + Box::new(VmMigrationProgress {}), + ); r.routes.insert( endpoint!("/vm.snapshot"), Box::new(VmActionHandler::new(&VmSnapshot)), From 1a16722ea9bfefecd566641feb99002faaba824f Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 22 Jan 2026 13:13:33 +0100 Subject: [PATCH 51/79] vmm: actually populate migration progress This is part of the commit series to enable live updates about an ongoing live migration. See the first commit for an introduction. This commit actually brings all the functionality together. The first version has the limitation that we populate the latest snapshot once per memory iteration, although this is the most interesting part by far. In a follow-up, we can make this more fine-grained. We guarantee that as soon as SendMigration returns, migration progress can be fetched as the underlying data source is populated. [ Adapted to the upstream migration worker: the snapshot is initialized next to the VM info snapshot in vm_send_migration() and the failure is marked next to upstream's try_resume_vm_after_failed_migration(). The fork's own worker spawn code, which this commit also touched, is gone; instead the snapshot is marked as failed when spawning the upstream worker fails, which would otherwise leave it ongoing forever. For the same reason the snapshot is installed only after the last fallible step before the hand-over, the eventfd clone. ] On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/context.rs | 4 +- vmm/src/lib.rs | 118 +++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/vm-migration/src/context.rs b/vm-migration/src/context.rs index 21801c0290..8e4d28c4f0 100644 --- a/vm-migration/src/context.rs +++ b/vm-migration/src/context.rs @@ -225,13 +225,13 @@ pub struct MemoryMigrationContext { /// Current iteration: 0 initial total transmission, >0 delta transmission. pub iteration: usize, /// Total bytes sent across all iterations. - total_sent_bytes: u64, + pub total_sent_bytes: u64, /// Total bytes to send in the current iteration. pub current_iteration_total_bytes: u64, /// The currently measured bandwidth. /// /// This is updated (at least) after each completed iteration. - bandwidth_bytes_per_second: f64, + pub bandwidth_bytes_per_second: f64, /// Calculated downtime in milliseconds regarding the current bandwidth and /// the remaining memory. /// diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 0026abc949..703104a702 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -30,6 +30,7 @@ use anyhow::{Context, anyhow}; #[cfg(feature = "dbus_api")] use api::dbus::{DBusApiOptions, DBusApiShutdownChannels}; use api::http::HttpApiHandle; +use arch::PAGE_SIZE; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] use arch::x86_64::MAX_SUPPORTED_CPUS_LEGACY; use console_devices::{ConsoleInfo, pre_create_console_devices}; @@ -47,7 +48,10 @@ use thiserror::Error; use tracer::trace_scoped; use vm_memory::GuestMemoryAtomic; use vm_memory::bitmap::AtomicBitmap; -use vm_migration::progress::MigrationProgress; +use vm_migration::progress::{ + MemoryTransmissionInfo, MigrationProgress, MigrationState, MigrationStateOngoingPhase, + TransportationMode, +}; use vm_migration::protocol::*; use vm_migration::{ MemoryMigrationContext, Migratable, MigratableError, OngoingMigrationContext, Pausable, @@ -297,6 +301,9 @@ impl From for EpollDispatch { } } +// TODO make this a member of Vmm? +static MIGRATION_PROGRESS_SNAPSHOT: Mutex> = Mutex::new(None); + pub struct EpollContext { epoll_file: File, } @@ -1360,6 +1367,36 @@ impl Vmm { is_converged: impl Fn(&MemoryMigrationContext) -> result::Result, mem_send: &mut SendAdditionalConnections, ) -> result::Result { + let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .update( + MigrationStateOngoingPhase::MemoryPrecopy, + Some(MemoryTransmissionInfo { + memory_iteration: s.iteration as u64, + memory_transmission_bps: s.current_iteration_total_bytes, + memory_bytes_total: s.bandwidth_bytes_per_second as u64, + memory_bytes_transmitted: s.total_sent_bytes, + memory_pages_4k_transmitted: s.total_sent_bytes.div_ceil(PAGE_SIZE as u64), + memory_pages_4k_remaining_iteration: s + .current_iteration_total_bytes + .div_ceil(PAGE_SIZE as u64), + memory_bytes_remaining_iteration: s.current_iteration_total_bytes, + memory_dirty_rate_pps: { + let pages = s.current_iteration_total_bytes.div_ceil(PAGE_SIZE as u64); + s.iteration_duration + .filter(|d| !d.is_zero()) + .map(|d| (pages as f64 / d.as_secs_f64()).ceil()) + .map_or(0, |dirty_rate| dirty_rate as u64) + }, + memory_pages_constant_count: 0, /* TODO */ + }), + Some(vm.throttle_percent()), + s.estimated_downtime, + ); + }; + loop { // todo: check if auto-converge is enabled at all? if Self::can_increase_autoconverge_step(ctx) @@ -1384,11 +1421,16 @@ impl Vmm { }; ctx.update_metrics_before_transfer(iteration_begin, &iteration_table); + // Update before we might exit the loop. + update_migration_progress(ctx, vm); if is_converged(ctx)? { info!("Precopy converged: {ctx}"); break Ok(iteration_table); } + // Update with new metrics before transmission. + update_migration_progress(ctx, vm); + // Send the current dirty pages let transfer_begin = Instant::now(); mem_send.send_memory(iteration_table, socket)?; @@ -1639,6 +1681,11 @@ impl Vmm { if send_data_migration.local { match &mut socket { SocketStream::Unix(unix_socket) => { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .update(MigrationStateOngoingPhase::MemoryFds, None, None, None); + // Proceed with sending memory file descriptors over UNIX socket vm.send_memory_fds(unix_socket)?; } @@ -1699,6 +1746,14 @@ impl Vmm { mem_send.cleanup()?; } + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .update(MigrationStateOngoingPhase::Completing, None, None, None); + } + // We release the locks early to enable locking them on the destination host. // The VM is already stopped. vm.release_disk_locks() @@ -1751,6 +1806,14 @@ impl Vmm { vm.stop_dirty_log()?; } + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_finished(); + } + // Let every Migratable object know about the migration being complete vm.complete_migration() } @@ -1939,6 +2002,14 @@ impl Vmm { Err(e) => { error!("Migration failed: {e}"); try_resume_vm_after_failed_migration(vm); + + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_failed(&e); + } } } } @@ -3046,6 +3117,34 @@ impl RequestHandler for Vmm { .with_context(|| "Failed to clone check_migration_evt FD") .map_err(MigratableError::MigrateSend)?; + // Update migration progress snapshot early: + // We guarantee that migration statistics can be fetched as soon as SendMigration returns. + // + // If the migration fails, the state will later be updated accordingly. + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + if lock + .as_ref() + .map(|p| &p.state) + .is_some_and(|snapshot| matches!(snapshot, MigrationState::Ongoing { .. })) + { + // If this panic triggers, we made a programming error in our state handling. + panic!("migration already ongoing"); + } + let transportation_mode = if send_data_migration.local { + TransportationMode::Local + } else { + TransportationMode::Tcp { + connections: send_data_migration.connections, + tls: send_data_migration.tls_dir.is_some(), + } + }; + lock.replace(MigrationProgress::new( + transportation_mode, + send_data_migration.downtime(), + )); + } + // Take VM ownership. This also means that API events can no longer // change the VM (e.g. net device hotplug). let vm = self @@ -3070,13 +3169,26 @@ impl RequestHandler for Vmm { } Err(e) => { self.vm = VmOwnership::Owned(e.vm); - Err(MigratableError::MigrateSend(e.spawn_error.into())) + let error = MigratableError::MigrateSend(e.spawn_error.into()); + + // The snapshot was already marked as ongoing above. + MIGRATION_PROGRESS_SNAPSHOT + .lock() + .unwrap() + .as_mut() + .expect("live migration should be ongoing") + .mark_as_failed(&error); + + Err(error) } } } fn vm_migration_progress(&mut self) -> Option { - None + // We explicitly do not check here for `is VM running?` to always + // enable querying the state of the last failed migration. + let lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.clone() } } From 1ec0d1573138e531e73f309dad765ddaaacf2bd8 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 12 Feb 2026 09:44:46 +0100 Subject: [PATCH 52/79] ch-remote: add `migration-progress` command On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/src/bin/ch-remote.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index f9675e3946..4a64c621cd 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -327,6 +327,8 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu Some("shutdown") => { simple_api_command(socket, "PUT", "shutdown", None).map_err(Error::HttpApiClient) } + Some("migration-progress") => simple_api_command(socket, "GET", "migration-progress", None) + .map_err(Error::HttpApiClient), Some("nmi") => simple_api_command(socket, "PUT", "nmi", None).map_err(Error::HttpApiClient), Some("resize") => { let resize = resize_config( @@ -1070,6 +1072,7 @@ fn get_cli_commands_sorted() -> Box<[Command]> { .arg(Arg::new("path").index(1).default_value("-")), Command::new("delete").about("Delete a VM"), Command::new("info").about("Info on the VM"), + Command::new("migration-progress"), Command::new("nmi").about("Trigger NMI"), Command::new("pause").about("Pause the VM"), Command::new("ping").about("Ping the VMM to check for API server availability"), From af24c95db3a65df6d6c1553ac8edc162228383af Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:30:44 +0200 Subject: [PATCH 53/79] vmm: keep the VMM alive after a migration if requested This restores the fork-only `keep_alive` option of the send-migration API call, which was dropped with the revert of commit 06a8b76a701734eb863d8930f8645c34fdbfd1b4 ("vmm: migration: switch to non-blocking SendMigration call"). The rest of that commit is obsolete: upstream's migration worker already dispatches the call without blocking. With `keep_alive=on` the VMM stays alive after a successful migration so that management software can fetch the final migration progress. It is then supposed to send a ShutdownVmm command. The migration progress is now marked as finished once the VMM collects the worker result, which is also where the option is evaluated. For that, the worker hands the migration configuration back to the VMM thread. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 15 +++++++++++++-- vmm/src/lib.rs | 22 +++++++++++++--------- vmm/src/migration/worker.rs | 3 +++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index b7c8e448c1..3f066bb93a 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -498,6 +498,8 @@ pub struct VmSendMigrationData { /// Path to the directory containing the TLS root CA certificate (ca-cert.pem), the TLS client certificate (client-cert.pem), and TLS client key (client-key.pem). #[serde(default)] pub tls_dir: Option, + /// Keep the VMM alive. + pub keep_alive: bool, } impl VmSendMigrationData { @@ -534,7 +536,8 @@ impl VmSendMigrationData { .add("timeout_s") .add("timeout_strategy") .add("connections") - .add("tls_dir"); + .add("tls_dir") + .add("keep_alive"); parser .parse(migration) .map_err(VmSendMigrationConfigError::ParseError)?; @@ -590,6 +593,11 @@ impl VmSendMigrationData { .convert::("tls_dir") .map_err(VmSendMigrationConfigError::ParseError)? .map(|path| PathBuf::from(&path)); + let keep_alive = parser + .convert::("keep_alive") + .map_err(VmSendMigrationConfigError::ParseError)? + .unwrap_or(Toggle(false)) + .0; let data = Self { destination_url, @@ -599,6 +607,7 @@ impl VmSendMigrationData { timeout_strategy, connections, tls_dir, + keep_alive, }; data.validate()?; @@ -2219,13 +2228,14 @@ mod unit_tests { timeout_strategy: Default::default(), connections: VmSendMigrationData::default_connections(), tls_dir: None, + keep_alive: false, } ); // Happy path, fully specified let tls_dir = std::env::temp_dir(); let data = - VmSendMigrationData::parse(&format!("destination_url=tcp:192.168.1.1:8080,downtime_ms=150,timeout_s=900,timeout_strategy=ignore,connections=4,tls_dir={}", tls_dir.display())) + VmSendMigrationData::parse(&format!("destination_url=tcp:192.168.1.1:8080,downtime_ms=150,timeout_s=900,timeout_strategy=ignore,connections=4,tls_dir={},keep_alive=true", tls_dir.display())) .unwrap(); assert_eq!( data, @@ -2237,6 +2247,7 @@ mod unit_tests { timeout_strategy: TimeoutStrategy::Ignore, connections: NonZeroU32::new(4).unwrap(), tls_dir: Some(tls_dir), + keep_alive: true } ); } diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 703104a702..8f7b10adb5 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1806,14 +1806,6 @@ impl Vmm { vm.stop_dirty_log()?; } - // Update migration progress snapshot - { - let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); - lock.as_mut() - .expect("live migration should be ongoing") - .mark_as_finished(); - } - // Let every Migratable object know about the migration being complete vm.complete_migration() } @@ -1960,6 +1952,7 @@ impl Vmm { vm, migration_result: migration_res, initial_vm_state, + config: migration_cfg, } = migration_worker_handle.join(); let mut try_resume_vm_after_failed_migration = |mut vm: Vm| { @@ -1995,7 +1988,18 @@ impl Vmm { error!("Failed shutting down the VM after migration: {e}"); } - if let Err(e) = self.exit_evt.write(1) { + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_finished(); + } + + if migration_cfg.keep_alive { + // API users can still query live-migration statistics + info!("Keeping VMM alive as requested"); + } else if let Err(e) = self.exit_evt.write(1) { error!("Failed exiting the VMM after migration: {e}"); } } diff --git a/vmm/src/migration/worker.rs b/vmm/src/migration/worker.rs index 986615aeed..8a224c7d78 100644 --- a/vmm/src/migration/worker.rs +++ b/vmm/src/migration/worker.rs @@ -101,6 +101,7 @@ impl MigrationWorker { vm, migration_result: res, initial_vm_state: self.initial_vm_state, + config: self.config, } } @@ -159,4 +160,6 @@ pub struct MigrationWorkerResult { /// The result of [`Vmm::send_migration`]. pub migration_result: Result<(), MigratableError>, pub initial_vm_state: VmState, + /// The configuration the migration was started with. + pub config: VmSendMigrationData, } From 722cce5cd7b99dbbef0503fd34a9b1ea81919f7e Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 12 Feb 2026 09:43:17 +0100 Subject: [PATCH 54/79] ch-remote: wait for migration to finish by querying migration progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We preserve the old behavior in ch-remote: SendMigration is blocking. A new ´--dispatch` flag however ensures that one can just dispatch the migration without waiting for it to finish (or fail). On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- Cargo.lock | 1 + cloud-hypervisor/Cargo.toml | 1 + cloud-hypervisor/src/bin/ch-remote.rs | 83 +++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 201acf7fbe..e5689e0e2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -508,6 +508,7 @@ dependencies = [ "tpm", "tracer", "vm-memory", + "vm-migration", "vmm", "vmm-sys-util", "wait-timeout", diff --git a/cloud-hypervisor/Cargo.toml b/cloud-hypervisor/Cargo.toml index 352a53b1bc..73a74be6f4 100644 --- a/cloud-hypervisor/Cargo.toml +++ b/cloud-hypervisor/Cargo.toml @@ -30,6 +30,7 @@ thiserror = { workspace = true } tpm = { path = "../tpm" } tracer = { path = "../tracer" } vm-memory = { workspace = true } +vm-migration = { path = "../vm-migration" } vmm = { path = "../vmm" } vmm-sys-util = { workspace = true } zbus = { version = "5.15.0", optional = true } diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index 4a64c621cd..f8075de113 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -11,17 +11,20 @@ use std::io::Read; use std::marker::PhantomData; use std::os::unix::net::UnixStream; use std::process; +use std::thread::sleep; +use std::time::Duration; use api_client::{ - Error as ApiClientError, simple_api_command, simple_api_command_with_fds, - simple_api_full_command, + Error as ApiClientError, StatusCode, simple_api_command, simple_api_command_with_fds, + simple_api_full_command, simple_api_full_command_and_response, }; #[cfg(feature = "dbus_api")] use clap::ArgAction; use clap::{Arg, ArgMatches, Command}; -use log::error; +use log::{error, info}; use option_parser::{ByteSized, ByteSizedParseError}; use thiserror::Error; +use vm_migration::progress::{MigrationProgress, MigrationState}; use vmm::config::RestoreConfig; use vmm::vm_config::{ DeviceConfig, DiskConfig, FsConfig, GenericVhostUserConfig, NetConfig, PmemConfig, @@ -531,6 +534,14 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu .map_err(Error::HttpApiClient) } Some("send-migration") => { + let just_dispatch = matches + .subcommand_matches("send-migration") + .unwrap() + .get_one::("dispatch") + .cloned() + .unwrap_or(false); + let wait_for_migration = !just_dispatch; + let send_migration_data = send_migration_data( matches .subcommand_matches("send-migration") @@ -539,7 +550,65 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu .unwrap(), )?; simple_api_command(socket, "PUT", "send-migration", Some(&send_migration_data)) - .map_err(Error::HttpApiClient) + .map_err(Error::HttpApiClient)?; + + if !wait_for_migration { + return Ok(()); + } + loop { + let response = simple_api_full_command_and_response( + socket, + "GET", + "vm.migration-progress", + None, + ) + .map_err(Error::HttpApiClient)? + // should have response + .ok_or(Error::HttpApiClient(ApiClientError::ServerResponse( + StatusCode::Ok, + None, + )))?; + + // This is guaranteed by the SendMigration call + assert_ne!( + response, "null", + "migration progress should be there immediately when the migration was dispatched" + ); + + let progress = serde_json::from_slice::(response.as_bytes()) + .map_err(|e| { + error!("failed to parse response as MigrationProgress: {e}"); + Error::HttpApiClient(ApiClientError::ServerResponse( + StatusCode::Ok, + Some(response), + )) + })?; + + match progress.state { + MigrationState::Cancelled { .. } => { + info!("Migration was cancelled"); + break; + } + MigrationState::Failed { + error_msg, + error_msg_debug, + } => { + error!("Migration failed! {error_msg}\n{error_msg_debug}"); + break; + } + MigrationState::Finished { .. } => { + info!("Migration finished successfully. Shutting down Cloud Hypervisor"); + simple_api_full_command(socket, "PUT", "vmm.shutdown", None) + .map_err(Error::HttpApiClient)?; + break; + } + MigrationState::Ongoing { .. } => { + sleep(Duration::from_millis(50)); + continue; + } + } + } + Ok(()) } Some("receive-migration") => { let receive_migration_data = receive_migration_data( @@ -1147,6 +1216,12 @@ fn get_cli_commands_sorted() -> Box<[Command]> { Command::new("resume").about("Resume the VM"), Command::new("send-migration") .about("Initiate a VM migration") + .arg( + Arg::new("dispatch") + .long("dispatch") + .help("just dispatch the migration without waiting for it to finish") + .num_args(0), + ) .arg( Arg::new("send_migration_config") .index(1) From 8c9b013c61da3700ba7fdfed383bdd4cfbbe577d Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 22 Sep 2026 18:31:35 +0200 Subject: [PATCH 55/79] tests: dispatch the migration in the TCP timeout test `ch-remote send-migration` waits for the migration to finish again, so its exit code no longer proves that the migration was dispatched. Pass `--dispatch` to keep the dispatch semantics the test asserts on and to keep the event-monitor assertions meaningful. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/tests/integration.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud-hypervisor/tests/integration.rs b/cloud-hypervisor/tests/integration.rs index e9c4066196..fb2ffe2e52 100644 --- a/cloud-hypervisor/tests/integration.rs +++ b/cloud-hypervisor/tests/integration.rs @@ -6734,6 +6734,7 @@ mod common_parallel { .args([ &format!("--api-socket={src_api_socket}"), "send-migration", + "--dispatch", &format!( "destination_url=tcp:{host_ip}:{migration_port},downtime_ms=1,timeout_s=1,timeout_strategy={timeout_strategy:?}" ), From aa67073358fec526d1f1ce8980eb7edf6bf3dd99 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 19 Feb 2026 11:26:42 +0100 Subject: [PATCH 56/79] vmm: api: less verbose log These events happen fairly often now and are very spammy in the log. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 3f066bb93a..dfeb184025 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -40,7 +40,7 @@ use std::str::FromStr; use std::sync::mpsc::{RecvError, SendError, Sender, channel}; use std::time::Duration; -use log::info; +use log::{debug, info}; use micro_http::Body; use option_parser::{OptionParser, OptionParserError, Toggle}; use serde::{Deserialize, Serialize}; @@ -1377,7 +1377,7 @@ impl ApiAction for VmInfo { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - info!("API request event: VmInfo"); + debug!("API request event: VmInfo"); let response = vmm .vm_info() @@ -2005,7 +2005,7 @@ impl ApiAction for VmMigrationProgress { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - info!("API request event: VmMigrationProgress"); + debug!("API request event: VmMigrationProgress"); let snapshot = Ok(vmm.vm_migration_progress()); let response = snapshot From 82fc5475ebbdf637560bc3001f3406592fbd3dc6 Mon Sep 17 00:00:00 2001 From: Leander Kohler Date: Tue, 17 Feb 2026 14:14:27 +0100 Subject: [PATCH 57/79] vmm: add post-migration event to VmSnapshot During live migration, VM ownership is moved away from the VMM thread. To preserve guest-triggered reboot and shutdown lifecycle intent across that ownership handover, we need a small lifecycle marker to travel with the migrated VM state. This change introduces `PostMigrationLifecycleEvent` and stores it in `VmSnapshot` with `#[serde(default)]` for backward compatibility. `Vm::snapshot()` now serializes the marker, and VM construction from a snapshot restores it. No control-loop behavior is changed in this commit. This is only the data model/plumbing needed by follow-up commits. On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler Signed-off-by: Philipp Schuster --- vmm/src/vm.rs | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 0725c63268..18f9200755 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -108,11 +108,9 @@ use crate::landlock::LandlockError; use crate::memory_manager::{ Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData, }; -#[cfg(target_arch = "x86_64")] -use crate::migration::get_vm_snapshot; #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] use crate::migration::url_to_file; -use crate::migration::{SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE, url_to_path}; +use crate::migration::{SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE, get_vm_snapshot, url_to_path}; #[cfg(all( feature = "kvm", feature = "sev_snp", @@ -583,6 +581,13 @@ pub struct Vm { stop_on_boot: bool, load_payload_handle: Option>>, vcpu_throttler: ThrottleThreadHandle, + post_migration_lifecycle_event: Option, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum PostMigrationLifecycleEvent { + VmReboot, + VmmShutdown, } impl Vm { @@ -741,6 +746,15 @@ impl Vm { } else { VmState::Created }; + let post_migration_lifecycle_event = snapshot + .as_ref() + .map(|snapshot| { + get_vm_snapshot(snapshot) + .map(|vm_snapshot| vm_snapshot.post_migration_lifecycle_event) + .map_err(Error::Restore) + }) + .transpose()? + .flatten(); // TODO we could also spawn the thread when a migration with auto-converge starts. // Probably this is the better design. @@ -766,6 +780,7 @@ impl Vm { stop_on_boot, load_payload_handle, vcpu_throttler, + post_migration_lifecycle_event, }) } @@ -1487,6 +1502,17 @@ impl Vm { self.vcpu_throttler.shutdown(); } + pub fn set_post_migration_lifecycle_event( + &mut self, + event: Option, + ) { + self.post_migration_lifecycle_event = event; + } + + pub fn post_migration_lifecycle_event(&self) -> Option { + self.post_migration_lifecycle_event + } + #[allow(clippy::too_many_arguments)] pub fn new( vm_config: Arc>, @@ -3463,6 +3489,8 @@ impl Pausable for Vm { #[derive(Serialize, Deserialize)] pub struct VmSnapshot { + #[serde(default)] + pub post_migration_lifecycle_event: Option, #[cfg(target_arch = "x86_64")] pub clock: Option, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -3525,6 +3553,7 @@ impl Snapshottable for Vm { }; let vm_snapshot_state = VmSnapshot { + post_migration_lifecycle_event: self.post_migration_lifecycle_event(), #[cfg(target_arch = "x86_64")] clock: self.saved_clock, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] From dab1d8f6a0ab035f13ce67e02bcb10f8aab65ba1 Mon Sep 17 00:00:00 2001 From: Leander Kohler Date: Tue, 17 Feb 2026 14:42:39 +0100 Subject: [PATCH 58/79] vmm: postpone reset/exit during migration While a live migration is running, the migration worker owns the VM and the VMM control loop cannot execute vm_reboot()/vmm_shutdown() directly. Guest-triggered reset/exit events in that window currently hit VmMigrating and fail. This change makes the control loop consume reset/exit as before, but when ownership is `MaybeVmOwnership::Migration` it postpones a post-migration lifecycle intent instead of calling lifecycle handlers directly. The postponed state is first-event-wins and is cleared when a new send migration starts, preventing stale lifecycle intent from leaking between migrations. This commit only introduces source-side postponing behavior and does not yet apply or replay the postponed event. [ Re-implemented on top of upstream's migration worker: the shared Arc>> is passed through MigrationWorker::spawn() instead of the fork's worker struct, the control loop matches on VmOwnership::Migration and thereby replaces upstream's two lifecycle TODO markers, and Vmm::send_migration() takes the shared state as an additional argument. The Arc/Mutex import in the worker module loses its kvm-and-x86_64 cfg, as the shared state is unconditional. ] On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 92 ++++++++++++++++++++++++++++++++----- vmm/src/migration/worker.rs | 10 ++-- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 8f7b10adb5..7d0dd28c62 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -72,15 +72,13 @@ use crate::coredump::GuestDebuggable; use crate::cpu::IS_IN_SHUTDOWN; use crate::landlock::Landlock; use crate::memory_manager::MemoryManager; -#[cfg(all(feature = "kvm", target_arch = "x86_64"))] -use crate::migration::get_vm_snapshot; use crate::migration::transport::{ self, ReceiveAdditionalConnections, ReceiveListener, SendAdditionalConnections, SocketStream, }; use crate::migration::worker::{MigrationWorker, MigrationWorkerHandle, MigrationWorkerResult}; -use crate::migration::{recv_vm_config, recv_vm_state}; +use crate::migration::{get_vm_snapshot, recv_vm_config, recv_vm_state}; use crate::seccomp_filters::{Thread, get_seccomp_filter}; -use crate::vm::{Error as VmError, Vm, VmState}; +use crate::vm::{Error as VmError, PostMigrationLifecycleEvent, Vm, VmState}; use crate::vm_config::{ DeviceConfig, DiskConfig, FsConfig, GenericVhostUserConfig, MemoryZoneConfig, NetConfig, PmemConfig, UserDeviceConfig, VdpaConfig, VmConfig, VsockConfig, @@ -701,6 +699,12 @@ pub struct Vmm { console_info: Option, no_shutdown: bool, check_migration_evt: EventFd, + /// Lifecycle event of the guest that was postponed because a migration + /// owned the VM. Shared with the migration worker. + postponed_lifecycle_event: Arc>>, + /// Lifecycle event that was postponed on the migration source and has to be + /// applied here after the VM was received. + received_postponed_lifecycle_event: Option, } /// Just a wrapper for the data that goes into @@ -927,9 +931,24 @@ impl Vmm { console_info: None, no_shutdown, check_migration_evt, + postponed_lifecycle_event: Arc::new(Mutex::new(None)), + received_postponed_lifecycle_event: None, }) } + fn postpone_lifecycle_event_during_migration(&self, event: PostMigrationLifecycleEvent) { + let mut postponed_event = self.postponed_lifecycle_event.lock().unwrap(); + if postponed_event.is_none() { + *postponed_event = Some(event); + info!("Postponed post-migration lifecycle event: {event:?}"); + } + } + + fn clear_postponed_lifecycle_event(&self) { + let mut postponed_event = self.postponed_lifecycle_event.lock().unwrap(); + *postponed_event = None; + } + /// Try to receive a file descriptor from a socket. Returns the slot number and the file descriptor. fn vm_receive_memory_fd( socket: &mut SocketStream, @@ -1095,11 +1114,34 @@ impl Vmm { // The thread in background periodically sends multiple messages. vm.post_migration_announce(); - let (_, resume_duration) = measure_ok(|| vm.resume())?; - debug!( - "Migration (incoming): resume:{}ms", - resume_duration.as_millis() - ); + // We are on the control-loop thread handling an API request, so + // there is no concurrent access from other VMM or migration + // threads. The VM is in the Paused state , which permits both + // the Running transition (resume) and the Shutdown transition (reboot / exit) + // triggered via the eventfds below. + match self.received_postponed_lifecycle_event { + None => { + let (_, resume_duration) = measure_ok(|| vm.resume())?; + debug!( + "Migration (incoming): resume:{}ms", + resume_duration.as_millis() + ); + } + Some(PostMigrationLifecycleEvent::VmReboot) => { + self.reset_evt + .write(1) + .context("Failed writing reset eventfd after migration") + .map_err(MigratableError::MigrateReceive)?; + } + Some(PostMigrationLifecycleEvent::VmmShutdown) => { + self.exit_evt + .write(1) + .context("Failed writing exit eventfd after migration") + .map_err(MigratableError::MigrateReceive)?; + } + } + self.received_postponed_lifecycle_event = None; + // This logs the downtime without the final memory delta, so // it does not reflect the actual downtime. While we could // pass along the timestamp from when the VM was paused, @@ -1277,6 +1319,11 @@ impl Vmm { .map_err(MigratableError::MigrateReceive) })?; + let vm_snapshot = get_vm_snapshot(&snapshot) + .context("Failed extracting VM snapshot data") + .map_err(MigratableError::MigrateReceive)?; + self.received_postponed_lifecycle_event = vm_snapshot.post_migration_lifecycle_event; + let exit_evt = self .exit_evt .try_clone() @@ -1616,6 +1663,7 @@ impl Vmm { hypervisor: &dyn hypervisor::Hypervisor, send_data_migration: &VmSendMigrationData, initial_vm_state: VmState, + postponed_lifecycle_event: &Mutex>, ) -> result::Result<(), MigratableError> { // State machine that is updated with more context as we progress. let mut ctx = OngoingMigrationContext::new(); @@ -1768,6 +1816,7 @@ impl Vmm { } // Capture snapshot and send it + vm.set_post_migration_lifecycle_event(*postponed_lifecycle_event.lock().unwrap()); let (vm_snapshot, snapshot_duration) = measure_ok(|| vm.snapshot())?; let (_, send_snapshot_duration) = measure_ok(|| transport::send_state(&mut socket, &vm_snapshot))?; @@ -2057,7 +2106,14 @@ impl Vmm { info!("VM exit event"); // Consume the event. self.exit_evt.read().map_err(Error::EventFdRead)?; - // TODO: Future follow-up must resolve lifecycle handling while migrating. + // The migration worker owns the VM, so the lifecycle + // change is applied once the migration finished. + if matches!(self.vm, VmOwnership::Migration { .. }) { + self.postpone_lifecycle_event_during_migration( + PostMigrationLifecycleEvent::VmmShutdown, + ); + continue; + } self.vmm_shutdown().map_err(Error::VmmShutdown)?; break 'outer; @@ -2066,7 +2122,14 @@ impl Vmm { info!("VM reset event"); // Consume the event. self.reset_evt.read().map_err(Error::EventFdRead)?; - // TODO: Future follow-up must resolve lifecycle handling while migrating. + // The migration worker owns the VM, so the lifecycle + // change is applied once the migration finished. + if matches!(self.vm, VmOwnership::Migration { .. }) { + self.postpone_lifecycle_event_during_migration( + PostMigrationLifecycleEvent::VmReboot, + ); + continue; + } self.vm_reboot().map_err(Error::VmReboot)?; } EpollDispatch::GuestExit => { @@ -2977,6 +3040,9 @@ impl RequestHandler for Vmm { .context("Invalid receive migration configuration") .map_err(MigratableError::MigrateReceive)?; + // Prevent stale lifecycle intent from a previous failed receive attempt. + self.received_postponed_lifecycle_event = None; + info!( "Receiving migration: receiver_url={},tls={},net_fds={:?}, tcp_url={:?}, zones={:?}", receive_data_migration.receiver_url, @@ -3085,6 +3151,9 @@ impl RequestHandler for Vmm { send_data_migration.timeout_strategy ); + // New migration attempt: clear postponed lifecycle from any previous run. + self.clear_postponed_lifecycle_event(); + if !self .vm_config .as_ref() @@ -3160,6 +3229,7 @@ impl RequestHandler for Vmm { vm, check_migration_evt, send_data_migration, + self.postponed_lifecycle_event.clone(), #[cfg(all(feature = "kvm", target_arch = "x86_64"))] self.hypervisor.clone(), initial_vm_state, diff --git a/vmm/src/migration/worker.rs b/vmm/src/migration/worker.rs index 8a224c7d78..a1fb9a4148 100644 --- a/vmm/src/migration/worker.rs +++ b/vmm/src/migration/worker.rs @@ -13,9 +13,8 @@ //! [`MigrationWorkerSpawnError`]. use std::fmt::{Debug, Formatter}; -#[cfg(all(feature = "kvm", target_arch = "x86_64"))] -use std::sync::Arc; use std::sync::mpsc::Receiver; +use std::sync::{Arc, Mutex}; use std::thread; use std::thread::JoinHandle; @@ -26,7 +25,7 @@ use vmm_sys_util::eventfd::EventFd; use crate::Vmm; use crate::api::VmSendMigrationData; -use crate::vm::{Vm, VmState}; +use crate::vm::{PostMigrationLifecycleEvent, Vm, VmState}; #[derive(thiserror::Error)] #[error("Migration worker could not be spawned: {spawn_error}")] @@ -72,6 +71,8 @@ pub struct MigrationWorker { vm_receiver: Receiver, check_migration_evt: EventFd, config: VmSendMigrationData, + /// Shared with the main VMM thread. + postponed_lifecycle_event: Arc>>, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: Arc, initial_vm_state: VmState, @@ -90,6 +91,7 @@ impl MigrationWorker { self.hypervisor.as_ref(), &self.config, self.initial_vm_state, + self.postponed_lifecycle_event.as_ref(), ) .inspect(|_| event!("vm", "migration-finished")) .inspect_err(|_| event!("vm", "migration-failed")); @@ -113,6 +115,7 @@ impl MigrationWorker { vm: Vm, check_migration_evt: EventFd, config: VmSendMigrationData, + postponed_lifecycle_event: Arc>>, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: Arc< dyn hypervisor::Hypervisor, >, @@ -123,6 +126,7 @@ impl MigrationWorker { vm_receiver, check_migration_evt, config, + postponed_lifecycle_event, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor, initial_vm_state, From fbd464248fc8d7724f77dcd312914d5f34cee711 Mon Sep 17 00:00:00 2001 From: Leander Kohler Date: Tue, 17 Feb 2026 15:00:59 +0100 Subject: [PATCH 59/79] vmm: migration: switch downtime on postponed event When a lifecycle event like reset or shutdown is postponed during pre-copy, switch to downtime at the next iteration boundary. This keeps the current iteration send intact and then transitions into the existing graceful downtime path (`stop_vcpu_throttling()`, `pause()`, final transfer, snapshot). To keep behavior deterministic on source migration failure, replay the postponed lifecycle event locally after ownership is returned: - VmReboot -> reset_evt - VmmShutdown -> exit_evt Postponed state is cleared on both success and failure paths to avoid stale state across migrations. [ Adapted to upstream's migration worker: the lifecycle event is replayed inside upstream's try_resume_vm_after_failed_migration(), right after it has given the VM back to the VMM. That keeps the original behaviour of replaying on every path that recovers the VM, which later also covers cancelled migrations. ] On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 7d0dd28c62..a21291e316 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -944,6 +944,10 @@ impl Vmm { } } + fn current_postponed_lifecycle_event(&self) -> Option { + *self.postponed_lifecycle_event.lock().unwrap() + } + fn clear_postponed_lifecycle_event(&self) { let mut postponed_event = self.postponed_lifecycle_event.lock().unwrap(); *postponed_event = None; @@ -1413,6 +1417,7 @@ impl Vmm { ctx: &mut MemoryMigrationContext, is_converged: impl Fn(&MemoryMigrationContext) -> result::Result, mem_send: &mut SendAdditionalConnections, + postponed_lifecycle_event: &Mutex>, ) -> result::Result { let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -1498,6 +1503,16 @@ impl Vmm { // Increment iteration last: This way we ensure that the logging // above matches the actual iteration. ctx.iteration += 1; + + let event = *postponed_lifecycle_event.lock().unwrap(); + if let Some(event) = event { + info!( + "Lifecycle event postponed during migration ({event:?}), switching to downtime phase early" + ); + // The current iteration has already been sent, therefore no extra range + // needs to be carried into the final transfer batch. + break Ok(MemoryRangeTable::default()); + } } } @@ -1607,6 +1622,7 @@ impl Vmm { send_data_migration: &VmSendMigrationData, mem_send: &mut SendAdditionalConnections, ctx: &mut OngoingMigrationContext, + postponed_lifecycle_event: &Mutex>, ) -> result::Result<(), MigratableError> { let mut mem_ctx = MemoryMigrationContext::new(); @@ -1618,6 +1634,7 @@ impl Vmm { // We bind send_data_migration to the callback |ctx| Self::is_precopy_converged(ctx, send_data_migration), mem_send, + postponed_lifecycle_event, )?; let downtime_begin = Instant::now(); // End throttle thread @@ -1782,6 +1799,7 @@ impl Vmm { send_data_migration, &mut mem_send, &mut ctx, + postponed_lifecycle_event, ) .inspect_err(|_| { // Calling cleanup multiple times is fine, thus here we just make sure @@ -2024,6 +2042,25 @@ impl Vmm { }); self.vm = VmOwnership::Owned(vm); + + if let Some(event) = self.current_postponed_lifecycle_event() { + match event { + PostMigrationLifecycleEvent::VmReboot => { + self.reset_evt + .write(1) + .context("Failed replaying reset event after failed migration") + .inspect_err(|write_err| error!("{write_err}")) + .ok(); + } + PostMigrationLifecycleEvent::VmmShutdown => { + self.exit_evt + .write(1) + .context("Failed replaying shutdown event after failed migration") + .inspect_err(|write_err| error!("{write_err}")) + .ok(); + } + } + } }; match migration_res { @@ -2065,6 +2102,7 @@ impl Vmm { } } } + self.clear_postponed_lifecycle_event(); } fn control_loop( From 54d497a62c3f4017e91f09eddb3d28c5de52551b Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 19 Feb 2026 15:37:21 +0100 Subject: [PATCH 60/79] vmm: api: add VmCancelMigration action [ The ownership match arm is adapted to upstream's VmOwnership enum. The todo!() in vm_cancel_migration() is filled in by the next commit, as in the original series. ] On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- fuzz/fuzz_targets/http_api.rs | 4 ++++ vm-migration/src/lib.rs | 3 +++ vmm/src/api/mod.rs | 44 +++++++++++++++++++++++++++++++++++ vmm/src/lib.rs | 13 +++++++++++ 4 files changed, 64 insertions(+) diff --git a/fuzz/fuzz_targets/http_api.rs b/fuzz/fuzz_targets/http_api.rs index 0b6c03c88f..41114e5d62 100644 --- a/fuzz/fuzz_targets/http_api.rs +++ b/fuzz/fuzz_targets/http_api.rs @@ -313,6 +313,10 @@ impl RequestHandler for StubApiRequestHandler { fn vm_migration_progress(&mut self) -> Option { None } + + fn vm_cancel_migration(&mut self) -> Result<(), MigratableError> { + Ok(()) + } } fn http_receiver_stub(exit_evt: EventFd, api_evt: EventFd, api_receiver: Receiver) { diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 3b5f25987c..89cbcecf87 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -86,6 +86,9 @@ pub enum MigratableError { #[error("Failed to retrieve dirty ranges for migratable component")] DirtyLog(#[source] anyhow::Error), + #[error("Failed to cancel migration")] + CancelMigration(#[source] anyhow::Error), + #[error("Failed to start migration for migratable component")] StartMigration(#[source] anyhow::Error), diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index dfeb184025..5e71b3d7a2 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -209,6 +209,10 @@ pub enum ApiError { #[error("Error starting migration sender")] VmSendMigration(#[source] MigratableError), + /// Error cancelling migration + #[error("Error cancelling migration")] + VmCancelMigration(#[source] MigratableError), + /// Error triggering power button #[error("Error triggering power button")] VmPowerButton(#[source] VmError), @@ -784,11 +788,18 @@ pub trait RequestHandler { receive_data_migration: VmReceiveMigrationData, ) -> Result<(), MigratableError>; + /// Dispatches the migration. fn vm_send_migration( &mut self, send_data_migration: VmSendMigrationData, ) -> Result<(), MigratableError>; + /// Triggers a migration cancellation. + /// + /// The cancellation is not guaranteed to succeed, as the migration may have + /// succeeded already. + fn vm_cancel_migration(&mut self) -> Result<(), MigratableError>; + fn vm_nmi(&mut self) -> Result<(), VmError>; /// Returns the progress of the currently active migration or any previous @@ -1539,6 +1550,39 @@ impl ApiAction for VmReceiveMigration { } } +pub struct VmCancelMigration; + +impl ApiAction for VmCancelMigration { + type RequestBody = (); + type ResponseBody = Option; + + fn request(&self, data: Self::RequestBody, response_sender: Sender) -> ApiRequest { + Box::new(move |vmm| { + info!("API request event: VmCancelMigration {data:?}"); + + let response = vmm + .vm_cancel_migration() + .map_err(ApiError::VmCancelMigration) + .map(|_| ApiResponsePayload::Empty); + + response_sender + .send(response) + .map_err(VmmError::ApiResponseSend)?; + + Ok(false) + }) + } + + fn send( + &self, + api_evt: EventFd, + api_sender: Sender, + data: Self::RequestBody, + ) -> ApiResult { + get_response_body(self, api_evt, api_sender, data) + } +} + pub struct VmRemoveDevice; impl ApiAction for VmRemoveDevice { diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index a21291e316..0b77de2ab2 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -3296,6 +3296,19 @@ impl RequestHandler for Vmm { } } + fn vm_cancel_migration(&mut self) -> result::Result<(), MigratableError> { + match self.vm { + VmOwnership::Migration { .. } => (), + _ => { + return Err(MigratableError::CancelMigration(anyhow!( + "There is no ongoing migration" + ))); + } + } + + todo!() + } + fn vm_migration_progress(&mut self) -> Option { // We explicitly do not check here for `is VM running?` to always // enable querying the state of the last failed migration. From 145e2ababcc3d715a1f702c9035190b849436a22 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 19 Feb 2026 15:39:31 +0100 Subject: [PATCH 61/79] vmm: http api: add VmCancelMigration action On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/http/http_endpoint.rs | 8 +++++--- vmm/src/api/http/mod.rs | 12 ++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs index 7cac9f5b2a..26d09d9242 100644 --- a/vmm/src/api/http/http_endpoint.rs +++ b/vmm/src/api/http/http_endpoint.rs @@ -47,9 +47,10 @@ use crate::api::http::{EndpointHandler, HttpError, error_response}; use crate::api::{ AddDisk, ApiAction, ApiError, ApiRequest, NetConfig, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, - VmConfig, VmCounters, VmDelete, VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, - VmPowerButton, VmReboot, VmReceiveMigration, VmReceiveMigrationData, VmRemoveDevice, VmResize, - VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, VmShutdown, VmSnapshot, + VmCancelMigration, VmConfig, VmCounters, VmDelete, VmMigrationProgress, VmNmi, VmPause, + VmPostMigrationAnnounce, VmPowerButton, VmReboot, VmReceiveMigration, VmReceiveMigrationData, + VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, VmSendMigration, + VmShutdown, VmSnapshot, }; use crate::config::RestoreConfig; use crate::cpu::Error as CpuError; @@ -417,6 +418,7 @@ vm_action_put_handler!(VmResume); vm_action_put_handler!(VmPostMigrationAnnounce); vm_action_put_handler!(VmPowerButton); vm_action_put_handler!(VmNmi); +vm_action_put_handler!(VmCancelMigration); vm_action_put_handler_body!(VmAddDevice); vm_action_put_handler_body!(AddDisk); diff --git a/vmm/src/api/http/mod.rs b/vmm/src/api/http/mod.rs index 5ac3b35672..5464ca87ab 100644 --- a/vmm/src/api/http/mod.rs +++ b/vmm/src/api/http/mod.rs @@ -29,10 +29,10 @@ use self::http_endpoint::{VmActionHandler, VmCreate, VmInfo, VmmPing, VmmShutdow use crate::api::VmCoredump; use crate::api::{ AddDisk, ApiError, ApiRequest, VmAddDevice, VmAddFs, VmAddGenericVhostUser, VmAddNet, - VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, - VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, VmReboot, - VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, VmResume, - VmSendMigration, VmShutdown, VmSnapshot, + VmAddPmem, VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCancelMigration, VmCounters, + VmDelete, VmMigrationProgress, VmNmi, VmPause, VmPostMigrationAnnounce, VmPowerButton, + VmReboot, VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeDisk, VmResizeZone, VmRestore, + VmResume, VmSendMigration, VmShutdown, VmSnapshot, }; use crate::landlock::Landlock; use crate::seccomp_filters::{Thread, get_seccomp_filter}; @@ -282,6 +282,10 @@ pub static HTTP_ROUTES: LazyLock = LazyLock::new(|| { endpoint!("/vm.send-migration"), Box::new(VmActionHandler::new(&VmSendMigration)), ); + r.routes.insert( + endpoint!("/vm.cancel-migration"), + Box::new(VmActionHandler::new(&VmCancelMigration)), + ); r.routes.insert( endpoint!("/vm.shutdown"), Box::new(VmActionHandler::new(&VmShutdown)), From 234c62733acd32a44ac053c092af7d27aa75e9fb Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 2 Mar 2026 14:37:13 +0100 Subject: [PATCH 62/79] vmm: migration: actually support cancellation Introduce the minimal functionality required to support canceling a live migration. This establishes the basic mechanism, while subsequent commits will reduce the latency of cancellation so that migrations stop more promptly. Management software can and should wait for the migration to be actually canceled via the vm.migration-progress endpoint. [ Adapted to upstream's migration worker: the cancellation flag lives on MigrationWorkerHandle in migration/worker.rs and is therefore reached through VmOwnership::Migration instead of a separate Vmm field, and the cancelled VM is handed back with upstream's try_resume_vm_after_failed_migration(). ] On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/lib.rs | 3 +++ vmm/src/lib.rs | 50 ++++++++++++++++++++++++++++++------- vmm/src/migration/worker.rs | 19 +++++++++++++- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 89cbcecf87..60b1a47496 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -95,6 +95,9 @@ pub enum MigratableError { #[error("Failed to complete migration for migratable component")] CompleteMigration(#[source] anyhow::Error), + #[error("Failed to continue the migration as it was cancelled")] + Cancelled, + #[error("Failed to release a disk lock")] UnlockError(#[source] anyhow::Error), diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 0b77de2ab2..61be0b4bf9 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -19,6 +19,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::panic::AssertUnwindSafe; #[cfg(feature = "guest_debug")] use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, RecvError, SendError, Sender}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -1681,9 +1682,19 @@ impl Vmm { send_data_migration: &VmSendMigrationData, initial_vm_state: VmState, postponed_lifecycle_event: &Mutex>, + cancel: Arc, ) -> result::Result<(), MigratableError> { // State machine that is updated with more context as we progress. let mut ctx = OngoingMigrationContext::new(); + let return_if_cancelled_cb = move |socket: &mut SocketStream| { + if cancel.load(Ordering::Acquire) { + info!("Cancelling migration now"); + Request::abandon().write_to(socket)?; + Err(MigratableError::Cancelled) + } else { + Ok(()) + } + }; // Set up the socket connection let mut socket = if send_data_migration.local { @@ -1812,6 +1823,10 @@ impl Vmm { mem_send.cleanup()?; } + // Very last cancellation check. After this, we release the disk locks and we can't cancel + // anymore. + return_if_cancelled_cb(&mut socket)?; + // Update migration progress snapshot { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -2089,6 +2104,19 @@ impl Vmm { error!("Failed exiting the VMM after migration: {e}"); } } + Err(MigratableError::Cancelled) => { + error!("Migration cancelled"); + event!("vm", "migration-cancelled"); + try_resume_vm_after_failed_migration(vm); + + // Update migration progress snapshot + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .mark_as_cancelled(); + } + } Err(e) => { error!("Migration failed: {e}"); try_resume_vm_after_failed_migration(vm); @@ -3297,16 +3325,20 @@ impl RequestHandler for Vmm { } fn vm_cancel_migration(&mut self) -> result::Result<(), MigratableError> { - match self.vm { - VmOwnership::Migration { .. } => (), - _ => { - return Err(MigratableError::CancelMigration(anyhow!( - "There is no ongoing migration" - ))); - } - } + let VmOwnership::Migration { + ref migration_worker_handle, + .. + } = self.vm + else { + return Err(MigratableError::CancelMigration(anyhow!( + "There is no ongoing migration" + ))); + }; + + // We just dispatch the cancellation. + migration_worker_handle.trigger_cancellation(); - todo!() + Ok(()) } fn vm_migration_progress(&mut self) -> Option { diff --git a/vmm/src/migration/worker.rs b/vmm/src/migration/worker.rs index a1fb9a4148..8808177958 100644 --- a/vmm/src/migration/worker.rs +++ b/vmm/src/migration/worker.rs @@ -13,13 +13,14 @@ //! [`MigrationWorkerSpawnError`]. use std::fmt::{Debug, Formatter}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::Receiver; use std::sync::{Arc, Mutex}; use std::thread; use std::thread::JoinHandle; use event_monitor::event; -use log::warn; +use log::{info, warn}; use vm_migration::MigratableError; use vmm_sys_util::eventfd::EventFd; @@ -45,9 +46,20 @@ impl Debug for MigrationWorkerSpawnError { pub struct MigrationWorkerHandle { handle: Option>, + cancel: Arc, } impl MigrationWorkerHandle { + /// Cancels the migration. + /// + /// Note that timing issues in the very last phase of the migration allow a + /// tiny window in that migration succeeds before they could be canceled. + pub fn trigger_cancellation(&self) { + info!("Will cancel ongoing live-migration"); + self.cancel.store(true, Ordering::Release); + // we just dispatch here and do not block for the migration thread + } + pub fn join(mut self) -> MigrationWorkerResult { self.handle .take() @@ -73,6 +85,7 @@ pub struct MigrationWorker { config: VmSendMigrationData, /// Shared with the main VMM thread. postponed_lifecycle_event: Arc>>, + cancel: Arc, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor: Arc, initial_vm_state: VmState, @@ -92,6 +105,7 @@ impl MigrationWorker { &self.config, self.initial_vm_state, self.postponed_lifecycle_event.as_ref(), + self.cancel.clone(), ) .inspect(|_| event!("vm", "migration-finished")) .inspect_err(|_| event!("vm", "migration-failed")); @@ -122,11 +136,13 @@ impl MigrationWorker { initial_vm_state: VmState, ) -> Result { let (vm_sender, vm_receiver) = std::sync::mpsc::sync_channel(0); + let cancel = Arc::new(AtomicBool::new(false)); let worker = MigrationWorker { vm_receiver, check_migration_evt, config, postponed_lifecycle_event, + cancel: cancel.clone(), #[cfg(all(feature = "kvm", target_arch = "x86_64"))] hypervisor, initial_vm_state, @@ -149,6 +165,7 @@ impl MigrationWorker { Ok(MigrationWorkerHandle { handle: Some(inner_handle), + cancel, }) } } From 6e0465d3525ac122aa692b26582d5793807ceb98 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 2 Mar 2026 14:51:22 +0100 Subject: [PATCH 63/79] vmm: migration: early cancellation (add more checks) This adds multiple points in the migration path where the migration can be canceled early. The pre-copy phase is not addressed here and will follow in the next commit! On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 18 ++++++++++++++++-- vmm/src/migration/transport.rs | 9 ++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 61be0b4bf9..c6fae8d890 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1419,6 +1419,7 @@ impl Vmm { is_converged: impl Fn(&MemoryMigrationContext) -> result::Result, mem_send: &mut SendAdditionalConnections, postponed_lifecycle_event: &Mutex>, + return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> result::Result<(), MigratableError>, ) -> result::Result { let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); @@ -1451,6 +1452,8 @@ impl Vmm { }; loop { + return_if_cancelled_cb(socket)?; + // todo: check if auto-converge is enabled at all? if Self::can_increase_autoconverge_step(ctx) && vm.throttle_percent() < AUTO_CONVERGE_MAX @@ -1486,7 +1489,7 @@ impl Vmm { // Send the current dirty pages let transfer_begin = Instant::now(); - mem_send.send_memory(iteration_table, socket)?; + mem_send.send_memory(iteration_table, socket, return_if_cancelled_cb)?; let transfer_duration = transfer_begin.elapsed(); ctx.update_metrics_after_transfer(transfer_begin, transfer_duration); @@ -1624,6 +1627,7 @@ impl Vmm { mem_send: &mut SendAdditionalConnections, ctx: &mut OngoingMigrationContext, postponed_lifecycle_event: &Mutex>, + return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> result::Result<(), MigratableError>, ) -> result::Result<(), MigratableError> { let mut mem_ctx = MemoryMigrationContext::new(); @@ -1636,6 +1640,7 @@ impl Vmm { |ctx| Self::is_precopy_converged(ctx, send_data_migration), mem_send, postponed_lifecycle_event, + return_if_cancelled_cb, )?; let downtime_begin = Instant::now(); // End throttle thread @@ -1658,7 +1663,7 @@ impl Vmm { mem_ctx.update_metrics_before_transfer(iteration_begin, &final_table); let transfer_begin = Instant::now(); - mem_send.send_memory(final_table, socket)?; + mem_send.send_memory(final_table, socket, return_if_cancelled_cb)?; let transfer_duration = transfer_begin.elapsed(); mem_ctx.update_metrics_after_transfer(transfer_begin, transfer_duration); mem_ctx.iteration += 1; @@ -1716,6 +1721,8 @@ impl Vmm { MigratableError::MigrateSend(anyhow!("Error starting migration (got bad response)")), )?; + return_if_cancelled_cb(&mut socket)?; + // Send config let vm_config = vm.get_config(); #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -1754,6 +1761,8 @@ impl Vmm { .map_err(MigratableError::MigrateSend)? }; + return_if_cancelled_cb(&mut socket)?; + if send_data_migration.local { match &mut socket { SocketStream::Unix(unix_socket) => { @@ -1773,6 +1782,8 @@ impl Vmm { } } + return_if_cancelled_cb(&mut socket)?; + let vm_migration_config = VmMigrationConfig { vm_config, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -1781,6 +1792,8 @@ impl Vmm { }; transport::send_config(&mut socket, &vm_migration_config)?; + return_if_cancelled_cb(&mut socket)?; + // Let every Migratable object know about the migration being started. vm.start_migration()?; @@ -1811,6 +1824,7 @@ impl Vmm { &mut mem_send, &mut ctx, postponed_lifecycle_event, + &return_if_cancelled_cb, ) .inspect_err(|_| { // Calling cleanup multiple times is fine, thus here we just make sure diff --git a/vmm/src/migration/transport.rs b/vmm/src/migration/transport.rs index e001e328ed..437ae4b8ab 100644 --- a/vmm/src/migration/transport.rs +++ b/vmm/src/migration/transport.rs @@ -749,6 +749,7 @@ impl SendAdditionalConnections { &mut self, table: MemoryRangeTable, socket: &mut SocketStream, + return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> Result<(), MigratableError>, ) -> Result { if table.regions().is_empty() { return Ok(false); @@ -756,13 +757,19 @@ impl SendAdditionalConnections { // If we use only one connection, we send the memory directly. if self.threads.is_empty() { - send_memory_ranges(&self.guest_memory, &table, socket)?; + for chunk in table.partition(Self::CHUNK_SIZE) { + return_if_cancelled_cb(socket) + .inspect_err(|_| info!("cancelling migration during memory iteration"))?; + send_memory_ranges(&self.guest_memory, &chunk, socket)?; + } return Ok(true); } // The chunk size is chosen to be big enough so that even very fast links need some // milliseconds to send it. for chunk in table.partition(Self::CHUNK_SIZE) { + return_if_cancelled_cb(socket) + .inspect_err(|_| info!("cancelling migration during memory iteration"))?; self.send_chunk(chunk)?; } From 43ee9ac39ee64c0e064ab68bbdc7ec4ed685413d Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 5 Mar 2026 09:26:37 +0100 Subject: [PATCH 64/79] vmm: migration cancellation: integrate into TCP threads This commit reduces the delay in stopping memory transmission during the pre-copy phase when a migration is canceled. The cancellation handling is implemented in SendAdditionalConnections, which coordinates all memory transmission threads. In the cloud-hypervisor log, we can now even see that the cancellation happens fairly quickly when in pre-copy phase with multiple connections: ``` cloud-hypervisor: 11.263371s: INFO:vmm/src/api/mod.rs:1147 -- API request event: VmCancelMigration () cloud-hypervisor: 11.263401s: INFO:vmm/src/lib.rs:805 -- Will cancel ongoing live-migration cloud-hypervisor: 11.263416s: INFO:vmm/src/lib.rs:2662 -- Cancelling migration now cloud-hypervisor: 11.263568s: INFO:vmm/src/lib.rs:1667 -- Sending disconnect message to channels cloud-hypervisor: 11.263594s: INFO:vmm/src/lib.rs:1677 -- Waiting for threads to finish cloud-hypervisor: 11.302994s: INFO:vmm/src/lib.rs:1531 -- Sent 128 MiB via additional connection. cloud-hypervisor: 11.303037s: INFO:vmm/src/lib.rs:1531 -- Sent 64 MiB via additional connection. cloud-hypervisor: 11.303062s: INFO:vmm/src/lib.rs:1531 -- Sent 64 MiB via additional connection. cloud-hypervisor: 11.303066s: INFO:vmm/src/lib.rs:1531 -- Sent 64 MiB via additional connection. cloud-hypervisor: 11.303354s: INFO:vmm/src/lib.rs:1681 -- Threads finished cloud-hypervisor: 11.303672s: ERROR:vmm/src/lib.rs:858 -- migrate error: Failed to continue the migration as it was cancelled ``` On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/migration/transport.rs | 55 ++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/vmm/src/migration/transport.rs b/vmm/src/migration/transport.rs index 437ae4b8ab..e73b8bfcd5 100644 --- a/vmm/src/migration/transport.rs +++ b/vmm/src/migration/transport.rs @@ -12,7 +12,9 @@ use std::os::unix::net::{UnixListener, UnixStream}; use std::path::{Path, PathBuf}; use std::result::Result; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{Receiver, Sender, SyncSender, TrySendError, channel, sync_channel}; +use std::sync::mpsc::{ + Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, +}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; @@ -583,6 +585,9 @@ pub(crate) struct SendAdditionalConnections { /// this using this flag. Only the main thread checks this variable, the worker /// threads will be stopped during cleanup. worker_error: Arc, + /// Externally triggered cancellation. Workers drain queued memory messages + /// after this is set and wait for the disconnect message. + external_cancel: Arc, /// After the main thread sent all memory chunks to the sender threads, it waits /// until one of the workers notifies it. Either because an error occurred, or /// because they arrived at the gate. @@ -625,6 +630,7 @@ impl SendAdditionalConnections { let buffer_size = Self::BUFFERED_REQUESTS_PER_THREAD * configured_connections as usize; let (message_tx, message_rx) = sync_channel::(buffer_size); let worker_error = Arc::new(AtomicBool::new(false)); + let external_cancel = Arc::new(AtomicBool::new(false)); let (notify_tx, notify_rx) = channel::(); // If one connection is configured, we don't have to create any additional threads. @@ -635,6 +641,7 @@ impl SendAdditionalConnections { threads, message_tx, worker_error, + external_cancel, notify_rx, }); } @@ -648,6 +655,7 @@ impl SendAdditionalConnections { let guest_memory = guest_memory.clone(); let message_rx = message_rx.clone(); let worker_error = worker_error.clone(); + let external_cancel = external_cancel.clone(); let notify_tx = notify_tx.clone(); let thread = thread::Builder::new() @@ -658,6 +666,7 @@ impl SendAdditionalConnections { &guest_memory, &message_rx, &worker_error, + &external_cancel, ¬ify_tx, ) }) @@ -680,6 +689,7 @@ impl SendAdditionalConnections { threads, message_tx, worker_error, + external_cancel, notify_rx, }) } @@ -689,6 +699,7 @@ impl SendAdditionalConnections { guest_memory: &GuestMemoryAtomic, message_rx: &Mutex>, worker_error: &AtomicBool, + external_cancel: &AtomicBool, notify_tx: &Sender, ) -> Result<(), MigratableError> { info!("Spawned thread to send VM memory."); @@ -713,6 +724,10 @@ impl SendAdditionalConnections { })?; match message { SendMemoryThreadMessage::Memory(table) => { + if external_cancel.load(Ordering::Acquire) { + continue; + } + send_memory_ranges(guest_memory, &table, socket) .inspect_err(|_| { worker_error.store(true, Ordering::Relaxed); @@ -768,12 +783,14 @@ impl SendAdditionalConnections { // The chunk size is chosen to be big enough so that even very fast links need some // milliseconds to send it. for chunk in table.partition(Self::CHUNK_SIZE) { - return_if_cancelled_cb(socket) - .inspect_err(|_| info!("cancelling migration during memory iteration"))?; + return_if_cancelled_cb(socket).inspect_err(|_| { + info!("cancelling migration during memory iteration"); + self.external_cancel.store(true, Ordering::Release); + })?; self.send_chunk(chunk)?; } - self.wait_for_pending_data()?; + self.wait_for_pending_data(socket, return_if_cancelled_cb)?; Ok(true) } @@ -808,7 +825,11 @@ impl SendAdditionalConnections { } /// Wait until all data that is in-flight has actually been sent and acknowledged. - fn wait_for_pending_data(&mut self) -> Result<(), MigratableError> { + fn wait_for_pending_data( + &mut self, + socket: &mut SocketStream, + return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> Result<(), MigratableError>, + ) -> Result<(), MigratableError> { let gate = Arc::new(Gate::new()); for _ in 0..self.threads.len() { self.message_tx @@ -822,26 +843,34 @@ impl SendAdditionalConnections { // they arrived at the gate. let mut seen_threads = 0; loop { - match self - .notify_rx - .recv() - .context("Error receiving message from workers") - .map_err(MigratableError::MigrateSend)? - { - SendMemoryThreadNotify::Gate => { + return_if_cancelled_cb(socket).inspect_err(|_| { + gate.open(); + self.external_cancel.store(true, Ordering::Release); + })?; + + thread::sleep(Duration::from_millis(2)); + + match self.notify_rx.try_recv() { + Ok(SendMemoryThreadNotify::Gate) => { seen_threads += 1; if seen_threads == self.threads.len() { gate.open(); return Ok(()); } } - SendMemoryThreadNotify::Error => { + Ok(SendMemoryThreadNotify::Error) => { // If an error occurred in one of the worker threads, we open // the gate to make sure that no thread hangs. After that, we // receive the error from Self::cleanup() and return it. gate.open(); return self.cleanup(); } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Err(MigratableError::MigrateSend(anyhow!( + "All senders died unexpectedly." + ))); + } } } } From 39296b45c73c7c7c67d1c4709c50dd7c79d5322e Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 19 Feb 2026 15:40:48 +0100 Subject: [PATCH 65/79] ch-remote: add cancel-migration On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- cloud-hypervisor/src/bin/ch-remote.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloud-hypervisor/src/bin/ch-remote.rs b/cloud-hypervisor/src/bin/ch-remote.rs index f8075de113..dd2eefb79e 100644 --- a/cloud-hypervisor/src/bin/ch-remote.rs +++ b/cloud-hypervisor/src/bin/ch-remote.rs @@ -636,6 +636,8 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu )?; simple_api_command(socket, "PUT", "create", Some(&data)).map_err(Error::HttpApiClient) } + Some("cancel-migration") => simple_api_command(socket, "PUT", "cancel-migration", None) + .map_err(Error::HttpApiClient), _ => unreachable!(), } } @@ -1132,6 +1134,7 @@ fn get_cli_commands_sorted() -> Box<[Command]> { .about("Add vsock device") .arg(Arg::new("vsock_config").index(1).help(VsockConfig::SYNTAX)), Command::new("boot").about("Boot a created VM"), + Command::new("cancel-migration").about("Cancel any ongoing migration"), Command::new("coredump") .about("Create a coredump from VM") .arg(Arg::new("coredump_config").index(1).help("")), From 8c4240180fcb0b8b0a2b5323b558a5fd7cddfcf6 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 30 Mar 2026 15:53:27 +0200 Subject: [PATCH 66/79] vmm: migration: properly print error chain on failure TL;DR: Major improvement for developers to see why a migration failed The error model of Cloud Hypervisor leverages std::error::Error and its source() function respectively to build chains of errors. This helps to spot the root cause and see how a certain operation failed throughout the stack. Therefore, the Display::fmt() impl of every error only prints what failed "on its level", but never appends ": {0}", i.e., the underling error's Display::fmt(). ch-remote and cloud-hypervisor can print these error chains nicely when they exit - in the case of a migration, we do not exit however. The solution is to print the error chain there as well to get more meaningful errors. As example: old: ``` cloud-hypervisor: 62.820480s: INFO:vmm/src/lib.rs:3382 -- VM migration check event cloud-hypervisor: 62.820576s: ERROR:vmm/src/lib.rs:3138 -- Migration failed: Failed to send migratable component snapshot ``` new: ``` cloud-hypervisor: 62.820480s: INFO:vmm/src/lib.rs:3382 -- VM migration check event cloud-hypervisor: 15.311401s: ERROR:vmm/src/lib.rs:3110 -- Migration failed with the following chain of errors: cloud-hypervisor: 15.311412s: ERROR:vmm/src/lib.rs:3118 -- 0: Failed to send migratable component snapshot cloud-hypervisor: 15.311422s: ERROR:vmm/src/lib.rs:3118 -- 1: Error connecting to TCP socket cloud-hypervisor: 15.311442s: ERROR:vmm/src/lib.rs:3118 -- 2: Connection refused (os error 111) cloud-hypervisor: 62.820618s: INFO:event_monitor/src/lib.rs:113 -- Event: source = vm event = migration-failed ``` On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index c6fae8d890..472385569e 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -2035,6 +2035,37 @@ impl Vmm { } } + /// Prints the error chain to `error!()` level, akin to user-facing errors when Cloud Hypervisor + /// or ch-remote fail. + // TODO: For upstreaming, we should unify this with the code-paths used by ch-remote and + // Cloud Hypervisor on failure. + fn log_print_error_chain<'a>(top_error: &'a (dyn std::error::Error + 'static)) { + // Print chain of errors + if top_error.source().is_none() { + error!("Migration failed with the following error:"); + error!(" {top_error}"); + } else { + // In cli_print_error_chain(), we also print the + // ::fmt() as oneliner so that we can see all + // properties. As we use anyhow errors in the migration path, + // Debug::fmt() is not helpful for us as it doesn't print the + // underlying properties (like the default Debug::fmt() impl would + // do). Instead, it would print a trace itself, which is not what + // we want to do here. + + error!("Migration failed with the following chain of errors:"); + std::iter::successors(Some(top_error), |sub_error| { + // Dereference necessary to mitigate rustc compiler bug. + // See + (*sub_error).source() + }) + .enumerate() + .for_each(|(level, error)| { + error!(" {level}: {error}"); + }); + } + } + /// Handles the outcome of the migration worker thread. fn check_migration(&mut self) { let VmOwnership::Migration { @@ -2132,7 +2163,7 @@ impl Vmm { } } Err(e) => { - error!("Migration failed: {e}"); + Self::log_print_error_chain(&e); try_resume_vm_after_failed_migration(vm); // Update migration progress snapshot From 17539fe3cda25bf0283f82454ded3e1bf6ff4a62 Mon Sep 17 00:00:00 2001 From: Leander Kohler Date: Mon, 9 Mar 2026 16:37:50 +0100 Subject: [PATCH 67/79] vmm: defer guest exit during migration Move the migration workaround from the shared Exit path to GuestExit and rename the postponed shutdown event to `VmShutdown`. With --no-shutdown, guest-triggered shutdown must keep following the guest exit path even when it is delayed until after migration completion. This preserves the distinction between guest shutdown and real VMM exit conditions. The existing fatal exit path stays unchanged. [ Adapted to upstream's migration worker: the postponement checks VmOwnership::Migration and replaces upstream's lifecycle TODO marker in the GuestExit arm of the control loop. ] On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 29 ++++++++++++++--------------- vmm/src/vm.rs | 2 +- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 472385569e..454de5d10e 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1138,10 +1138,10 @@ impl Vmm { .context("Failed writing reset eventfd after migration") .map_err(MigratableError::MigrateReceive)?; } - Some(PostMigrationLifecycleEvent::VmmShutdown) => { - self.exit_evt + Some(PostMigrationLifecycleEvent::VmShutdown) => { + self.guest_exit_evt .write(1) - .context("Failed writing exit eventfd after migration") + .context("Failed writing guest exit eventfd after migration") .map_err(MigratableError::MigrateReceive)?; } } @@ -2112,10 +2112,10 @@ impl Vmm { .inspect_err(|write_err| error!("{write_err}")) .ok(); } - PostMigrationLifecycleEvent::VmmShutdown => { - self.exit_evt + PostMigrationLifecycleEvent::VmShutdown => { + self.guest_exit_evt .write(1) - .context("Failed replaying shutdown event after failed migration") + .context("Failed replaying guest exit event after failed migration") .inspect_err(|write_err| error!("{write_err}")) .ok(); } @@ -2217,14 +2217,6 @@ impl Vmm { info!("VM exit event"); // Consume the event. self.exit_evt.read().map_err(Error::EventFdRead)?; - // The migration worker owns the VM, so the lifecycle - // change is applied once the migration finished. - if matches!(self.vm, VmOwnership::Migration { .. }) { - self.postpone_lifecycle_event_during_migration( - PostMigrationLifecycleEvent::VmmShutdown, - ); - continue; - } self.vmm_shutdown().map_err(Error::VmmShutdown)?; break 'outer; @@ -2246,7 +2238,14 @@ impl Vmm { EpollDispatch::GuestExit => { info!("VM guest exit event"); self.guest_exit_evt.read().map_err(Error::EventFdRead)?; - // TODO: Future follow-up must resolve lifecycle handling while migrating. + // The migration worker owns the VM, so the lifecycle + // change is applied once the migration finished. + if matches!(self.vm, VmOwnership::Migration { .. }) { + self.postpone_lifecycle_event_during_migration( + PostMigrationLifecycleEvent::VmShutdown, + ); + continue; + } if self.no_shutdown { self.vm_shutdown().map_err(Error::VmShutdown)?; } else { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 18f9200755..513cf6bb1e 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -587,7 +587,7 @@ pub struct Vm { #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum PostMigrationLifecycleEvent { VmReboot, - VmmShutdown, + VmShutdown, } impl Vm { From dbcc81545605c96f7552e05afc5fae7dbb19c738 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Fri, 13 Mar 2026 06:38:29 +0100 Subject: [PATCH 68/79] vmm: reduce API event verbosity These are called very frequently by libvirt and spam the log. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/api/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 5e71b3d7a2..03caaa5b82 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -40,7 +40,7 @@ use std::str::FromStr; use std::sync::mpsc::{RecvError, SendError, Sender, channel}; use std::time::Duration; -use log::{debug, info}; +use log::{info, trace}; use micro_http::Body; use option_parser::{OptionParser, OptionParserError, Toggle}; use serde::{Deserialize, Serialize}; @@ -1283,7 +1283,7 @@ impl ApiAction for VmCounters { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - info!("API request event: VmCounters"); + trace!("API request event: VmCounters"); let response = vmm .vm_counters() @@ -1388,7 +1388,7 @@ impl ApiAction for VmInfo { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - debug!("API request event: VmInfo"); + trace!("API request event: VmInfo"); let response = vmm .vm_info() @@ -2049,7 +2049,7 @@ impl ApiAction for VmMigrationProgress { fn request(&self, _: Self::RequestBody, response_sender: Sender) -> ApiRequest { Box::new(move |vmm| { - debug!("API request event: VmMigrationProgress"); + trace!("API request event: VmMigrationProgress"); let snapshot = Ok(vmm.vm_migration_progress()); let response = snapshot From 3c44fa95f49b384e4ce6513d40a70cd91c7fdce0 Mon Sep 17 00:00:00 2001 From: Leander Kohler Date: Tue, 10 Mar 2026 14:57:49 +0100 Subject: [PATCH 69/79] vmm: keep virtio activation alive in migration Live migration can deadlock if the guest triggers a virtio device activation while the migration worker owns the VM. The failure shows up during boot and firmware, where the guest can reset and reinitialize virtio devices while precopy is running. In the failing case, the source log shows a pending virtio activation that never completes: 8.115833s _virtio-pci-net_0: Needs activation; returning barrier 8.115854s vmm/src/vm.rs:464 -- Waiting for barrier 24.875452s Entering downtime phase 24.875481s stopping vcpu throttling thread ... vCPU thread did not respond in 10ms to signal - retrying vCPU thread did not respond in 20ms to signal - retrying ... thread 'throttle-vcpu' (1029) panicked ... Pause(Error signalling vCPUs: Timeout when waiting for signal to be acknowledged) The vCPU blocks on the activation barrier and never reaches the normal pause checkpoint. Later, migration enters downtime and stops the vCPU throttle thread. In the failing case, that thread is still inside a CpuManager::pause() call, which waits for every vCPU to acknowledge the signal. The blocked vCPU never does, so the pause times out. The VMM already receives ActivateVirtioDevices events during migration, but it only drains pending activations when self.vm is in MaybeVmOwnership::Vmm. Once vm_send_migration() moves the Vm into the migration worker, self.vm becomes MaybeVmOwnership::Migration and the event handler no longer has a path to call activate_virtio_devices(). Fix this by storing the DeviceManager inside MaybeVmOwnership::Migration. This keeps just enough state on the VMM thread to drain pending virtio activations while the migration worker owns the Vm. The barrier logic stays unchanged. The VMM now releases the same activation barrier during migration that it already released before migration started. This keeps the guest from getting stuck in the activation wait and lets the later pause succeed. [ Re-implemented on top of upstream's migration worker: the weak device manager reference is a field of VmOwnership::Migration instead of the fork's MigrationVmState helper struct, and the control loop's activation arm replaces upstream's virtio activation TODO marker. ] On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 36 ++++++++++++++++++++++++++++++------ vmm/src/vm.rs | 4 ++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 454de5d10e..1e7c759a14 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -21,7 +21,7 @@ use std::panic::AssertUnwindSafe; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, RecvError, SendError, Sender}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, Weak}; use std::time::Duration; #[cfg(not(target_arch = "riscv64"))] use std::time::Instant; @@ -71,6 +71,7 @@ use crate::config::{MemoryRestoreMode, RestoreConfig, add_to_config}; use crate::coredump::GuestDebuggable; #[cfg(feature = "kvm")] use crate::cpu::IS_IN_SHUTDOWN; +use crate::device_manager::DeviceManager; use crate::landlock::Landlock; use crate::memory_manager::MemoryManager; use crate::migration::transport::{ @@ -651,6 +652,12 @@ pub enum VmOwnership { migration_worker_handle: MigrationWorkerHandle, /// Snapshot returned while the VMM cannot inspect the worker-owned VM. vm_info_response: VmInfoResponse, + /// Keeps the device manager reachable so the epoll thread can drain + /// pending virtio activations while the migration worker owns the VM. + /// + /// The migration worker owns the VM during migration, so this should + /// stop working once that VM has been dropped. + device_manager: Weak>, }, None, } @@ -2254,12 +2261,27 @@ impl Vmm { } } EpollDispatch::ActivateVirtioDevices => { - // TODO: Future follow-up must resolve virtio activation handling while migrating. let count = self.activate_evt.read().map_err(Error::EventFdRead)?; - if let VmOwnership::Owned(ref vm) = self.vm { - info!("Trying to activate pending virtio devices: count = {count}"); - vm.activate_virtio_devices() - .map_err(Error::ActivateVirtioDevices)?; + match &self.vm { + VmOwnership::Owned(vm) => { + info!("Trying to activate pending virtio devices: count = {count}"); + vm.activate_virtio_devices() + .map_err(Error::ActivateVirtioDevices)?; + } + VmOwnership::Migration { device_manager, .. } => { + info!( + "Trying to activate pending virtio devices of migrating VM: count = {count}" + ); + device_manager + .upgrade() + .expect("device manager should remain alive during migration") + .lock() + .unwrap() + .activate_virtio_devices() + .map_err(VmError::ActivateVirtioDevices) + .map_err(Error::ActivateVirtioDevices)?; + } + VmOwnership::None => {} } } EpollDispatch::Api => { @@ -3334,6 +3356,7 @@ impl RequestHandler for Vmm { .vm .take_owned_or(VmError::VmNotRunning) .expect("should have VM ownership as we just checked it"); + let device_manager = Arc::downgrade(vm.device_manager()); match MigrationWorker::spawn( vm, @@ -3348,6 +3371,7 @@ impl RequestHandler for Vmm { self.vm = VmOwnership::Migration { migration_worker_handle: handle, vm_info_response: vm_info_snapshot, + device_manager, }; Ok(()) } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 513cf6bb1e..3d28954d57 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -3248,6 +3248,10 @@ impl Vm { self.device_manager.lock().unwrap().device_tree() } + pub fn device_manager(&self) -> &Arc> { + &self.device_manager + } + /// Release all advisory locks held for the disk images. /// /// This should only be called when the VM is stopped and the VMM supposed From 62c9d6796b56ee5a23d4af938a57112a3910bc57 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Wed, 27 May 2026 12:30:32 +0200 Subject: [PATCH 70/79] vmm: Fix memory_bytes_total value in do_memory_iterations Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 1e7c759a14..60ad2c6ffa 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1428,6 +1428,12 @@ impl Vmm { postponed_lifecycle_event: &Mutex>, return_if_cancelled_cb: &impl Fn(&mut SocketStream) -> result::Result<(), MigratableError>, ) -> result::Result { + let total_memory_size_bytes = vm + .memory_range_table()? + .ranges() + .iter() + .map(|range| range.length) + .sum::(); let update_migration_progress = |s: &mut MemoryMigrationContext, vm: &Vm| { let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); lock.as_mut() @@ -1437,7 +1443,7 @@ impl Vmm { Some(MemoryTransmissionInfo { memory_iteration: s.iteration as u64, memory_transmission_bps: s.current_iteration_total_bytes, - memory_bytes_total: s.bandwidth_bytes_per_second as u64, + memory_bytes_total: total_memory_size_bytes, memory_bytes_transmitted: s.total_sent_bytes, memory_pages_4k_transmitted: s.total_sent_bytes.div_ceil(PAGE_SIZE as u64), memory_pages_4k_remaining_iteration: s From 33f709fc4d219a66822161ef5ca06b7d164d93d7 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Wed, 27 May 2026 12:39:03 +0200 Subject: [PATCH 71/79] vmm: Fix memory_transmission_bps value in do_memory_iterations Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 60ad2c6ffa..20e220bc9a 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1442,7 +1442,7 @@ impl Vmm { MigrationStateOngoingPhase::MemoryPrecopy, Some(MemoryTransmissionInfo { memory_iteration: s.iteration as u64, - memory_transmission_bps: s.current_iteration_total_bytes, + memory_transmission_bps: s.bandwidth_bytes_per_second as u64, memory_bytes_total: total_memory_size_bytes, memory_bytes_transmitted: s.total_sent_bytes, memory_pages_4k_transmitted: s.total_sent_bytes.div_ceil(PAGE_SIZE as u64), From 48de5cffd344f9c2ebadf6120cc724be87f285c0 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Wed, 27 May 2026 12:43:25 +0200 Subject: [PATCH 72/79] vmm: Remove duplicated update_migration_progress in do_memory_iterations Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 20e220bc9a..3ba988867a 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1490,16 +1490,13 @@ impl Vmm { }; ctx.update_metrics_before_transfer(iteration_begin, &iteration_table); - // Update before we might exit the loop. + // Update before we either exit the loop or transfer memory update_migration_progress(ctx, vm); if is_converged(ctx)? { info!("Precopy converged: {ctx}"); break Ok(iteration_table); } - // Update with new metrics before transmission. - update_migration_progress(ctx, vm); - // Send the current dirty pages let transfer_begin = Instant::now(); mem_send.send_memory(iteration_table, socket, return_if_cancelled_cb)?; From 90d72a2bc63ec52983f6177395f3bc0ae224b65e Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Wed, 1 Jul 2026 13:30:39 +0200 Subject: [PATCH 73/79] vmm, vm-migration: add MigrationStateOngoingPhase::Started This will help us in libvirt/ch to distinguish short send/receive races from a real migration startup failure. By being able to check if the migration has been successfully started (rather than just the migration worker thread), we can improve the reliability of the start_migration() logic in libvirt/ch. On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/progress.rs | 7 ++++++- vmm/src/lib.rs | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/vm-migration/src/progress.rs b/vm-migration/src/progress.rs index 8a5083068d..c5babdad92 100644 --- a/vm-migration/src/progress.rs +++ b/vm-migration/src/progress.rs @@ -76,8 +76,12 @@ pub struct MemoryTransmissionInfo { Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] pub enum MigrationStateOngoingPhase { - /// The migration starts. Handshake and transfer of VM config. + /// The migration process is initiated. No checks or connections are + /// established yet. Starting, + /// The initial connection is established and the migration protocol + /// handshake succeeded. + Started, /// Transfer of memory FDs. /// /// Only used for local migrations. @@ -96,6 +100,7 @@ impl Display for MigrationStateOngoingPhase { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Starting => write!(f, "starting"), + Self::Started => write!(f, "started"), Self::MemoryFds => write!(f, "memory FDs"), Self::MemoryPrecopy => write!(f, "memory (precopy)"), Self::Completing => write!(f, "completing"), diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 3ba988867a..cd91175f59 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1731,6 +1731,16 @@ impl Vmm { MigratableError::MigrateSend(anyhow!("Error starting migration (got bad response)")), )?; + // Signal that the migration connection has been established. Management + // software can use this to distinguish short send/receive races from a + // real migration startup failure. + { + let mut lock = MIGRATION_PROGRESS_SNAPSHOT.lock().unwrap(); + lock.as_mut() + .expect("live migration should be ongoing") + .update(MigrationStateOngoingPhase::Started, None, None, None); + } + return_if_cancelled_cb(&mut socket)?; // Send config From aaa060dfc3d4485fae93a24780a2854e0c96b6b2 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Wed, 1 Jul 2026 15:12:26 +0200 Subject: [PATCH 74/79] vmm, vm-migration: streamline error chain message with upstream Streamlines the printing of the error message with upstream [0, 1]. [0] https://github.com/cloud-hypervisor/cloud-hypervisor/pull/8469/changes/fcf2b5bbe6afeebb17dba5bace72a6896531be5e [1] https://github.com/cloud-hypervisor/cloud-hypervisor/commit/252702049e55cc2903a07fd24e6a165a33c54026 On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vm-migration/src/lib.rs | 18 +++++++++++++++++ vm-migration/src/progress.rs | 4 +++- vmm/src/lib.rs | 38 +++++------------------------------- 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/vm-migration/src/lib.rs b/vm-migration/src/lib.rs index 60b1a47496..1941e6f830 100644 --- a/vm-migration/src/lib.rs +++ b/vm-migration/src/lib.rs @@ -3,6 +3,9 @@ // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause // +use std::error::Error; +use std::iter; + use anyhow::anyhow; pub use context::{ CompletedMigrationContext, DowntimeContext, MemoryMigrationContext, MigrationContextError, @@ -20,6 +23,21 @@ pub mod progress; pub mod protocol; pub mod tls; +/// Mimics the error chain printing of CH for migration-related errors, where we +/// do not exit the VMM (which would print the error chain). +pub fn nested_error_to_flat_chain_as_string(top_error: &dyn Error) -> String { + iter::successors(Some(top_error), |sub_error| { + // Dereference necessary to mitigate rustc compiler bug. + // See + (*sub_error).source() + }) + // Important to use the plain Display impl to not interfere + // with anyhow's "smart" printing + .map(|e| format!("{e}")) + .collect::>() + .join(" => ") +} + #[derive(Error, Debug)] pub enum UffdError { #[error("Snapshot ranges are not page-aligned")] diff --git a/vm-migration/src/progress.rs b/vm-migration/src/progress.rs index c5babdad92..dc2f358b82 100644 --- a/vm-migration/src/progress.rs +++ b/vm-migration/src/progress.rs @@ -22,6 +22,8 @@ use std::fmt::Display; use std::num::NonZeroU32; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use crate::nested_error_to_flat_chain_as_string; + #[derive( Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] @@ -329,7 +331,7 @@ impl MigrationProgress { self.timestamp_snapshot_ms = current_unix_timestamp_ms(); self.timestamp_snapshot_relative_ms = self.timestamp_snapshot_ms - self.timestamp_begin_ms; self.state = MigrationState::Failed { - error_msg: format!("{error}",), + error_msg: nested_error_to_flat_chain_as_string(error), error_msg_debug: format!("{error:?}",), }; } diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index cd91175f59..6454a7a3d8 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -56,7 +56,7 @@ use vm_migration::progress::{ use vm_migration::protocol::*; use vm_migration::{ MemoryMigrationContext, Migratable, MigratableError, OngoingMigrationContext, Pausable, - Snapshot, Snapshottable, Transportable, + Snapshot, Snapshottable, Transportable, nested_error_to_flat_chain_as_string, }; use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::unblock_signal; @@ -2055,37 +2055,6 @@ impl Vmm { } } - /// Prints the error chain to `error!()` level, akin to user-facing errors when Cloud Hypervisor - /// or ch-remote fail. - // TODO: For upstreaming, we should unify this with the code-paths used by ch-remote and - // Cloud Hypervisor on failure. - fn log_print_error_chain<'a>(top_error: &'a (dyn std::error::Error + 'static)) { - // Print chain of errors - if top_error.source().is_none() { - error!("Migration failed with the following error:"); - error!(" {top_error}"); - } else { - // In cli_print_error_chain(), we also print the - // ::fmt() as oneliner so that we can see all - // properties. As we use anyhow errors in the migration path, - // Debug::fmt() is not helpful for us as it doesn't print the - // underlying properties (like the default Debug::fmt() impl would - // do). Instead, it would print a trace itself, which is not what - // we want to do here. - - error!("Migration failed with the following chain of errors:"); - std::iter::successors(Some(top_error), |sub_error| { - // Dereference necessary to mitigate rustc compiler bug. - // See - (*sub_error).source() - }) - .enumerate() - .for_each(|(level, error)| { - error!(" {level}: {error}"); - }); - } - } - /// Handles the outcome of the migration worker thread. fn check_migration(&mut self) { let VmOwnership::Migration { @@ -2183,7 +2152,10 @@ impl Vmm { } } Err(e) => { - Self::log_print_error_chain(&e); + error!( + "Migration failed: {}", + nested_error_to_flat_chain_as_string(&e) + ); try_resume_vm_after_failed_migration(vm); // Update migration progress snapshot From fd16dfea422dd18176a15c7b7f01f5907a70c056 Mon Sep 17 00:00:00 2001 From: Julian Schindel Date: Wed, 29 Jul 2026 15:27:16 +0200 Subject: [PATCH 75/79] vmm: clean up migration threads on failure Migration threads may be left orphaned and keep the socket bound after a migration has failed. Prevent this by signaling termination via the `terminate_fd` in `ReceiveAdditionalConnections`'s `Drop` impl. On-behalf-of: SAP julian.schindel@sap.com Signed-off-by: Julian Schindel Signed-off-by: Philipp Schuster --- vmm/src/migration/transport.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vmm/src/migration/transport.rs b/vmm/src/migration/transport.rs index e73b8bfcd5..95ea03b9fa 100644 --- a/vmm/src/migration/transport.rs +++ b/vmm/src/migration/transport.rs @@ -543,6 +543,9 @@ impl ReceiveAdditionalConnections { impl Drop for ReceiveAdditionalConnections { fn drop(&mut self) { + if let Err(error) = self.terminate_fd.write(1) { + warn!("Failed to write to termination fd: {error}"); + } if self.accept_thread.is_some() { warn!( "ReceiveAdditionalConnections was not cleaned up! Either cleanup() was never called (programming error) or it failed before completing." From 6dd5f1586b2d0ffe397d8247d57580f4b99f0214 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Mon, 3 Aug 2026 14:15:12 +0200 Subject: [PATCH 76/79] vmm: Do not stop vCPU throttle thread at the end of a live migration The current behavior of joining the vCPU throttling thread towards the end of a live migration is problematic when the live migration fails because then auto-converge is no longer possible on a second attempt. We fix this by instead resetting the throttling thread to its initial state. The throttling thread is now instead gracefully stopped by the ThrottleThreadHandle's destructor which runs whenever the Vm instance goes out of scope. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 6 ++--- vmm/src/vcpu_throttling.rs | 50 ++++++++++++++++++++++++++++++++++---- vmm/src/vm.rs | 8 +++--- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 6454a7a3d8..635941f120 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1654,9 +1654,9 @@ impl Vmm { )?; let downtime_begin = Instant::now(); // End throttle thread - info!("stopping vcpu thread"); - vm.stop_vcpu_throttling(); - info!("stopped vcpu thread"); + info!("stopping vcpu throttling"); + vm.reset_vcpu_throttle_thread(); + info!("stopped vcpu throttling"); // Skip if already paused, e.g. when migrating a paused VM. if vm.get_state() != VmState::Paused { info!("pausing VM"); diff --git a/vmm/src/vcpu_throttling.rs b/vmm/src/vcpu_throttling.rs index 464728b9e6..840fa06e67 100644 --- a/vmm/src/vcpu_throttling.rs +++ b/vmm/src/vcpu_throttling.rs @@ -30,13 +30,13 @@ use std::cell::Cell; use std::cmp::min; -use std::sync::mpsc::RecvTimeoutError; +use std::sync::mpsc::{RecvTimeoutError, SyncSender}; use std::sync::{Arc, Mutex, mpsc}; use std::thread; use std::thread::JoinHandle; use std::time::{Duration, Instant}; -use log::{debug, warn}; +use log::{debug, error, info, warn}; use vm_migration::Pausable; use crate::cpu::CpuManager; @@ -50,6 +50,11 @@ enum ThrottleCommand { Throttle(u8 /* `1..=99` */), /// Gracefully shutdown the vCPU throttling thread. Exit, + /// Exit the throttle loop then rendezvous with the receiver before proceeding to wait for the next command. + /// + /// In other words the `report` is used to synchronize the throttle thread reset event with the thread that + /// sent this command. + Reset { report: SyncSender<()> }, } /// Helper to adapt the throttling timeslice as we go, depending on the time it @@ -258,6 +263,7 @@ impl ThrottleWorker { None } Some(cmd @ (ThrottleCommand::Exit | ThrottleCommand::Wait)) => Some(cmd), + Some(ThrottleCommand::Reset { report }) => Some(ThrottleCommand::Reset { report }), } } @@ -345,10 +351,31 @@ impl ThrottleWorker { &callback_pause_vcpus, &callback_resume_vcpus, ); - if matches!(next_task, ThrottleCommand::Exit) { - break 'control; + match next_task { + ThrottleCommand::Exit => { + break 'control; + } + // else: thread needs to go into waiting state + ThrottleCommand::Reset { report } => { + // Inform sender that we are back in the waiting state: Since `report` has capacity 0 + // this call will block until the command sender has received our message. + if let Err(e) = report.send(()) { + error!( + "Unable to synchronize throttle thread reset event: error = {e:#?}" + ); + } + } + _ => { + continue 'control; + } + } + } + ThrottleCommand::Reset { report } => { + if let Err(e) = report.send(()) { + error!( + "Unable to synchronize throttle thread reset event: error = {e:#?}" + ); } - // else: thread is in Waiting state } } } @@ -527,6 +554,19 @@ impl ThrottleThreadHandle { ); } } + + /// Stops throttling and returns the throttle thread to the waiting state. + /// + /// This blocks until the throttling thread has exited the throttling loop. + pub fn reset(&self) { + let (report, recv) = mpsc::sync_channel(0); + self.state_sender + .send(ThrottleCommand::Reset { report }) + .expect("channel should not be closed"); + self.current_throttle.set(0); + info!("Waiting for throttle thread to acknowledge reset"); + recv.recv().expect("The throttle thread should acknowledge the reset event before dropping rendezvous channel"); + } } impl Drop for ThrottleThreadHandle { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 3d28954d57..1a9d2e4063 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -1495,11 +1495,11 @@ impl Vm { self.vcpu_throttler.throttle_percent() } - /// Stops and terminates the thread gracefully. + /// Sets the vCPU throttling thread back to its initial waiting state. /// - /// Waits for the thread to finish. - pub fn stop_vcpu_throttling(&mut self) { - self.vcpu_throttler.shutdown(); + /// Blocks until the throttling thread acknowledges the reset event. + pub fn reset_vcpu_throttle_thread(&self) { + self.vcpu_throttler.reset(); } pub fn set_post_migration_lifecycle_event( From a1cc49e8cfddcd62e9d2dd6673c7658b4f66b6f3 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Thu, 6 Aug 2026 14:53:08 +0200 Subject: [PATCH 77/79] vmm: Reset throttle thread when do_memory_iteration fails When a live migration fails before all memory iterations have been sent we also need to reset the throttling thread, otherwise the VM continues existing on the migration source with throttled vCPUs. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 635941f120..04770c98b4 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1651,12 +1651,16 @@ impl Vmm { mem_send, postponed_lifecycle_event, return_if_cancelled_cb, - )?; + ); + let downtime_begin = Instant::now(); // End throttle thread info!("stopping vcpu throttling"); vm.reset_vcpu_throttle_thread(); info!("stopped vcpu throttling"); + + let remaining = remaining?; + // Skip if already paused, e.g. when migrating a paused VM. if vm.get_state() != VmState::Paused { info!("pausing VM"); From 1800bddddef9f7a5469b9e0b168d44bb97ac0159 Mon Sep 17 00:00:00 2001 From: Leander Kohler Date: Tue, 18 Aug 2026 16:37:06 +0200 Subject: [PATCH 78/79] vmm: fix migration sender hang on worker failure A single failing parallel migration connection can deadlock the migration sender. When the bounded send-channel is full, `SendAdditionalConnections::cleanup()` silently drops the Disconnect messages via the non-blocking `try_send`. The surviving workers drain the channel and then block forever in `recv()`, while the main thread blocks forever in `join()`. The VM stays stuck in the "migrating" state. We fix this by sending the `Disconnect` messages with a blocking send, ensuring the messages reach the migration workers. On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler Signed-off-by: Philipp Schuster --- vmm/src/migration/transport.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmm/src/migration/transport.rs b/vmm/src/migration/transport.rs index 95ea03b9fa..42c9017fff 100644 --- a/vmm/src/migration/transport.rs +++ b/vmm/src/migration/transport.rs @@ -885,7 +885,7 @@ impl SendAdditionalConnections { // All threads may have terminated, leading to a dropped receiver. Thus we ignore // errors here. self.message_tx - .try_send(SendMemoryThreadMessage::Disconnect) + .send(SendMemoryThreadMessage::Disconnect) .ok(); } From a562ac21700c1a6fce4dac2e6ca06b12c73361a7 Mon Sep 17 00:00:00 2001 From: Leander Kohler Date: Tue, 18 Aug 2026 16:37:06 +0200 Subject: [PATCH 79/79] vmm: skip pending memory sends on worker failure When a memory-sending worker fails, the migration thread calls `SendAdditionalConnections::cleanup()`. Cleanup tries to enqueue one `Disconnect` message per worker, but the bounded channel may still contain `SendMemoryThreadMessage::Memory` values. The surviving workers currently call `send_memory_ranges()` for each such value, even though the migration has already failed. Skip that call when `worker_error` is set. This consumes the queued values without processing them and lets cleanup enqueue the `Disconnect` messages. On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler Signed-off-by: Philipp Schuster --- vmm/src/migration/transport.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vmm/src/migration/transport.rs b/vmm/src/migration/transport.rs index 42c9017fff..dbaf785c02 100644 --- a/vmm/src/migration/transport.rs +++ b/vmm/src/migration/transport.rs @@ -727,7 +727,9 @@ impl SendAdditionalConnections { })?; match message { SendMemoryThreadMessage::Memory(table) => { - if external_cancel.load(Ordering::Acquire) { + if external_cancel.load(Ordering::Acquire) + || worker_error.load(Ordering::Acquire) + { continue; }