From 5db0d29bffee8ba24aee8786cb8bd641f2b48a05 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Thu, 24 Sep 2026 10:40:49 +0000 Subject: [PATCH 1/4] hibernating fix --- daemon/lib/source/DobbyManager.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/daemon/lib/source/DobbyManager.cpp b/daemon/lib/source/DobbyManager.cpp index 6c6ed2ae..06eb8182 100644 --- a/daemon/lib/source/DobbyManager.cpp +++ b/daemon/lib/source/DobbyManager.cpp @@ -1389,11 +1389,15 @@ bool DobbyManager::stopContainer(int32_t cd, bool withPrejudice) container->state == DobbyContainer::State::Hibernated || container->state == DobbyContainer::State::Awakening) { - // If a hibernation is in progress, abort it in a blocking manner before - // killing the container. This ensures memcr_worker fully unseizes any - // in-flight PID before it is killed, preventing an assert crash in - // memcr_worker when it tries to operate on a terminated PID. - if (container->state == DobbyContainer::State::Hibernating) + // If a hibernation is in progress or has entered the abort path, stop + // it in a blocking manner before killing the container. This ensures + // memcr_worker has fully unseized any in-flight PID before it is killed. + // Awakening alone is not enough to identify an in-flight hibernation + // abort: wakeupContainer() also sets Awakening before issuing WakeupProcess + // for a container that is already hibernated. Only treat Awakening as a + // hibernation abort when there is still a live in-flight PID associated + // with the active HibernateProcess() call. + if (container->state == DobbyContainer::State::Hibernating || (container->state == DobbyContainer::State::Awakening && container->hibernatingPid != 0)) { if (!abortContainerHibernationIfNeeded(cd)) { @@ -1860,11 +1864,15 @@ bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd) // assert(WIFSTOPPED(status)) inside memcr_worker. We must drive memcr to // unseize_target() for this PID before issuing killCont(). const uint32_t inflightPid = it->second->hibernatingPid; + AI_LOG_INFO("Abort hibernation pre-check for '%s': currentState=%d inFlightPid=%u", + id.c_str(), static_cast(it->second->state), inflightPid); // Set state to Awakening: the hibernate thread checks this at the top of // each PID loop iteration and will abort before starting any further // HibernateProcess() call. it->second->state = DobbyContainer::State::Awakening; + AI_LOG_INFO("Abort hibernation state transition for '%s': Hibernating -> Awakening (inFlightPid=%u)", + id.c_str(), inflightPid); if (inflightPid != 0) { From 805ef048f7fb070319f0e05f6a85837a49dbf288 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Mon, 28 Sep 2026 06:28:09 +0000 Subject: [PATCH 2/4] copilot suggestion --- daemon/lib/source/DobbyManager.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/daemon/lib/source/DobbyManager.cpp b/daemon/lib/source/DobbyManager.cpp index 06eb8182..d4a883e9 100644 --- a/daemon/lib/source/DobbyManager.cpp +++ b/daemon/lib/source/DobbyManager.cpp @@ -1392,12 +1392,13 @@ bool DobbyManager::stopContainer(int32_t cd, bool withPrejudice) // If a hibernation is in progress or has entered the abort path, stop // it in a blocking manner before killing the container. This ensures // memcr_worker has fully unseized any in-flight PID before it is killed. - // Awakening alone is not enough to identify an in-flight hibernation + // Awakening alone is not enough to identify an in-flight hibernation // abort: wakeupContainer() also sets Awakening before issuing WakeupProcess // for a container that is already hibernated. Only treat Awakening as a // hibernation abort when there is still a live in-flight PID associated // with the active HibernateProcess() call. - if (container->state == DobbyContainer::State::Hibernating || (container->state == DobbyContainer::State::Awakening && container->hibernatingPid != 0)) + if (container->state == DobbyContainer::State::Hibernating || + (container->state == DobbyContainer::State::Awakening && container->hibernatingPid != 0)) { if (!abortContainerHibernationIfNeeded(cd)) { @@ -1851,7 +1852,14 @@ bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd) const ContainerId id = it->first; - if (it->second->state != DobbyContainer::State::Hibernating) + // Awakening + hibernatingPid != 0 means wakeupContainer() raced ahead of the + // hibernate thread finishing its current PID: the state was already flipped + // but the in-flight PID hasn't been unseized yet, so it still needs an abort. + const bool inFlightAbortNeeded = + (it->second->state == DobbyContainer::State::Hibernating) || + (it->second->state == DobbyContainer::State::Awakening && it->second->hibernatingPid != 0); + + if (!inFlightAbortNeeded) { AI_LOG_INFO("Container '%s' is not hibernating, abort not needed", id.c_str()); AI_LOG_FN_EXIT(); @@ -1864,15 +1872,11 @@ bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd) // assert(WIFSTOPPED(status)) inside memcr_worker. We must drive memcr to // unseize_target() for this PID before issuing killCont(). const uint32_t inflightPid = it->second->hibernatingPid; - AI_LOG_INFO("Abort hibernation pre-check for '%s': currentState=%d inFlightPid=%u", - id.c_str(), static_cast(it->second->state), inflightPid); // Set state to Awakening: the hibernate thread checks this at the top of // each PID loop iteration and will abort before starting any further // HibernateProcess() call. it->second->state = DobbyContainer::State::Awakening; - AI_LOG_INFO("Abort hibernation state transition for '%s': Hibernating -> Awakening (inFlightPid=%u)", - id.c_str(), inflightPid); if (inflightPid != 0) { @@ -3740,3 +3744,4 @@ bool DobbyManager::shouldEnableSTrace(const std::shared_ptr &config return std::find(apps.begin(), apps.end(), hostName) != apps.end(); } + From aea895d0b831565ee4746c33dfbef462b59433ef Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Mon, 28 Sep 2026 09:44:20 +0000 Subject: [PATCH 3/4] release mlock --- daemon/lib/source/DobbyManager.cpp | 33 +++++++++++++++++++----- daemon/lib/source/include/DobbyManager.h | 3 ++- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/daemon/lib/source/DobbyManager.cpp b/daemon/lib/source/DobbyManager.cpp index d4a883e9..df87492f 100644 --- a/daemon/lib/source/DobbyManager.cpp +++ b/daemon/lib/source/DobbyManager.cpp @@ -1400,7 +1400,7 @@ bool DobbyManager::stopContainer(int32_t cd, bool withPrejudice) if (container->state == DobbyContainer::State::Hibernating || (container->state == DobbyContainer::State::Awakening && container->hibernatingPid != 0)) { - if (!abortContainerHibernationIfNeeded(cd)) + if (!abortContainerHibernationIfNeeded(cd, locker)) { AI_LOG_WARN("failed to abort hibernation for container %d", cd); AI_LOG_FN_EXIT(); @@ -1813,15 +1813,20 @@ bool DobbyManager::hibernateContainer(int32_t cd, const std::string& options) * which is written under mLock immediately before each HibernateProcess() call * and cleared under mLock when the full hibernation sequence completes. * - * mLock is held throughout this function, including during the WakeupProcess - * call. The hibernate thread is blocked inside HibernateProcess() (a memcr - * socket call) when inflightPid != 0 and does not hold mLock at that point, - * so there is no deadlock risk. + * mLock is released only around the blocking WakeupProcess socket call so + * that this call doesn't monopolize memcr (a resource shared with other + * memcr clients, e.g. JSPP) for the whole duration of the round-trip. It is + * re-acquired immediately afterwards before touching container state. The + * hibernate thread is blocked inside HibernateProcess() (a memcr socket + * call) when inflightPid != 0 and does not hold mLock at that point, so + * there is no deadlock risk from releasing/reacquiring here. * * Unlike wakeupContainer(), this runs entirely on the calling thread (no new * thread spawned) and does not emit the awoken callback. * - * @param[in] cd The descriptor of the container to abort hibernation for. + * @param[in] cd The descriptor of the container to abort hibernation for. + * @param[in,out] locker The caller's lock on mLock; briefly released around + * the blocking WakeupProcess call. * * @return true on success (or if no abort was needed); false if: * - the container was not found by descriptor at entry, or @@ -1831,7 +1836,7 @@ bool DobbyManager::hibernateContainer(int32_t cd, const std::string& options) * state != Hibernating and abort its loop). * Callers must not proceed with killCont() when false is returned. */ -bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd) +bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd, std::unique_lock& locker) { AI_LOG_FN_ENTRY(); @@ -1891,7 +1896,21 @@ bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd) // Future PIDs will not be reached because state is now Awakening. AI_LOG_INFO("Aborting hibernation of '%s': sending WakeupProcess for in-flight PID %u", id.c_str(), inflightPid); + // Release mLock for the blocking memcr round-trip: this is a shared, + // serialized resource also used by other memcr clients, and holding + // the lock here otherwise starves them for the round-trip duration. + locker.unlock(); const DobbyHibernate::Error wakeRet = DobbyHibernate::WakeupProcess(static_cast(inflightPid)); + locker.lock(); + + // Container may have been removed while we were unlocked; re-find it. + it = mContainers.find(id); + if (it == mContainers.cend()) + { + AI_LOG_WARN("container '%s' disappeared while aborting hibernation", id.c_str()); + AI_LOG_FN_EXIT(); + return false; + } if (wakeRet != DobbyHibernate::Error::ErrorNone) { diff --git a/daemon/lib/source/include/DobbyManager.h b/daemon/lib/source/include/DobbyManager.h index aa688890..527a5a8b 100644 --- a/daemon/lib/source/include/DobbyManager.h +++ b/daemon/lib/source/include/DobbyManager.h @@ -182,7 +182,7 @@ class DobbyManager bool restartContainer(const ContainerId& id, const std::unique_ptr& container); - bool abortContainerHibernationIfNeeded(int32_t cd); + bool abortContainerHibernationIfNeeded(int32_t cd, std::unique_lock& locker); private: ContainerStartedFunc mContainerStartedCb; @@ -252,3 +252,4 @@ class DobbyManager #endif // !defined(DOBBYMANAGER_H) + From 1917cf000a66f48e2d81655a827c5f7be92c04e8 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Mon, 28 Sep 2026 11:31:55 +0000 Subject: [PATCH 4/4] Revert "release mlock" This reverts commit aea895d0b831565ee4746c33dfbef462b59433ef. --- daemon/lib/source/DobbyManager.cpp | 33 +++++------------------- daemon/lib/source/include/DobbyManager.h | 3 +-- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/daemon/lib/source/DobbyManager.cpp b/daemon/lib/source/DobbyManager.cpp index df87492f..d4a883e9 100644 --- a/daemon/lib/source/DobbyManager.cpp +++ b/daemon/lib/source/DobbyManager.cpp @@ -1400,7 +1400,7 @@ bool DobbyManager::stopContainer(int32_t cd, bool withPrejudice) if (container->state == DobbyContainer::State::Hibernating || (container->state == DobbyContainer::State::Awakening && container->hibernatingPid != 0)) { - if (!abortContainerHibernationIfNeeded(cd, locker)) + if (!abortContainerHibernationIfNeeded(cd)) { AI_LOG_WARN("failed to abort hibernation for container %d", cd); AI_LOG_FN_EXIT(); @@ -1813,20 +1813,15 @@ bool DobbyManager::hibernateContainer(int32_t cd, const std::string& options) * which is written under mLock immediately before each HibernateProcess() call * and cleared under mLock when the full hibernation sequence completes. * - * mLock is released only around the blocking WakeupProcess socket call so - * that this call doesn't monopolize memcr (a resource shared with other - * memcr clients, e.g. JSPP) for the whole duration of the round-trip. It is - * re-acquired immediately afterwards before touching container state. The - * hibernate thread is blocked inside HibernateProcess() (a memcr socket - * call) when inflightPid != 0 and does not hold mLock at that point, so - * there is no deadlock risk from releasing/reacquiring here. + * mLock is held throughout this function, including during the WakeupProcess + * call. The hibernate thread is blocked inside HibernateProcess() (a memcr + * socket call) when inflightPid != 0 and does not hold mLock at that point, + * so there is no deadlock risk. * * Unlike wakeupContainer(), this runs entirely on the calling thread (no new * thread spawned) and does not emit the awoken callback. * - * @param[in] cd The descriptor of the container to abort hibernation for. - * @param[in,out] locker The caller's lock on mLock; briefly released around - * the blocking WakeupProcess call. + * @param[in] cd The descriptor of the container to abort hibernation for. * * @return true on success (or if no abort was needed); false if: * - the container was not found by descriptor at entry, or @@ -1836,7 +1831,7 @@ bool DobbyManager::hibernateContainer(int32_t cd, const std::string& options) * state != Hibernating and abort its loop). * Callers must not proceed with killCont() when false is returned. */ -bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd, std::unique_lock& locker) +bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd) { AI_LOG_FN_ENTRY(); @@ -1896,21 +1891,7 @@ bool DobbyManager::abortContainerHibernationIfNeeded(int32_t cd, std::unique_loc // Future PIDs will not be reached because state is now Awakening. AI_LOG_INFO("Aborting hibernation of '%s': sending WakeupProcess for in-flight PID %u", id.c_str(), inflightPid); - // Release mLock for the blocking memcr round-trip: this is a shared, - // serialized resource also used by other memcr clients, and holding - // the lock here otherwise starves them for the round-trip duration. - locker.unlock(); const DobbyHibernate::Error wakeRet = DobbyHibernate::WakeupProcess(static_cast(inflightPid)); - locker.lock(); - - // Container may have been removed while we were unlocked; re-find it. - it = mContainers.find(id); - if (it == mContainers.cend()) - { - AI_LOG_WARN("container '%s' disappeared while aborting hibernation", id.c_str()); - AI_LOG_FN_EXIT(); - return false; - } if (wakeRet != DobbyHibernate::Error::ErrorNone) { diff --git a/daemon/lib/source/include/DobbyManager.h b/daemon/lib/source/include/DobbyManager.h index 527a5a8b..aa688890 100644 --- a/daemon/lib/source/include/DobbyManager.h +++ b/daemon/lib/source/include/DobbyManager.h @@ -182,7 +182,7 @@ class DobbyManager bool restartContainer(const ContainerId& id, const std::unique_ptr& container); - bool abortContainerHibernationIfNeeded(int32_t cd, std::unique_lock& locker); + bool abortContainerHibernationIfNeeded(int32_t cd); private: ContainerStartedFunc mContainerStartedCb; @@ -252,4 +252,3 @@ class DobbyManager #endif // !defined(DOBBYMANAGER_H) -