Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/tests-slow-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ jobs:
cli_cluster::core_live_apply_captures_server_file_root_without_local_storage_access \
cli_cluster::managed_lifecycle_wait_reports_tombstone_and_checks_every_poll_identity \
cli_cluster::managed_plan_and_apply_poll_original_identity_without_resubmission_or_cancellation \
cli_cluster::managed_plan_and_apply_submissions_wait_until_their_local_deadline \
cli_cluster::managed_native_outcomes_preserve_activation_archive_and_uncertainty \
cli_cluster::managed_poll_identity_mismatch_retains_original_delivery_and_native_ids \
cli_cluster::managed_lifecycle_bad_acceptance_and_deadline_keep_recovery_identity \
Expand Down
3 changes: 3 additions & 0 deletions changelog.d/managed-plan-apply-wait-for-timeout.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
- Managed `cluster plan` and `cluster apply` now wait for their submission until the `--timeout` deadline instead of a fixed 10 seconds. See [managed configuration][managed-plan-apply-timeout].

[managed-plan-apply-timeout]: ../docs/user/cli/managed-lifecycle.md
16 changes: 15 additions & 1 deletion crates/omnigraph-cli/src/managed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -952,7 +952,21 @@ impl Api {
body: Option<&Value>,
key: Option<&str>,
) -> Result<Value> {
let response = self.raw(method, path, body, key).await?;
self.request_with_timeout(method, path, body, key, REQUEST_TIMEOUT)
.await
}

async fn request_with_timeout(
&self,
method: Method,
path: &str,
body: Option<&Value>,
key: Option<&str>,
timeout: Duration,
) -> Result<Value> {
let response = self
.raw_with_timeout(method, path, body, key, timeout)
.await?;
if response.status.is_success() {
Ok(response.body)
} else {
Expand Down
30 changes: 28 additions & 2 deletions crates/omnigraph-cli/src/managed/deployment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -183,11 +183,37 @@ async fn submit(
stderr: &mut dyn Write,
) -> Result<(Value, String)> {
let key = idempotency_key(run.idempotency_key.as_deref(), stderr)?;
let request = api.request(Method::POST, path, Some(body), Some(&key));
// A submission is answered only after the service's own work, such as a
// plan's preview capture, which can outlast the bound of an ordinary
// control call. The local deadline bounds the request instead.
let request = api.request_with_timeout(
Method::POST,
path,
Some(body),
Some(&key),
deadline.saturating_duration_since(Instant::now()),
);
let deadline_reached = || {
Failure::new(
"wait_timeout",
"local submission deadline reached; acceptance is unknown; retain the original idempotency key",
5,
)
};
let result = if run.no_wait {
request.await
} else {
tokio::time::timeout_at(deadline, request).await.unwrap_or_else(|_| Err(Failure::new("wait_timeout", "local submission deadline reached; acceptance is unknown; retain the original idempotency key", 5)))
match tokio::time::timeout_at(deadline, request).await {
// The request's own bound ends no earlier than the deadline and
// can fire in the same timer tick: that is the deadline too.
Ok(Err(error))
if error.body["type"] == "transport_failed" && Instant::now() >= deadline =>
{
Err(deadline_reached())
}
Ok(result) => result,
Err(_) => Err(deadline_reached()),
}
};
result.map(|body| (body, key.clone())).map_err(|error| {
if error.exit == 2
Expand Down
80 changes: 80 additions & 0 deletions crates/omnigraph-cli/tests/integration/cli_cluster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2001,6 +2001,86 @@ fn managed_plan_and_apply_submit_exact_native_intent_without_waiting() {
}
}

#[test]
#[ignore = "nightly: holds plan and apply replies past the real 10 s control request bound"]
fn managed_plan_and_apply_submissions_wait_until_their_local_deadline() {
// A submission is answered only after the service's own work, so
// `--timeout` bounds it, not the bound of an ordinary control call.
let slow = std::time::Duration::from_millis(10_500);
for (arguments, path, expected, body, delay, code) in [
(
vec!["plan", "--rev", MANAGED_REVISION, "--timeout", "30"],
"/v1/clusters/managed-test/plans",
serde_json::json!({"revision":MANAGED_REVISION}),
managed_preview("ready"),
slow,
0,
),
(
vec![
"apply",
"--plan",
"saved-plan",
"--no-wait",
"--timeout",
"30",
],
"/v1/clusters/managed-test/deployments",
serde_json::json!({"preview_id":"saved-plan"}),
managed_delivery("queued"),
slow,
0,
),
(
vec!["plan", "--rev", MANAGED_REVISION, "--timeout", "1"],
"/v1/clusters/managed-test/plans",
serde_json::json!({"revision":MANAGED_REVISION}),
managed_preview("ready"),
std::time::Duration::from_millis(2_500),
5,
),
] {
let temp = tempdir().unwrap();
let api = IntentApiFixture::with_session_and_first_reply_delay(
vec![IntentReply::json(202, body.clone())],
lifecycle_session("principal-one", "account-one"),
delay,
);
write_managed_target(temp.path(), &api.origin);
let started = std::time::Instant::now();
let output = managed_cli(temp.path(), &api.origin)
.args(&arguments)
.args(["--idempotency-key", "slow-key", "--json"])
// The held reply alone outlasts most of the helper's 15 s bound.
.timeout(std::time::Duration::from_secs(40))
.output()
.unwrap();
assert_eq!(
output.status.code(),
Some(code),
"{arguments:?}: {output:?}"
);
let result = parse_stdout_json(&output);
if code == 0 {
assert_eq!(result, body);
assert!(started.elapsed() >= slow);
} else {
assert_eq!(result["type"], "wait_timeout");
assert_eq!(result["submission"]["idempotency_key"], "slow-key");
assert_eq!(result["submission"]["acceptance"], "unknown");
}
assert_control_request(
&api.workflow_requests()[0],
"POST",
path,
expected,
Some("slow-key"),
);
api.assert_complete();
assert_no_core_effects(temp.path());
}
}

#[test]
fn managed_plan_omitted_revision_captures_public_source_head_once() {
for state in ["ready", "capturing", "failed", "expired"] {
Expand Down
18 changes: 18 additions & 0 deletions crates/omnigraph-cli/tests/support/managed_http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,24 @@ impl IntentApiFixture {
Self::start(replies, Some(session), Duration::ZERO, false)
}

/// Hold the first scripted reply while the session lookup answers at once.
pub fn with_session_and_first_reply_delay(
replies: Vec<IntentReply>,
session: Value,
delay: Duration,
) -> Self {
assert!(delay <= Duration::from_secs(32), "fixture delay bound");
Self::start_with_origin(
|_| replies,
Some(session),
Duration::ZERO,
false,
None,
None,
Some((0, delay)),
)
}

/// Graph data fixtures answer public discovery without consuming a scripted
/// reply. Control-plane and OAuth fixtures retain their separate protocol.
pub fn graph(replies: Vec<IntentReply>) -> Self {
Expand Down
4 changes: 4 additions & 0 deletions docs/user/cli/managed-lifecycle.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ local wait (300 seconds by default). A timeout or disconnect does not cancel
the preview or submit another one. Progress goes to stderr; stdout contains
the final response, including the native plan when ready.

The `plan` and `apply` submissions also stay open until that deadline, so a
service that captures the preview before answering is not cut off. Each
source lookup and poll keeps its own 10-second bound.

To resume waiting, repeat `plan` with the same `--rev` and
`--idempotency-key` printed by the first invocation. The original preview's
deadline does not change. A `failed` or `expired` preview requires a fresh
Expand Down
Loading