Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 83 additions & 1 deletion src/phase_environment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,41 @@ pub struct PreparedEnvironment {
}

impl PreparedEnvironment {
pub(crate) fn attach_with_backend(
backend: Arc<dyn EnvironmentExecBackend>,
backend_label: String,
environment: &ResolvedEnvironment,
handle: EnvironmentHandle,
) -> Result<Self> {
let probe = HarnessCommand { program: "true".to_string(), args: Vec::new(), env: BTreeMap::new(), cwd: None };
let response = backend.exec(&handle, probe, None, Some(Duration::from_secs(10))).with_context(|| {
format!(
"retained publication environment '{}' (handle {}) is unavailable; refusing to prepare a \
replacement because it would lose the unpublished commit",
environment.id, handle.id
)
})?;
if response.timed_out || response.exit_code != Some(0) {
anyhow::bail!(
"retained publication environment '{}' (handle {}) failed its liveness probe \
(exit_code={:?}, timed_out={}); refusing to prepare a replacement because it would lose the \
unpublished commit",
environment.id,
handle.id,
response.exit_code,
response.timed_out
);
}
Ok(Self {
backend,
handle,
id: environment.id.clone(),
backend_label,
torn_down: AtomicBool::new(false),
runtime: None,
})
}

/// Resolve the environment plugin and prepare a BARE node for the whole run.
/// Blocking (the [`EnvironmentClient`] surface is blocking) — call via
/// [`Self::prepare_off_runtime`] from an async context.
Expand All @@ -326,7 +361,7 @@ impl PreparedEnvironment {
/// [`EnvironmentClient`]; tests inject a fake). Builds a BARE
/// [`EnvironmentSpec`] (no repos) and applies any routing-rule `spec`
/// overrides.
fn prepare_with_backend(
pub(crate) fn prepare_with_backend(
backend: Arc<dyn EnvironmentExecBackend>,
backend_label: String,
environment: &ResolvedEnvironment,
Expand Down Expand Up @@ -396,11 +431,58 @@ impl PreparedEnvironment {
rx.await.map_err(|_| anyhow!("environment prepare thread terminated unexpectedly"))?
}

/// Reattach to a retained publication node on a dedicated host runtime.
/// A retained handle is authoritative: when it cannot be probed, fail
/// loudly instead of preparing an empty replacement and losing the exact
/// unpublished commit/tree that caused the hold.
pub(crate) async fn attach_off_runtime(
project_root: &Path,
environment: &ResolvedEnvironment,
handle: EnvironmentHandle,
) -> Result<Self> {
let project_root = project_root.to_path_buf();
let environment = environment.clone();
let (tx, rx) = tokio::sync::oneshot::channel();
std::thread::spawn(move || {
let result = (|| {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.context("building dedicated runtime for the retained environment host")?;
let client = runtime.block_on(async {
EnvironmentClient::resolve(&project_root, &environment.id).map_err(|err| {
anyhow!(
"workflow has a retained publication environment '{}' but its plugin could not be \
resolved: {err}",
environment.id
)
})
})?;
let backend_label = format!("environment:{}", client.plugin_name());
let mut prepared = runtime.block_on(async {
Self::attach_with_backend(Arc::new(client), backend_label, &environment, handle)
})?;
prepared.runtime = Some(runtime);
Ok::<_, anyhow::Error>(prepared)
})();
let _ = tx.send(result);
});
rx.await.map_err(|_| anyhow!("environment attach thread terminated unexpectedly"))?
}

/// The resolved environment plugin id.
pub(crate) fn id(&self) -> &str {
&self.id
}

/// Serializable identity needed by restart reconciliation when publication
/// is paused. The caller persists this handle before releasing the resident
/// host so a later process can reattach to or explicitly reap the node.
pub(crate) fn handle(&self) -> &EnvironmentHandle {
&self.handle
}

/// Execute a phase's harness command INSIDE the held node — `exec_stream` on
/// the SAME pinned client + handle, with NO prepare and NO teardown (those
/// bracket the whole run). Returns a [`SessionRun`] whose event stream
Expand Down
73 changes: 73 additions & 0 deletions src/phase_git.rs
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,50 @@ pub fn redact_git_diagnostic(input: &str) -> String {
value
}

/// Whether a failed Git publication was rejected by a server-side
/// authorization or repository policy that requires operator action.
///
/// This deliberately excludes non-fast-forward and transport failures: those
/// retain the existing recovery-ref and retry behavior. Callers must also
/// scope this classifier to a publication phase so an unrelated command that
/// happens to contain one of these phrases is not paused.
pub(crate) fn is_non_retryable_publication_denial(input: &str) -> bool {
let message = input.to_ascii_lowercase();
[
"refusing to allow a github app to create or update workflow",
"workflows permission",
"resource not accessible by integration",
"permission to ",
"protected branch hook declined",
"protected branch update failed",
"repository rule violations",
"push declined due to repository rule",
"changes must be made through a pull request",
"not allowed to push",
"insufficient permission",
"insufficient scope",
"write access to repository not granted",
]
.iter()
.any(|needle| message.contains(needle))
}

pub(crate) fn publication_denial_escalation(diagnostic: &str, commit: Option<&str>, tree: Option<&str>) -> String {
let diagnostic = redact_git_diagnostic(diagnostic);
let commit = commit.filter(|value| !value.trim().is_empty()).unwrap_or("unavailable");
let tree = tree.filter(|value| !value.trim().is_empty()).unwrap_or("unavailable");
format!(
"Git publication is blocked by a non-retryable authorization or repository-policy denial. \
The execution environment is retained and no code rework was consumed. \
Unpublished commit: {commit}; tree: {tree}. \
Redacted remote diagnostic: {diagnostic}. \
Remediation: grant the publishing GitHub App/token permission to update the rejected ref \
(including Actions workflows permission when workflow files changed), or adjust the \
repository/branch policy, then explicitly resume publication. The environment must remain \
held until durable publication or explicit workflow cancellation."
)
}

fn sanitize_ref_component(value: &str) -> String {
let cleaned: String = value
.chars()
Expand Down Expand Up @@ -615,6 +659,35 @@ mod publication_tests {
assert!(redacted.contains("[REDACTED]"));
}

#[test]
fn publication_authorization_denials_are_non_retryable_but_collisions_and_transport_are_not() {
assert!(is_non_retryable_publication_denial(
"remote: refusing to allow a GitHub App to create or update workflow `.github/workflows/ci.yml` \
without `workflows` permission"
));
assert!(is_non_retryable_publication_denial(
"remote: error: GH013: Repository rule violations found for refs/heads/main"
));
assert!(!is_non_retryable_publication_denial("! [rejected] reviewed -> reviewed (non-fast-forward)"));
assert!(!is_non_retryable_publication_denial(
"fatal: unable to access 'https://github.com/o/r': Could not resolve host"
));
}

#[test]
fn authorization_escalation_is_redacted_and_preserves_exact_git_identity() {
let escalation = publication_denial_escalation(
"fatal: https://secret-token@github.com/o/r Authorization: Bearer ghp_secret; workflows permission",
Some("1111111111111111111111111111111111111111"),
Some("2222222222222222222222222222222222222222"),
);
assert!(!escalation.contains("secret-token"));
assert!(!escalation.contains("ghp_secret"));
assert!(escalation.contains("1111111111111111111111111111111111111111"));
assert!(escalation.contains("2222222222222222222222222222222222222222"));
assert!(escalation.contains("explicit workflow cancellation"));
}

#[test]
fn delegated_proof_is_revalidated_by_git_and_rejects_tampering() {
let (root, remote, work) = fixture();
Expand Down
68 changes: 68 additions & 0 deletions src/phase_session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,44 @@ pub fn find_reusable_binding(
Ok(None)
}

/// Find the authoritative node retained after a non-retryable publication
/// denial. Unlike generic restart reuse, this only accepts a blocked checkpoint
/// without positive publication evidence. Its handle must never be replaced by
/// a newly prepared node: the retained workspace is where the reviewed,
/// unpublished commit and tree live.
pub fn find_retained_publication_binding(
scoped_root: &Path,
workflow_id: &str,
environment_id: &str,
) -> io::Result<Option<EnvironmentBinding>> {
let phases_dir = scoped_root.join("runs").join(sanitize(workflow_id)).join("phases");
let entries = match fs::read_dir(&phases_dir) {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err),
};
for entry in entries {
let entry = entry?;
let path = entry.path();
if !path.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.ends_with(".session.json")) {
continue;
}
if let Some(checkpoint) = read_path(&path)? {
let is_publication_hold =
checkpoint.status == SessionCheckpointStatus::Blocked && checkpoint.publication_durable != Some(true);
if !is_publication_hold {
continue;
}
if let Some(binding) = checkpoint.environment {
if !binding.torn_down && binding.environment_id == environment_id {
return Ok(Some(binding));
}
}
}
}
Ok(None)
}

pub fn update_session_running_after_resume(
scoped_root: &Path,
workflow_id: &str,
Expand Down Expand Up @@ -634,6 +672,36 @@ mod tests {
assert!(find_reusable_binding(scoped_root, "wf-reuse", "animus-environment-railway").expect("scan").is_none());
}

#[test]
fn retained_publication_binding_requires_blocked_unpublished_checkpoint() {
let temp = tempdir().expect("tempdir");
let scoped_root = temp.path();
write_session_pending(scoped_root, "wf-held", "code-open-pr", "environment", "run-held", None)
.expect("pending");
update_session_environment(scoped_root, "wf-held", "code-open-pr", sample_binding()).expect("binding");

assert!(
find_retained_publication_binding(scoped_root, "wf-held", "animus-environment-railway")
.expect("scan pending")
.is_none(),
"an active checkpoint is not a publication hold"
);

update_session_blocked(scoped_root, "wf-held", "code-open-pr", "repository policy denied").expect("block");
let retained = find_retained_publication_binding(scoped_root, "wf-held", "animus-environment-railway")
.expect("scan blocked")
.expect("blocked unpublished binding");
assert_eq!(retained.handle.id, "node-abc");

update_publication_durable(scoped_root, "wf-held", "code-open-pr", true).expect("durable");
assert!(
find_retained_publication_binding(scoped_root, "wf-held", "animus-environment-railway")
.expect("scan durable")
.is_none(),
"positive publication proof releases the hold"
);
}

// Backward-compat: a checkpoint JSON written by an OLDER runner that does not
// know the `environment` field must still deserialize.
#[test]
Expand Down
Loading
Loading