fix(code-runner): 容器清理失败重试+日志告警,避免静默泄漏

ContainerGuard::drop 之前用 let _ = 吞掉 remove_container 的错误,
Docker daemon 瞬时不可用会让容器泄漏且无人知晓。

改为最多重试 3 次(指数退避 200/400/800ms),仍失败则记录 error 级日志
(带 container_id 便于 docker rm -f 兜底)。容器清理是 fire-and-forget,
无法把错误回传业务层,所以用日志暴露故障而非 panic。
This commit is contained in:
xfy 2026-07-13 17:56:42 +08:00
parent ee9d2ecd6e
commit cc4c551aaa

View File

@ -61,12 +61,42 @@ impl Drop for ContainerGuard {
fn drop(&mut self) {
let docker = self.docker.clone();
let container_id = self.container_id.clone();
// 容器清理是 fire-and-forget调用方已返回无法把错误回传给业务层。
// 因此重试几次以抵抗瞬时故障daemon 繁忙 / socket 抖动),
// 仍失败则记录 error 级日志并带上 container_id便于运维手动 `docker rm -f` 清理,
// 避免容器静默泄漏、长期堆积。
tokio::spawn(async move {
let max_attempts = 3u8;
let mut backoff = Duration::from_millis(200);
for attempt in 1..=max_attempts {
let remove_options = Some(RemoveContainerOptions {
force: true,
..Default::default()
});
let _ = docker.remove_container(&container_id, remove_options).await;
match docker.remove_container(&container_id, remove_options).await {
Ok(()) => return,
Err(e) if attempt < max_attempts => {
tracing::warn!(
attempt,
max_attempts,
"remove_container 失败,稍后重试: {:?}",
e
);
tokio::time::sleep(backoff).await;
backoff *= 2;
}
Err(e) => {
tracing::error!(
container_id = %container_id,
"重试 {} 次后仍无法删除容器,可能泄漏;请手动执行 `docker rm -f {}`: {:?}",
max_attempts,
container_id,
e
);
return;
}
}
}
});
}
}