fix(code-runner): 容器清理失败重试+日志告警,避免静默泄漏
ContainerGuard::drop 之前用 let _ = 吞掉 remove_container 的错误, Docker daemon 瞬时不可用会让容器泄漏且无人知晓。 改为最多重试 3 次(指数退避 200/400/800ms),仍失败则记录 error 级日志 (带 container_id 便于 docker rm -f 兜底)。容器清理是 fire-and-forget, 无法把错误回传业务层,所以用日志暴露故障而非 panic。
This commit is contained in:
parent
ee9d2ecd6e
commit
cc4c551aaa
@ -61,12 +61,42 @@ impl Drop for ContainerGuard {
|
|||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
let docker = self.docker.clone();
|
let docker = self.docker.clone();
|
||||||
let container_id = self.container_id.clone();
|
let container_id = self.container_id.clone();
|
||||||
|
// 容器清理是 fire-and-forget:调用方已返回,无法把错误回传给业务层。
|
||||||
|
// 因此重试几次以抵抗瞬时故障(daemon 繁忙 / socket 抖动),
|
||||||
|
// 仍失败则记录 error 级日志并带上 container_id,便于运维手动 `docker rm -f` 清理,
|
||||||
|
// 避免容器静默泄漏、长期堆积。
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
let max_attempts = 3u8;
|
||||||
|
let mut backoff = Duration::from_millis(200);
|
||||||
|
for attempt in 1..=max_attempts {
|
||||||
let remove_options = Some(RemoveContainerOptions {
|
let remove_options = Some(RemoveContainerOptions {
|
||||||
force: true,
|
force: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
let _ = docker.remove_container(&container_id, remove_options).await;
|
match docker.remove_container(&container_id, remove_options).await {
|
||||||
|
Ok(()) => return,
|
||||||
|
Err(e) if attempt < max_attempts => {
|
||||||
|
tracing::warn!(
|
||||||
|
attempt,
|
||||||
|
max_attempts,
|
||||||
|
"remove_container 失败,稍后重试: {:?}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
tokio::time::sleep(backoff).await;
|
||||||
|
backoff *= 2;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!(
|
||||||
|
container_id = %container_id,
|
||||||
|
"重试 {} 次后仍无法删除容器,可能泄漏;请手动执行 `docker rm -f {}`: {:?}",
|
||||||
|
max_attempts,
|
||||||
|
container_id,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user