fix(code-runner): 容器清理失败重试+日志告警,避免静默泄漏
ContainerGuard::drop 之前用 let _ = 吞掉 remove_container 的错误, Docker daemon 瞬时不可用会让容器泄漏且无人知晓。 改为最多重试 3 次(指数退避 200/400/800ms),仍失败则记录 error 级日志 (带 container_id 便于 docker rm -f 兜底)。容器清理是 fire-and-forget, 无法把错误回传业务层,所以用日志暴露故障而非 panic。
This commit is contained in:
parent
ee9d2ecd6e
commit
cc4c551aaa
@ -61,12 +61,42 @@ impl Drop for ContainerGuard {
|
||||
fn drop(&mut self) {
|
||||
let docker = self.docker.clone();
|
||||
let container_id = self.container_id.clone();
|
||||
// 容器清理是 fire-and-forget:调用方已返回,无法把错误回传给业务层。
|
||||
// 因此重试几次以抵抗瞬时故障(daemon 繁忙 / socket 抖动),
|
||||
// 仍失败则记录 error 级日志并带上 container_id,便于运维手动 `docker rm -f` 清理,
|
||||
// 避免容器静默泄漏、长期堆积。
|
||||
tokio::spawn(async move {
|
||||
let remove_options = Some(RemoveContainerOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
});
|
||||
let _ = docker.remove_container(&container_id, remove_options).await;
|
||||
let max_attempts = 3u8;
|
||||
let mut backoff = Duration::from_millis(200);
|
||||
for attempt in 1..=max_attempts {
|
||||
let remove_options = Some(RemoveContainerOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
});
|
||||
match docker.remove_container(&container_id, remove_options).await {
|
||||
Ok(()) => return,
|
||||
Err(e) if attempt < max_attempts => {
|
||||
tracing::warn!(
|
||||
attempt,
|
||||
max_attempts,
|
||||
"remove_container 失败,稍后重试: {:?}",
|
||||
e
|
||||
);
|
||||
tokio::time::sleep(backoff).await;
|
||||
backoff *= 2;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
container_id = %container_id,
|
||||
"重试 {} 次后仍无法删除容器,可能泄漏;请手动执行 `docker rm -f {}`: {:?}",
|
||||
max_attempts,
|
||||
container_id,
|
||||
e
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user