fix(prover): Remove deleted pods from autoscaler-agent cluster cache (matter-labs#3739)

## What ❔

Remove deleted pods from autoscaler-agent cluster cache.

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->

## Why ❔

Pods deleted in status Pending could cause incorrect evaluation of the
cluster capacity.
<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- The `Why` has to be clear to non-Matter Labs entities running their
own ZK Chain -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Is this a breaking change?
- [ ] Yes
- [x] No

## Operational changes
<!-- Any config changes? Any new flags? Any changes to any scripts? -->
<!-- Please add anything that non-Matter Labs entities running their own
ZK Chain may need to know -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [x] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [x] Code has been formatted via `zkstack dev fmt` and `zkstack dev
lint`.

Author: yorik

prover/Cargo.lock

@@ -33,8 +33,8 @@ humantime = "2.1"
 humantime-serde = "1.1"
 indicatif = "0.16"
 jemallocator = "0.5"
-k8s-openapi = { version = "0.23.0", features = ["v1_30"] }
-kube = { version = "0.96.0", features = ["runtime", "derive"] }
+k8s-openapi = { version = "0.24.0", features = ["v1_30"] }
+kube = { version = "0.99.0", features = ["runtime", "derive"] }
 log = "0.4.20"
 md5 = "0.7.0"
 once_cell = "1.18"

prover/Cargo.toml

@@ -398,7 +398,7 @@ mod tests {
 
     #[tracing_test::traced_test]
     #[test]
-    fn test_run() {
+    fn test_calculate() {
         let scaler = Scaler::<GpuKey>::new(
             QueueReportFields::prover_jobs,
             "circuit-prover-gpu".into(),
@@ -540,7 +540,7 @@ mod tests {
 
     #[tracing_test::traced_test]
     #[test]
-    fn test_run_min_provers() {
+    fn test_calculate_min_provers() {
         let scaler = Scaler::new(
             QueueReportFields::prover_jobs,
             "circuit-prover-gpu".into(),
@@ -739,7 +739,7 @@ mod tests {
 
     #[tracing_test::traced_test]
     #[test]
-    fn test_run_need_move() {
+    fn test_calculate_need_move() {
         let scaler = Scaler::new(
             QueueReportFields::prover_jobs,
             "circuit-prover-gpu".into(),
@@ -859,7 +859,7 @@ mod tests {
 
     #[tracing_test::traced_test]
     #[test]
-    fn test_run_nokey() {
+    fn test_calculate_nokey() {
         let scaler = Scaler::<NoKey>::new(
             QueueReportFields::prover_jobs,
             "some-deployment".into(),

prover/crates/bin/prover_autoscaler/src/global/scaler.rs

@@ -7,6 +7,7 @@ use k8s_openapi::api;
 use kube::{
     api::{Api, ResourceExt},
     runtime::{watcher, WatchStreamExt},
+    Resource,
 };
 use reqwest::{
     header::{HeaderMap, HeaderValue},
@@ -172,8 +173,12 @@ impl Watcher {
                         }
                         pod.status = phase;
 
-                        if pod.status == "Succeeded" || pod.status == "Failed" {
+                        if p.meta().deletion_timestamp.is_some()
+                            || pod.status == "Succeeded"
+                            || pod.status == "Failed"
+                        {
                             // Cleaning up list of pods.
+                            tracing::debug!("Remove pod: {}", &p.name_any());
                             v.pods.remove(&p.name_any());
                         }