[Monitoring] Update auto scale worker script (#4235)

* [Monitoring] Remove excluded challenges from auto scaling

* Auto scale non-remote workers

* Fix flake8 issue

scripts/monitoring/auto_scale_workers.py

@@ -10,28 +10,6 @@
 warnings.filterwarnings("ignore")
 
 utc = pytz.UTC
-PROD_EXCLUDED_CHALLENGE_QUEUES = [
-    "textvqa-challenge-2021-874-production-41966973-4d99-4326-a402-b749b1d89aad",
-    "vqa-challenge-2021-830-production-6343db53-af82-4618-8c51-0e294611315a",
-    "ego4d-state-change-object-detection-challenge-1632",
-    "argoverse-3d-tracking-competition-52b36364-110d-47f1-8fa7-8873aa2d9965",
-    "argoverse-motion-forecasting-competition-81d1a3c6-f7b3-4830-9dcf-3ccc9f29d3a6",
-    "argoverse-3d-detection-competition-725-71837a72-afb0-4403-9cfc-bb3e80d733ab",
-    "argoverse-stereo-competition-917-production-884becea-a882-43bd-8ac7-0561bd705c4e",
-    "nuscenes-prediction-challenge-448a7b5e-6b08-4587-9293-ac07a530b426",
-    "nuscenes-detection-challenge-510c8c6d-a0d2-40bd-95dd-b7c8ea593d03",
-    "gqa-real-world-visual-reasoning-challenge-2019-2e1c901b-5d86-4516-b82d-8d98",
-    "nocaps-xd-9f4bead9-b3d6-4207-8e68-a6807d786c3c",
-    "nocaps-18b403b5-8946-4319-9fc1-758981f7f724",
-    "nuscenes-tracking-challenge-b1ea8e46-cee9-4591-856a-7c31947d74ed",
-    "ego4d-poc-your-challenge-name--1637-production-bfb81565-71fd-4b7b-949d-2560dd523",
-    "ego4d-poc-your-challenge-name--1624-production-6e1419b9-f909-4428-951d-6e927676f",
-    "dialoglue-708-d2225333-1ade-41c5-bb0c-c02c15d43e05",
-    "vizwiz-caption-challenge-2021-739-production-a6420029-bf1c-4339-9c55-36a8913cca8",
-    "textcaps-challenge-2020-906-production-fcc74455-ce3e-4622-b6fe-0340bcc3d228",
-    "vizwiz-vqa-challenge-2021-743-production-862e0cf8-0611-4aa5-a05a-14e07576a513",
-    "cvpr2023-bdd100k-multiple-object-tracking-challeng-1989-production-cdba4595-fb13",
-]
 
 ENV = os.environ.get("ENV", "dev")
 
@@ -113,11 +91,11 @@ def scale_up_or_down_workers(challenge, challenge_metrics):
 def scale_up_or_down_workers_for_challenges(response, metrics):
     for challenge in response["results"]:
         if ENV == "prod":
-            if challenge["queue"] not in PROD_EXCLUDED_CHALLENGE_QUEUES:
-                try:
+            try:
+                if challenge["remote_evaluation"] is False:
                     scale_up_or_down_workers(challenge, metrics[str(challenge["id"])])
-                except Exception as e:
-                    print(e)
+            except Exception as e:
+                print(e)
         else:
             try:
                 scale_up_or_down_workers(challenge, metrics[str(challenge["id"])])