fix eviction logic

Signed-off-by: Praneeth Bedapudi <[email protected]>
notAI-tech · Dec 12, 2023 · b24c02a · b24c02a
1 parent 47e01a1
commit b24c02a
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 4 deletions.
diff --git a/fastdeploy/_loop.py b/fastdeploy/_loop.py
@@ -68,16 +68,19 @@ def start_loop(
             # delete older than 15 min, all successful and returned predictions from main index
             _utils.MAIN_INDEX.delete(
                 query={
+                    f"-1.predicted_at": {
+                        "$ne": 0,
+                        "$lt": time.time() - 15 * 60,
+                    },
                     "last_predictor_success": True,
-                    "-1.predicted_at": {"$lt": time.time() - 900},
                 }
             )
             __last_deletion_run_at = time.time()
 
         for unique_id, data in _utils.MAIN_INDEX.search(
             query={
-                "last_predictor_sequence": predictor_sequence - 1,
                 "last_predictor_success": True,
+                "last_predictor_sequence": predictor_sequence - 1,
             },
             n=optimal_batch_size,
             select_keys=[
@@ -117,6 +120,13 @@ def start_loop(
             _utils.logger.exception(ex, exc_info=True)
             results = [None] * current_batch_length
 
+        if len(results) != current_batch_length:
+            raise Exception(
+                f"Predictor returned {len(results)} results for {current_batch_length} inputs",
+                input_batch,
+                results,
+            )
+
         predicted_at = time.time()
 
         unique_id_wise_results = {}

diff --git a/fastdeploy/_rest.py b/fastdeploy/_rest.py
@@ -63,7 +63,7 @@ def on_get(self, req, resp):
 
         # HELP successful_requests The number of failed requests.
         # TYPE successful_requests gauge
-        successful_requests {_utils.MAIN_INDEX.count(query={"last_predictor_success": True, "-1.predicted_at": {"$gt": 0}})}
+        successful_requests {_utils.MAIN_INDEX.count(query={"-1.predicted_at": {"$ne": 0}, "last_predictor_success": True})}
         """
 
         resp.status = falcon.HTTP_200

diff --git a/fastdeploy/_utils.py b/fastdeploy/_utils.py
@@ -86,7 +86,8 @@
     db_path=os.path.join("fastdeploy_dbs", f"main_index.db"),
 )
 
-MAIN_INDEX.optimize_for_query(["last_predictor_sequence", "last_predictor_success"])
+MAIN_INDEX.optimize_for_query(["last_predictor_success", "last_predictor_sequence"])
+MAIN_INDEX.optimize_for_query(["-1.predicted_at", "last_predictor_success"])
 
 
 def warmup(predictor, example_input, n=3):