performance upgrades

Signed-off-by: Praneeth Bedapudi <[email protected]>

fastdeploy/_infer.py

@@ -215,7 +215,18 @@ def infer(
                         == _utils.LAST_PREDICTOR_SEQUENCE
                     ):
                         _utils.MAIN_INDEX.update(
-                            {unique_id: {"-1.predicted_at": time.time()}}
+                            {
+                                unique_id: {
+                                    **{
+                                        "-1.predicted_at": time.time(),
+                                        "-1.outputs": None,
+                                    },
+                                    **{
+                                        f"{__}.outputs": None
+                                        for __ in _utils.PREDICTOR_SEQUENCE_TO_FILES
+                                    },
+                                }
+                            }
                         )
 
                         _utils.logger.debug(f"{unique_id}: predictor finished")
@@ -257,6 +268,22 @@ def infer(
                             _utils.logger.debug(
                                 f"{unique_id}: predictor timedout at {current_results['last_predictor_sequence']}"
                             )
+
+                            _utils.MAIN_INDEX.update(
+                                {
+                                    unique_id: {
+                                        **{
+                                            f"{_}.outputs": None
+                                            for _ in _utils.PREDICTOR_SEQUENCE_TO_FILES
+                                        },
+                                        **{
+                                            "-1.outputs": None,
+                                            "-1.predicted_at": time.time(),
+                                        },
+                                    }
+                                }
+                            )
+
                             return self.create_response(
                                 unique_id,
                                 {

fastdeploy/_loop.py

@@ -94,12 +94,12 @@ def update_main_index(unique_id_wise_results: Dict[str, Dict[str, Any]]):
 
 def perform_maintenance(main_index: Any):
     """Perform maintenance tasks on the main index."""
-    # Delete older than 15 min, all successful and returned predictions from main index
+    # Delete older than 7 seconds, all successful and returned predictions from main index
     main_index.delete(
         query={
             f"-1.predicted_at": {
                 "$ne": 0,
-                "$lt": time.time() - 15 * 60,
+                "$lt": time.time() - 7,
             },
             "last_predictor_success": True,
         }

fastdeploy/_rest.py

@@ -58,7 +58,7 @@ def on_post(self, req, resp):
 class PrometheusMetrics(object):
     def on_get(self, req, resp):
         _LAST_X_SECONDS = int(
-            req.params.get("last_x_seconds", int(os.getenv("LAST_X_SECONDS", 60)))
+            req.params.get("last_x_seconds", int(os.getenv("LAST_X_SECONDS", 5)))
         )
         CURRENT_TIME = time.time()
         LAST_X_SECONDS = time.time() - _LAST_X_SECONDS

fastdeploy/_utils.py

@@ -69,6 +69,7 @@
 KV_STORE = KVIndex(os.path.join("fastdeploy_dbs", f"kv_store.db"))
 KV_STORE.clear()
 
+
 MAIN_INDEX = DefinedIndex(
     "main_index",
     schema={

setup.py

@@ -18,7 +18,7 @@
 EMAIL = "[email protected]"
 AUTHOR = "BEDAPUDI PRANEETH"
 REQUIRES_PYTHON = ">=3.6.0"
-VERSION = "3.0.15"
+VERSION = "3.0.16"
 
 # What packages are required for this module to be executed?
 REQUIRED = ["falcon", "liteindex", "zstandard", "gunicorn[gevent]", "msgpack"]