generate token until context filled

VJHack · Oct 28, 2024 · b4bf5df · b4bf5df
1 parent e0614ca
commit b4bf5df
Showing 1 changed file with 10 additions and 3 deletions.
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
@@ -219,14 +219,15 @@ struct server_slot {
         if (params.n_predict == -1 && global_params.n_predict == -1) {
             return true; // limitless
         }
+        else if (global_params.n_predict == -2) {
+            return true; // generate until context is filled
+        }
 
         n_remaining = -1;
 
         if (params.n_predict != -1) {
             n_remaining = params.n_predict - n_decoded;
-        } else if (global_params.n_predict == -2) {
-            n_remaining = n_ctx - n_past;
-        } else if (global_params.n_predict != -1) {
+        }else if (global_params.n_predict != -1) {
             n_remaining = global_params.n_predict - n_decoded;
         }
 
@@ -1814,6 +1815,12 @@ struct server_context {
                     continue;
                 }
 
+                if (params.n_predict == -2) {
+                    slot.release();
+                    send_final_response(slot);
+                    continue;
+                }
+
                 // Shift context
                 const int n_keep    = slot.params.n_keep + add_bos_token;
                 const int n_left    = slot.n_past - n_keep;