From 4c744e7b0434d41154c97ba40181eeac693bd866 Mon Sep 17 00:00:00 2001
From: Alind Khare <alindkhare@gatech.edu>
Date: Wed, 21 Feb 2024 14:10:28 -0500
Subject: [PATCH] Update Dynamic Discretization to enable Warm Starts  (#87)

* Fixed Capacity Constraint Map for dynamic Discretization

* Implemented STRL choose exprs reuse from prev sovler invocation in dynamic discretization

* Added finer Grained discretization around prev solutions

* Added input of lower discretization around prev sol

* Rectified Bug

* Added limitation of total solver time to 1 min

* changed time and added more info in errors

* Rectified Dynamic Discretization Bugs

* Implemented getResourceQuantity() for allocation exprs

* Added conf file for dynamic discretization

* Format changes to pass builds.

* Format changes to main to fix builds.

* Changes to pass flake8 compatibility.

---------

Co-authored-by: Sukrit Kalra <sukritkalra@gmail.com>
---
 ...w_motiv_new_conf_sukrit_runs_strl_1_8.conf |  45 ++++++++
 ...w_conf_sukrit_runs_strl_1_8_selective.conf |  47 ++++++++
 main.py                                       |  17 +++
 profiles/workers/alibaba_cluster.yaml         |   2 +-
 .../include/tetrisched/CapacityConstraint.hpp |   3 +-
 .../include/tetrisched/Expression.hpp         |  11 ++
 .../include/tetrisched/OptimizationPasses.hpp |  14 ++-
 .../include/tetrisched/Scheduler.hpp          |   4 +-
 schedulers/tetrisched/python/Expressions.cpp  |  25 +++--
 schedulers/tetrisched/python/TetrischedPy.cpp |  48 ++++----
 .../tetrisched/src/CapacityConstraint.cpp     |  33 +++++-
 schedulers/tetrisched/src/Expression.cpp      |  34 +++++-
 .../tetrisched/src/OptimizationPasses.cpp     | 105 +++++++++++++-----
 schedulers/tetrisched/src/Scheduler.cpp       |  11 +-
 schedulers/tetrisched_scheduler.py            |  34 ++++++
 15 files changed, 359 insertions(+), 74 deletions(-)
 create mode 100644 configs/new_motiv_new_conf_sukrit_runs_strl_1_8.conf
 create mode 100644 configs/new_motiv_new_conf_sukrit_runs_strl_1_8_selective.conf

diff --git a/configs/new_motiv_new_conf_sukrit_runs_strl_1_8.conf b/configs/new_motiv_new_conf_sukrit_runs_strl_1_8.conf
new file mode 100644
index 00000000..ef9be034
--- /dev/null
+++ b/configs/new_motiv_new_conf_sukrit_runs_strl_1_8.conf
@@ -0,0 +1,45 @@
+# Output configs.
+--log_dir=experiments/new_exps_new_conf/strl_1_8
+--log_file_name=alibaba_trace_replay_tetrisched.log
+--csv_file_name=alibaba_trace_replay_tetrisched.csv
+--log_level=debug
+
+# Workload configs.
+--execution_mode=replay
+--replay_trace=alibaba
+--workload_profile_paths=traces/alibaba-cluster-trace-v2018/easy_dag_sukrit_10k.pkl,traces/alibaba-cluster-trace-v2018/medium_dag_sukrit_10k.pkl,traces/alibaba-cluster-trace-v2018/hard_dag_sukrit_10k.pkl
+--workload_profile_path_labels=easy,medium,hard
+--override_release_policies=poisson,poisson,poisson
+--override_num_invocations=0,300,150
+--override_poisson_arrival_rates=0.005,0.0125,0.0125
+--randomize_start_time_max=50
+--min_deadline=5
+--max_deadline=500
+--min_deadline_variances=25,50,10
+--max_deadline_variances=50,100,25
+
+# Worker configs.
+--worker_profile_path=./profiles/workers/alibaba_cluster.yaml
+
+# Loader configs.
+--alibaba_loader_task_cpu_divisor=10
+--alibaba_loader_min_critical_path_runtimes=200,500,600
+--alibaba_loader_max_critical_path_runtimes=500,1000,1000
+
+# Scheduler configs.
+#--scheduler=EDF
+--scheduler=TetriSched
+#--drop_skipped_tasks
+--release_taskgraphs
+--scheduler_runtime=0
+--enforce_deadlines
+--scheduler_time_discretization=1
+--scheduler_enable_optimization_pass
+--random_seed=420665456
+--retract_schedules
+#--scheduler_log_to_file
+--scheduler_dynamic_discretization
+--scheduler_max_time_discretization=8
+--scheduler_max_occupancy_threshold=0.999
+--finer_discretization_at_prev_solution
+--finer_discretization_window=4
\ No newline at end of file
diff --git a/configs/new_motiv_new_conf_sukrit_runs_strl_1_8_selective.conf b/configs/new_motiv_new_conf_sukrit_runs_strl_1_8_selective.conf
new file mode 100644
index 00000000..9697d01d
--- /dev/null
+++ b/configs/new_motiv_new_conf_sukrit_runs_strl_1_8_selective.conf
@@ -0,0 +1,47 @@
+# Output configs.
+--log_dir=experiments/new_exps_new_conf/strl_1_8_selective_new
+--log_file_name=alibaba_trace_replay_tetrisched.log
+--csv_file_name=alibaba_trace_replay_tetrisched.csv
+--log_level=debug
+
+# Workload configs.
+--execution_mode=replay
+--replay_trace=alibaba
+--workload_profile_paths=traces/alibaba-cluster-trace-v2018/easy_dag_sukrit_10k.pkl,traces/alibaba-cluster-trace-v2018/medium_dag_sukrit_10k.pkl,traces/alibaba-cluster-trace-v2018/hard_dag_sukrit_10k.pkl
+--workload_profile_path_labels=easy,medium,hard
+--override_release_policies=poisson,poisson,poisson
+--override_num_invocations=0,300,150
+--override_poisson_arrival_rates=0.005,0.0125,0.0125
+--randomize_start_time_max=50
+--min_deadline=5
+--max_deadline=500
+--min_deadline_variances=25,50,10
+--max_deadline_variances=50,100,25
+
+# Worker configs.
+--worker_profile_path=./profiles/workers/alibaba_cluster.yaml
+
+# Loader configs.
+--alibaba_loader_task_cpu_divisor=10
+--alibaba_loader_min_critical_path_runtimes=200,500,600
+--alibaba_loader_max_critical_path_runtimes=500,1000,1000
+
+# Scheduler configs.
+#--scheduler=EDF
+--scheduler=TetriSched
+#--drop_skipped_tasks
+--release_taskgraphs
+--scheduler_runtime=0
+--enforce_deadlines
+--scheduler_time_discretization=1
+--scheduler_enable_optimization_pass
+--random_seed=420665456
+--retract_schedules
+#--scheduler_log_to_file
+--scheduler_dynamic_discretization
+--scheduler_max_time_discretization=8
+--scheduler_max_occupancy_threshold=0.999
+--finer_discretization_at_prev_solution
+--finer_discretization_window=4
+--scheduler_selective_rescheduling
+--scheduler_log_to_file
diff --git a/main.py b/main.py
index fe3e49b9..e38d71cf 100644
--- a/main.py
+++ b/main.py
@@ -334,6 +334,18 @@
     "The discretization is dynamically decided based on the occupancy request for "
     "each time slice. (default: False)",
 )
+flags.DEFINE_bool(
+    "finer_discretization_at_prev_solution",
+    False,
+    "If `True`, the scheduler keeps discretization of 1 around previous solution. "
+    "The discretization is dynamically decided based on the occupancy request for "
+    "each time slice. (default: False)",
+)
+flags.DEFINE_integer(
+    "finer_discretization_window",
+    5,
+    "The window around previous solution that keeps discretization of 1.",
+)
 flags.DEFINE_integer(
     "scheduler_max_time_discretization",
     5,
@@ -782,6 +794,7 @@ def main(args):
             _flags=FLAGS,
         )
     elif FLAGS.scheduler == "TetriSched":
+        finer_discretization = FLAGS.finer_discretization_at_prev_solution
         scheduler = TetriSchedScheduler(
             preemptive=FLAGS.preemption,
             runtime=EventTime(FLAGS.scheduler_runtime, EventTime.Unit.US),
@@ -802,6 +815,10 @@ def main(args):
             ),
             dynamic_discretization=FLAGS.scheduler_dynamic_discretization,
             max_occupancy_threshold=FLAGS.scheduler_max_occupancy_threshold,
+            finer_discretization_at_prev_solution=finer_discretization,
+            finer_discretization_window=EventTime(
+                FLAGS.finer_discretization_window, EventTime.Unit.US
+            ),
         )
     elif FLAGS.scheduler == "GraphenePrime":
         scheduler = TetriSchedScheduler(
diff --git a/profiles/workers/alibaba_cluster.yaml b/profiles/workers/alibaba_cluster.yaml
index 38dd94aa..b87b93fd 100644
--- a/profiles/workers/alibaba_cluster.yaml
+++ b/profiles/workers/alibaba_cluster.yaml
@@ -3,4 +3,4 @@
       - name: Worker_1_1
         resources:
             - name: Slot_1
-              quantity: 60
+              quantity: 70
diff --git a/schedulers/tetrisched/include/tetrisched/CapacityConstraint.hpp b/schedulers/tetrisched/include/tetrisched/CapacityConstraint.hpp
index a0f11385..fb48da06 100644
--- a/schedulers/tetrisched/include/tetrisched/CapacityConstraint.hpp
+++ b/schedulers/tetrisched/include/tetrisched/CapacityConstraint.hpp
@@ -144,7 +144,8 @@ class CapacityConstraintMap {
                                 const Time startTime, const Time duration,
                                 const IndicatorT usageIndicator,
                                 const PartitionUsageT usageVariable,
-                                std::optional<Time> granularity);
+                                std::optional<Time> granularity,
+                                bool print = false, std::string exprName = "");
 
   /// Translate the CapacityConstraintMap by moving its constraints
   /// to the given model.
diff --git a/schedulers/tetrisched/include/tetrisched/Expression.hpp b/schedulers/tetrisched/include/tetrisched/Expression.hpp
index 6da03c71..93c87b92 100644
--- a/schedulers/tetrisched/include/tetrisched/Expression.hpp
+++ b/schedulers/tetrisched/include/tetrisched/Expression.hpp
@@ -113,6 +113,8 @@ struct SolutionResult {
   std::optional<TETRISCHED_ILP_TYPE> utility;
   /// The placement objects being bubbled up in the solution.
   std::unordered_map<std::string, PlacementPtr> placements;
+  /// list of satisfied leaf expressions
+  std::set<std::string> satsifiedExpressionNames;
 };
 using SolutionResultPtr = std::shared_ptr<SolutionResult>;
 
@@ -224,6 +226,8 @@ class Expression : public std::enable_shared_from_this<Expression> {
   /// A mutex representing a lock on the parsing and population of results
   /// from this Expression.
   std::mutex expressionMutex;
+  /// if this choose expression was previously satisfied
+  bool previouslySatisfied;
 
   /// Adds a parent to this expression.
   void addParent(const Expression* parent);
@@ -254,6 +258,12 @@ class Expression : public std::enable_shared_from_this<Expression> {
   /// Removes a child from this Expression.
   void removeChild(ExpressionPtr child);
 
+  /// sets that the expression was previously satisfied
+  void setPreviouslySatisfied(bool satisfied);
+
+  /// check of the expression was previously satisfied
+  bool isPreviouslySatisfied();
+
   /// Replaces the children of this node with the given children.
   void replaceChildren(std::vector<ExpressionPtr> children);
 
@@ -480,6 +490,7 @@ class AllocationExpression : public Expression {
                        Time currentTime) override;
   SolutionResultPtr populateResults(SolverModelPtr solverModel) override;
   std::string getDescriptiveName() const override;
+  uint32_t getResourceQuantity() const override;
 };
 
 /// An `ObjectiveExpression` collates the objectives from its children and
diff --git a/schedulers/tetrisched/include/tetrisched/OptimizationPasses.hpp b/schedulers/tetrisched/include/tetrisched/OptimizationPasses.hpp
index 029764a6..a2570ad3 100644
--- a/schedulers/tetrisched/include/tetrisched/OptimizationPasses.hpp
+++ b/schedulers/tetrisched/include/tetrisched/OptimizationPasses.hpp
@@ -92,13 +92,17 @@ class DiscretizationSelectorOptimizationPass : public OptimizationPass {
   Time minDiscretization;
   Time maxDiscretization;
   float maxOccupancyThreshold;
+  bool finerDiscretizationAtPrevSolution;
+  Time finerDiscretizationWindow;
 
  public:
   /// Instantiate the DiscretizationSelectorOptimizationPass.
   DiscretizationSelectorOptimizationPass();
-  DiscretizationSelectorOptimizationPass(Time minDiscretization = 1,
-                                         Time maxDiscretization = 5,
-                                         float maxOccupancyThreshold = 0.8);
+  DiscretizationSelectorOptimizationPass(
+      Time minDiscretization = 1, Time maxDiscretization = 5,
+      float maxOccupancyThreshold = 0.8,
+      bool finerDiscretizationAtPrevSolution = false,
+      Time finerDiscretizationWindow = 5);
 
   /// Run the DiscretizationSelectorOptimizationPass on the given STRL
   /// expression.
@@ -163,7 +167,9 @@ class OptimizationPassRunner {
   OptimizationPassRunner(bool debug = false,
                          bool enableDynamicDiscretization = false,
                          Time minDiscretization = 1, Time maxDiscretization = 5,
-                         float maxOccupancyThreshold = 0.8);
+                         float maxOccupancyThreshold = 0.8,
+                         bool finerDiscretizationAtPrevSolution = false,
+                         Time finerDiscretizationWindow = 5);
 
   /// Run the pre-translation optimization passes on the given STRL expression.
   void runPreTranslationPasses(Time currentTime, ExpressionPtr strlExpression,
diff --git a/schedulers/tetrisched/include/tetrisched/Scheduler.hpp b/schedulers/tetrisched/include/tetrisched/Scheduler.hpp
index 117c4be4..65c46da3 100644
--- a/schedulers/tetrisched/include/tetrisched/Scheduler.hpp
+++ b/schedulers/tetrisched/include/tetrisched/Scheduler.hpp
@@ -46,7 +46,9 @@ class Scheduler {
   /// Initialize the scheduler with a solver backend.
   Scheduler(Time discretization, SolverBackendType solverBackend,
             std::string logDir = "./", bool enableDynamicDiscretization = false,
-            Time maxDiscretization = 5, float maxOccupancyThreshold = 0.8);
+            Time maxDiscretization = 5, float maxOccupancyThreshold = 0.8,
+            bool finerDiscretizationAtPrevSolution = false,
+            Time finerDiscretizationWindow = 5);
 
   /// Registers the STRL expression for the scheduler to schedule from
   /// and parses it to populate the SolverModel.
diff --git a/schedulers/tetrisched/python/Expressions.cpp b/schedulers/tetrisched/python/Expressions.cpp
index c588f3b6..1e16570f 100644
--- a/schedulers/tetrisched/python/Expressions.cpp
+++ b/schedulers/tetrisched/python/Expressions.cpp
@@ -4,7 +4,7 @@
 
 namespace py = pybind11;
 
-void defineSTRLExpressions(py::module_& tetrisched_m) {
+void defineSTRLExpressions(py::module_ &tetrisched_m) {
   // Define the ExpressionType enum.
   py::enum_<tetrisched::ExpressionType>(tetrisched_m, "ExpressionType")
       .value("EXPR_CHOOSE", tetrisched::ExpressionType::EXPR_CHOOSE)
@@ -46,25 +46,31 @@ void defineSTRLExpressions(py::module_& tetrisched_m) {
       tetrisched_m, "SolutionResult")
       .def_property_readonly(
           "startTime",
-          [](const tetrisched::SolutionResult& result) {
+          [](const tetrisched::SolutionResult &result) {
             return result.startTime;
           },
           "The start time of the expression.")
       .def_property_readonly(
           "endTime",
-          [](const tetrisched::SolutionResult& result) {
+          [](const tetrisched::SolutionResult &result) {
             return result.endTime;
           },
           "The end time of the expression.")
       .def_property_readonly(
           "utility",
-          [](const tetrisched::SolutionResult& result) {
+          [](const tetrisched::SolutionResult &result) {
             return result.utility;
           },
           "The utility of the expression.")
+      .def_property_readonly(
+          "satsifiedExpressionNames",
+          [](const tetrisched::SolutionResult &result) {
+            return result.satsifiedExpressionNames;
+          },
+          "The choose expressions that were satisfied.")
       .def(
           "getPlacement",
-          [](const tetrisched::SolutionResult& result,
+          [](const tetrisched::SolutionResult &result,
              std::string taskName) -> std::optional<tetrisched::PlacementPtr> {
             if (result.placements.find(taskName) == result.placements.end()) {
               return std::nullopt;
@@ -75,7 +81,7 @@ void defineSTRLExpressions(py::module_& tetrisched_m) {
           "\nArgs:\n"
           "  taskName (str): The name of the task to get the Placement for.",
           py::arg("taskName"))
-      .def("__str__", [](const tetrisched::SolutionResult& result) {
+      .def("__str__", [](const tetrisched::SolutionResult &result) {
         return "Placement<start=" +
                (result.startTime.has_value()
                     ? std::to_string(result.startTime.value())
@@ -109,6 +115,9 @@ void defineSTRLExpressions(py::module_& tetrisched_m) {
            py::arg("child"))
       .def("getSolution", &tetrisched::Expression::getSolution,
            "Returns the solution for this Expression.")
+      .def("setPreviouslySatisfied",
+           &tetrisched::Expression::setPreviouslySatisfied,
+           "Sets the flag that this expression was previously satisfied.")
       .def("exportToDot", &tetrisched::Expression::exportToDot,
            "Exports the Expression to a dot file.\n"
            "\nArgs:\n"
@@ -117,11 +126,13 @@ void defineSTRLExpressions(py::module_& tetrisched_m) {
            "    included in the dot file. If False, they are not included.",
            py::arg("fileName"), py::arg("emitChooseExpressions") = false)
       .def("__str__",
-           [](const tetrisched::Expression& expr) {
+           [](const tetrisched::Expression &expr) {
              return "Expression<name=" + expr.getName() +
                     ", type=" + expr.getTypeString() + ">";
            })
       .def_property_readonly("name", &tetrisched::Expression::getName)
+      .def_property_readonly("discriptiveName",
+                             &tetrisched::Expression::getDescriptiveName)
       .def_property_readonly("id", &tetrisched::Expression::getId);
 
   // Define the ChooseExpression.
diff --git a/schedulers/tetrisched/python/TetrischedPy.cpp b/schedulers/tetrisched/python/TetrischedPy.cpp
index 776107f6..234519c5 100644
--- a/schedulers/tetrisched/python/TetrischedPy.cpp
+++ b/schedulers/tetrisched/python/TetrischedPy.cpp
@@ -72,26 +72,34 @@ void defineScheduler(py::module_& tetrisched_m) {
 
   // Define the interface to the Scheduler.
   py::class_<tetrisched::Scheduler>(tetrisched_m, "Scheduler")
-      .def(py::init<tetrisched::Time, tetrisched::SolverBackendType,
-                    std::string, bool, tetrisched::Time, float>(),
-           "Initializes the Scheduler with the given backend.\n"
-           "\nArgs:\n"
-           "  discretization (int): The time discretization to use for the "
-           "scheduler.\n"
-           "  solverBackend (SolverBackendType): The solver backend to use for "
-           "the scheduler.\n"
-           "  logDir (str): The directory where the logs are to be output.\n"
-           "  enableDynamicDiscretization (bool): Whether to enable dynamic "
-           "discretization.\n"
-           "  maxDiscretization (int): The maximum discretization to use for "
-           "dynamic discretization.\n"
-           "  maxOccupancyThreshold (float): The maximum occupancy threshold "
-           "to use for dynamic discretization.",
-           py::arg("discretization"), py::arg("solverBackend"),
-           py::arg("logDir") = "./",
-           py::arg("enableDynamicDiscretization") = false,
-           py::arg("maxDiscretization") = 5,
-           py::arg("maxOccupancyThreshold") = 0.8)
+      .def(
+          py::init<tetrisched::Time, tetrisched::SolverBackendType, std::string,
+                   bool, tetrisched::Time, float, bool, tetrisched::Time>(),
+          "Initializes the Scheduler with the given backend.\n"
+          "\nArgs:\n"
+          "  discretization (int): The time discretization to use for the "
+          "scheduler.\n"
+          "  solverBackend (SolverBackendType): The solver backend to use for "
+          "the scheduler.\n"
+          "  logDir (str): The directory where the logs are to be output.\n"
+          "  enableDynamicDiscretization (bool): Whether to enable dynamic "
+          "discretization.\n"
+          "  maxDiscretization (int): The maximum discretization to use for "
+          "dynamic discretization.\n"
+          "  maxOccupancyThreshold (float): The maximum occupancy threshold "
+          "to use for dynamic discretization.\n",
+          "  finerDiscretizationAtPrevSolution (bool): Enables finer "
+          "discretization "
+          "At previous solution.\n",
+          "  finerDiscretizationWindow (int): The discretization around prev "
+          "solution until which the discretization would be 1 ",
+          py::arg("discretization"), py::arg("solverBackend"),
+          py::arg("logDir") = "./",
+          py::arg("enableDynamicDiscretization") = false,
+          py::arg("maxDiscretization") = 5,
+          py::arg("maxOccupancyThreshold") = 0.8,
+          py::arg("finerDiscretizationAtPrevSolution") = false,
+          py::arg("finerDiscretizationWindow") = 5)
       .def(
           "registerSTRL", &tetrisched::Scheduler::registerSTRL,
           "Registers the STRL expression for the scheduler to schedule from.\n"
diff --git a/schedulers/tetrisched/src/CapacityConstraint.cpp b/schedulers/tetrisched/src/CapacityConstraint.cpp
index 8f988189..d0b1f704 100644
--- a/schedulers/tetrisched/src/CapacityConstraint.cpp
+++ b/schedulers/tetrisched/src/CapacityConstraint.cpp
@@ -206,7 +206,8 @@ void CapacityConstraintMap::setDynamicDiscretization(
 void CapacityConstraintMap::registerUsageForDuration(
     const Expression* expression, const Partition& partition,
     const Time startTime, const Time duration, const IndicatorT usageIndicator,
-    const PartitionUsageT variable, std::optional<Time> granularity) {
+    const PartitionUsageT variable, std::optional<Time> granularity, bool print,
+    std::string exprName) {
   if (!useDynamicDiscretization) {
     // If we are not using dynamic discretization, then we can just
     // register the usage at the provided granularity.
@@ -251,7 +252,16 @@ void CapacityConstraintMap::registerUsageForDuration(
 
     // Time currentTime = startTime;
     Time currentTime = timeRangeToGranularities[granularityIndex].first.first;
-    Time remainderTime = duration;
+    Time remainderTime = duration + (startTime - currentTime);
+    // if (print)
+    //   {
+    //     std::cout << "[CapConstraint] Registering usage for expression: " <<
+    //     exprName << " starting at "
+    //                      << startTime << " with duration " << duration
+    //                      << " at time: " << currentTime
+    //                      << " and remainder time: " << remainderTime << "."
+    //                      << std::endl;
+    //   }
     while (remainderTime > 0) {
       auto& timeRange = timeRangeToGranularities[granularityIndex].first;
       auto& granularity = timeRangeToGranularities[granularityIndex].second;
@@ -279,8 +289,25 @@ void CapacityConstraintMap::registerUsageForDuration(
           currentTime += granularity;
           remainderTime = 0;
         }
+        // if(print){
+        //   std::cout << "\t[CapConstraintInside] Registering usage for
+        //   expression starting at "
+        //                    << startTime << " with duration " << duration
+        //                    << " at time: " << currentTime
+        //                    << " within the time range [" << timeRange.first
+        //                    << ", "
+        //                    << timeRange.second
+        //                    << "] with granularity: " << granularity
+        //                    << " and remainder time: " << remainderTime << "."
+        //                    << std::endl;
+        // }
+        if (remainderTime != 0 &&
+            currentTime >= std::min(startTime + duration, timeRange.second) &&
+            granularityIndex == timeRangeToGranularities.size() - 1) {
+          remainderTime = 0;
+          break;
+        }
       }
-
       if (currentTime >=
           timeRangeToGranularities[granularityIndex].first.second) {
         // We have passed the interval. We need to skip to the next interval,
diff --git a/schedulers/tetrisched/src/Expression.cpp b/schedulers/tetrisched/src/Expression.cpp
index 969fd37f..b78dfaab 100644
--- a/schedulers/tetrisched/src/Expression.cpp
+++ b/schedulers/tetrisched/src/Expression.cpp
@@ -139,7 +139,8 @@ Expression::Expression(std::string name, ExpressionType type,
       id(tetrisched::uuid::generate_uuid()),
       type(type),
       timeBounds(ExpressionTimeBounds()),
-      status(status) {}
+      status(status),
+      previouslySatisfied(false) {}
 
 std::string Expression::getName() const { return name; }
 
@@ -172,6 +173,14 @@ void Expression::replaceChildren(std::vector<ExpressionPtr> children) {
   this->children = children;
 }
 
+void Expression::setPreviouslySatisfied(bool satisfied) {
+  this->previouslySatisfied = satisfied;
+}
+
+bool Expression::isPreviouslySatisfied() {
+  return this->previouslySatisfied == true;
+}
+
 ExpressionType Expression::getType() const { return type; }
 
 std::string Expression::getTypeString() const {
@@ -375,6 +384,11 @@ SolutionResultPtr Expression::populateResults(SolverModelPtr solverModel) {
          childExpressionSolution.value()->placements) {
       solution->placements[taskName] = placement;
     }
+    // the child was satisfied, merge its satisfied leaf expressions
+    for (auto& leafExprName :
+         childExpressionSolution.value()->satsifiedExpressionNames) {
+      solution->satsifiedExpressionNames.insert(leafExprName);
+    }
   }
 
   TETRISCHED_DEBUG("Finished populating results for " << name << " of type "
@@ -696,6 +710,7 @@ SolutionResultPtr ChooseExpression::populateResults(
                                       variableValue.value());
   }
   solution->placements[name] = std::move(placement);
+  solution->satsifiedExpressionNames.insert(getDescriptiveName());
   return solution;
 }
 
@@ -1041,6 +1056,7 @@ SolutionResultPtr WindowedChooseExpression::populateResults(
     placement->setEndTime(chooseTime + duration);
   }
   solution->placements[name] = std::move(placement);
+  solution->satsifiedExpressionNames.insert(getDescriptiveName());
   return solution;
 }
 
@@ -1422,6 +1438,7 @@ SolutionResultPtr MalleableChooseExpression::populateResults(
     placement->addPartitionAllocation(partitionId, time, variableValue.value());
   }
   solution->placements[name] = std::move(placement);
+  solution->satsifiedExpressionNames.insert(getDescriptiveName());
   return solution;
 }
 
@@ -1488,7 +1505,8 @@ ParseResultPtr AllocationExpression::parse(
   // Add the allocation variables to the CapacityConstraintMap.
   for (const auto& [partition, allocation] : allocatedResources) {
     capacityConstraints->registerUsageForDuration(
-        this, *partition, startTime, duration, 1, allocation, std::nullopt);
+        this, *partition, startTime, duration, 1, allocation, std::nullopt,
+        true, getDescriptiveName());
   }
   TETRISCHED_DEBUG("Finished parsing AllocationExpression for "
                    << name << " to be placed starting at time " << startTime
@@ -1510,6 +1528,14 @@ std::string AllocationExpression::getDescriptiveName() const {
          ", F=" + std::to_string(endTime) + ")";
 }
 
+uint32_t AllocationExpression::getResourceQuantity() const {
+  uint32_t numResources = 0;
+  for (const auto& [partition, allocation] : allocatedResources) {
+    numResources += allocation;
+  }
+  return numResources;
+}
+
 /* Method definitions for ObjectiveExpression */
 
 ObjectiveExpression::ObjectiveExpression(std::string name)
@@ -2734,7 +2760,9 @@ ParseResultPtr ScaleExpression::parse(
       if (!childParseResult->utility) {
         throw tetrisched::exceptions::ExpressionConstructionException(
             "ScaleExpression applied to a child that does not have any "
-            "utility.");
+            "utility. Scale Expr Name: " +
+            this->getDescriptiveName() +
+            " Child Name: " + children[0]->getDescriptiveName());
       }
       TETRISCHED_DEBUG(
           "[" << name << "] is amplifying the utility for the child "
diff --git a/schedulers/tetrisched/src/OptimizationPasses.cpp b/schedulers/tetrisched/src/OptimizationPasses.cpp
index d573629d..d5441e51 100644
--- a/schedulers/tetrisched/src/OptimizationPasses.cpp
+++ b/schedulers/tetrisched/src/OptimizationPasses.cpp
@@ -457,15 +457,20 @@ DiscretizationSelectorOptimizationPass::DiscretizationSelectorOptimizationPass()
                        OptimizationPassType::PRE_TRANSLATION_PASS),
       minDiscretization(1),
       maxDiscretization(5),
-      maxOccupancyThreshold(0.8) {}
+      maxOccupancyThreshold(0.8),
+      finerDiscretizationAtPrevSolution(false),
+      finerDiscretizationWindow(5) {}
 
 DiscretizationSelectorOptimizationPass::DiscretizationSelectorOptimizationPass(
-    Time minDiscretization, Time maxDiscretization, float maxOccupancyThreshold)
+    Time minDiscretization, Time maxDiscretization, float maxOccupancyThreshold,
+    bool finerDiscretizationAtPrevSolution, Time finerDiscretizationWindow)
     : OptimizationPass("DiscretizationSelectorOptimizationPass",
                        OptimizationPassType::PRE_TRANSLATION_PASS),
       minDiscretization(minDiscretization),
       maxDiscretization(maxDiscretization),
-      maxOccupancyThreshold(maxOccupancyThreshold) {}
+      maxOccupancyThreshold(maxOccupancyThreshold),
+      finerDiscretizationAtPrevSolution(finerDiscretizationAtPrevSolution),
+      finerDiscretizationWindow(finerDiscretizationWindow) {}
 
 void DiscretizationSelectorOptimizationPass::runPass(
     ExpressionPtr strlExpression, CapacityConstraintMapPtr capacityConstraints,
@@ -475,7 +480,8 @@ void DiscretizationSelectorOptimizationPass::runPass(
   firstStack.push(strlExpression);
 
   std::vector<ExpressionPtr> maxOverNckExprs;
-  std::vector<ExpressionPtr> independentNcks;
+  std::vector<ExpressionPtr> independentLeafExprs;
+  std::set<Time> chooseExprStartTimes;
 
   while (!firstStack.empty()) {
     // Move the expression to the second stack.
@@ -484,14 +490,33 @@ void DiscretizationSelectorOptimizationPass::runPass(
     bool isMaxExpr = false;
     if (currentExpression->getType() == ExpressionType::EXPR_MAX) {
       isMaxExpr = true;
-    } else if ((currentExpression->getType() == ExpressionType::EXPR_CHOOSE ||
-                currentExpression->getType() ==
-                    ExpressionType::EXPR_WINDOWED_CHOOSE) &&
-               currentExpression->getParents()[0]->getType() !=
-                   ExpressionType::EXPR_MAX) {
+    } else if (currentExpression->getType() == ExpressionType::EXPR_CHOOSE ||
+               currentExpression->getType() ==
+                   ExpressionType::EXPR_WINDOWED_CHOOSE ||
+               currentExpression->getType() ==
+                   ExpressionType::EXPR_ALLOCATION) {
       // TODO(Alind): Actual check should be that MAX has all children NCK for
       // EXPR_CHOOSE
-      independentNcks.push_back(currentExpression);
+      if ((currentExpression->getType() == ExpressionType::EXPR_CHOOSE ||
+           currentExpression->getType() ==
+               ExpressionType::EXPR_WINDOWED_CHOOSE ||
+           currentExpression->getType() == ExpressionType::EXPR_ALLOCATION) &&
+          finerDiscretizationAtPrevSolution &&
+          currentExpression->isPreviouslySatisfied()) {
+        // register the start time of prev solved choose expr +-
+        // finerDiscretizationWindow
+        Time startTimeChooseExpr =
+            currentExpression->getTimeBounds().startTimeRange.first;
+        Time tbegin = startTimeChooseExpr;
+        Time tend = startTimeChooseExpr;
+        for (auto finerTime = tbegin; finerTime <= tend; finerTime++)
+          chooseExprStartTimes.insert(finerTime);
+      }
+      if (currentExpression->getParents()[0]->getType() !=
+              ExpressionType::EXPR_MAX ||
+          currentExpression->getType() == ExpressionType::EXPR_ALLOCATION) {
+        independentLeafExprs.push_back(currentExpression);
+      }
     }
     bool allChildrenNck = false;
     size_t numChildNcks = 0;
@@ -512,7 +537,7 @@ void DiscretizationSelectorOptimizationPass::runPass(
       maxOverNckExprs.push_back(currentExpression);
     }
   }
-  if (maxOverNckExprs.size() == 0 && independentNcks.size() == 0) {
+  if (maxOverNckExprs.size() == 0 && independentLeafExprs.size() == 0) {
     throw tetrisched::exceptions::RuntimeException(
         "Inside Discretization Optimization pass: but no max over nck or "
         "independent ncks found!");
@@ -537,8 +562,8 @@ void DiscretizationSelectorOptimizationPass::runPass(
     // Push the occupancy request range into the vector.
     occupancyRequestRanges.push_back({occupancyRequestRange, numResources});
   }
-  for (auto iNck : independentNcks) {
-    auto timeBounds = iNck->getTimeBounds();
+  for (auto iLeafExpr : independentLeafExprs) {
+    auto timeBounds = iLeafExpr->getTimeBounds();
     TimeRange occupancyRequestRange = std::make_pair(
         timeBounds.startTimeRange.first, timeBounds.endTimeRange.second);
 
@@ -549,7 +574,7 @@ void DiscretizationSelectorOptimizationPass::runPass(
     if (occupancyRequestRange.second > maxOccupancyTime) {
       maxOccupancyTime = occupancyRequestRange.second;
     }
-    auto numResources = iNck->getResourceQuantity();
+    auto numResources = iLeafExpr->getResourceQuantity();
     // Push the occupancy request range into the vector.
     occupancyRequestRanges.push_back({occupancyRequestRange, numResources});
   }
@@ -611,7 +636,12 @@ void DiscretizationSelectorOptimizationPass::runPass(
     if (planAheadIndex > -1) {
       Time endTime = timeRangeToGranularities[planAheadIndex].first.second;
       Time currentTime = minOccupancyTime + i;
-      if (currentTime < endTime) {
+      if (finerDiscretizationAtPrevSolution &&
+          chooseExprStartTimes.contains(currentTime)) {
+        timeRangeToGranularities[planAheadIndex].first.second = currentTime;
+        timeRangeToGranularities[planAheadIndex].second =
+            currentTime - timeRangeToGranularities[planAheadIndex].first.first;
+      } else if (currentTime < endTime) {
         continue;
       }
     }
@@ -621,7 +651,6 @@ void DiscretizationSelectorOptimizationPass::runPass(
     uint32_t predictedDiscretization = linearDecayFunction(currentOccupancy);
     auto nextPlanAhead =
         std::min(i + predictedDiscretization, occupancyRequests.size());
-
     // find the right discretization such that average occupancy for that
     // period predicts same discretization
     while (nextPlanAhead >= (i + 1)) {
@@ -651,11 +680,10 @@ void DiscretizationSelectorOptimizationPass::runPass(
   // set capacity constraint map to dynamic and update it
   capacityConstraints->setDynamicDiscretization(timeRangeToGranularities);
 
-  // // Log the found dynamic discretizations and times.
-  // std::cout <<
-  //     "[DiscretizationSelectorOptimizationPass] Dynamic Discretization
-  //     between "
-  //     << minOccupancyTime << " and " << maxOccupancyTime << ": "<<
+  // Log the found dynamic discretizations and times.
+  // std::cout << "[DiscretizationSelectorOptimizationPass] Dynamic
+  // Discretization between " << minOccupancyTime << " and " << maxOccupancyTime
+  // << ": "<<
   //     std::endl;
 
   // for (auto &[discretizationTimeRange, granularity] :
@@ -665,8 +693,7 @@ void DiscretizationSelectorOptimizationPass::runPass(
   //             << "[DiscretizationSelectorOptimizationPassDiscreteTime] "
   //             << discretizationTimeRange.first << " - " <<
   //             discretizationTimeRange.second << " : " << granularity << "
-  //             Occuapncy: " <<
-  //             occupancyRequests[discretizationTimeRange.first
+  //             Occuapncy: " << occupancyRequests[discretizationTimeRange.first
   //             - minOccupancyTime] << std::endl;
   // }
 
@@ -682,11 +709,16 @@ void DiscretizationSelectorOptimizationPass::runPass(
       std::vector<ExpressionPtr> ncksWithinTimeRange;
       auto expressionChildren = maxNckExpr->getChildren();
       ExpressionPtr minStartTimeNckExpr = nullptr;
+      ExpressionPtr prevSolutionNckExpr = nullptr;
       for (auto child = expressionChildren.rbegin();
            child != expressionChildren.rend(); ++child) {
         auto startTimeNck = (*child)->getTimeBounds().startTimeRange.first;
         if (startTimeNck >= startTime && startTimeNck < endTime) {
           ncksWithinTimeRange.push_back(*child);
+          if (prevSolutionNckExpr == nullptr &&
+              (*child)->isPreviouslySatisfied()) {
+            prevSolutionNckExpr = *child;
+          }
           if (minStartTimeNckExpr != nullptr) {
             if (minStartTimeNckExpr->getTimeBounds().startTimeRange.first >
                 startTimeNck) {
@@ -697,12 +729,25 @@ void DiscretizationSelectorOptimizationPass::runPass(
           }
         }
       }
+      ExpressionPtr chosenNckExpr = nullptr;
+      if (prevSolutionNckExpr != nullptr) {
+        chosenNckExpr = prevSolutionNckExpr;
+        // std::cout << "\t ***** "
+        //           << "[" << minOccupancyTime << "]"
+        //           << "[DiscretizationSelectorOptimizationPassDiscreteTime]
+        //           Not Removing Previous Solution: "
+        //           << chosenNckExpr->getDescriptiveName() << " Start Time: "
+        //           << chosenNckExpr->getTimeBounds().startTimeRange.first <<
+        //           std::endl;
+      } else {
+        chosenNckExpr = minStartTimeNckExpr;
+      }
       if (ncksWithinTimeRange.size() > 1) {
         // if more than one nck found within the time range, remove it as only
         // one nck within granularity is sufficient. Nck with minimum start
         // time is kept within this time range
         for (auto redundantNckExpr : ncksWithinTimeRange) {
-          if (redundantNckExpr->getId() != minStartTimeNckExpr->getId()) {
+          if (redundantNckExpr->getId() != chosenNckExpr->getId()) {
             maxNckExpr->removeChild(redundantNckExpr);
             // std::cout << "[DiscretizationSelectorOptimizationPassRemoveNck]
             // Removing NCK: " + redundantNckExpr->getName() + " From Max: " +
@@ -1078,11 +1123,10 @@ void CapacityConstraintMapPurgingOptimizationPass::runPass(
 void CapacityConstraintMapPurgingOptimizationPass::clean() { cliques.clear(); }
 
 /* Methods for OptimizationPassRunner */
-OptimizationPassRunner::OptimizationPassRunner(bool debug,
-                                               bool enableDynamicDiscretization,
-                                               Time minDiscretization,
-                                               Time maxDiscretization,
-                                               float maxOccupancyThreshold)
+OptimizationPassRunner::OptimizationPassRunner(
+    bool debug, bool enableDynamicDiscretization, Time minDiscretization,
+    Time maxDiscretization, float maxOccupancyThreshold,
+    bool finerDiscretizationAtPrevSolution, Time finerDiscretizationWindow)
     : debug(debug), enableDynamicDiscretization(enableDynamicDiscretization) {
   // Register the Critical Path optimization pass.
   registeredPasses.push_back(std::make_shared<CriticalPathOptimizationPass>());
@@ -1091,7 +1135,8 @@ OptimizationPassRunner::OptimizationPassRunner(bool debug,
     // Register the DiscretizationGenerator pass.
     registeredPasses.push_back(
         std::make_shared<DiscretizationSelectorOptimizationPass>(
-            minDiscretization, maxDiscretization, maxOccupancyThreshold));
+            minDiscretization, maxDiscretization, maxOccupancyThreshold,
+            finerDiscretizationAtPrevSolution, finerDiscretizationWindow));
   }
 
   // Register the CapacityConstraintMapPurging optimization pass.
diff --git a/schedulers/tetrisched/src/Scheduler.cpp b/schedulers/tetrisched/src/Scheduler.cpp
index 7d230ee8..97a1af91 100644
--- a/schedulers/tetrisched/src/Scheduler.cpp
+++ b/schedulers/tetrisched/src/Scheduler.cpp
@@ -13,7 +13,9 @@
 namespace tetrisched {
 Scheduler::Scheduler(Time discretization, SolverBackendType solverBackend,
                      std::string logDir, bool enableDynamicDiscretization,
-                     Time maxDiscretization, float maxOccupancyThreshold)
+                     Time maxDiscretization, float maxOccupancyThreshold,
+                     bool finerDiscretizationAtPrevSolution,
+                     Time finerDiscretizationWindow)
     : solverBackend(solverBackend),
       discretization(discretization),
       logDir(logDir) {
@@ -34,9 +36,10 @@ Scheduler::Scheduler(Time discretization, SolverBackendType solverBackend,
           "The solver backend type is not supported.");
   }
   solverModel = solver->getModel();
-  optimizationPasses =
-      OptimizationPassRunner(false, enableDynamicDiscretization, discretization,
-                             maxDiscretization, maxOccupancyThreshold);
+  optimizationPasses = OptimizationPassRunner(
+      false, enableDynamicDiscretization, discretization, maxDiscretization,
+      maxOccupancyThreshold, finerDiscretizationAtPrevSolution,
+      finerDiscretizationWindow);
   solverConfig = std::make_shared<SolverConfig>();
 }
 
diff --git a/schedulers/tetrisched_scheduler.py b/schedulers/tetrisched_scheduler.py
index 514be3b8..b5a9bd7c 100644
--- a/schedulers/tetrisched_scheduler.py
+++ b/schedulers/tetrisched_scheduler.py
@@ -180,6 +180,8 @@ def __init__(
         max_time_discretization: EventTime = EventTime(5, EventTime.Unit.US),
         dynamic_discretization: bool = False,
         max_occupancy_threshold: float = 0.8,
+        finer_discretization_at_prev_solution: bool = False,
+        finer_discretization_window: EventTime = EventTime(5, EventTime.Unit.US),
     ):
         if preemptive:
             raise ValueError("TetrischedScheduler does not support preemption.")
@@ -215,6 +217,8 @@ def __init__(
             self._dynamic_discretization,  # enable dynamic discretization
             self._max_discretization.time,
             max_occupancy_threshold,
+            finer_discretization_at_prev_solution,
+            finer_discretization_window.to(EventTime.Unit.US).time,
         )
         self._use_task_graph_indicator_utility = self._goal == "max_goodput"
         self._previously_placed_reward_scale_factor = 1.0
@@ -259,6 +263,9 @@ def __init__(
         # sure that no optimization passes remove it.
         # Reset at the beginning of each Scheduler invocation (`schedule()`).
         self._individual_task_strls: Mapping[str, tetrisched.Expression] = {}
+        # A cache for previously selected choose expressions, this is done for
+        # providing better hints to the solver
+        self._previously_satisfied_choose_exprs = set([])
         if (
             self._adaptive_discretization
             and self._max_discretization.time < self._time_discretization.time
@@ -831,7 +838,15 @@ def schedule(
 
             # Retrieve the solution and check if we were able to schedule anything.
             solverSolution = objective_strl.getSolution()
+
             if solverSolution is not None and solverSolution.utility > 0:
+                self._previously_satisfied_choose_exprs = (
+                    solverSolution.satsifiedExpressionNames
+                )
+                self._logger.info(
+                    f"The satisfied Choose Expressions: "
+                    f"{self._previously_satisfied_choose_exprs}"
+                )
                 self._logger.info(
                     f"[{sim_time.time}] Solver returned utility of "
                     f"{solverSolution.utility} and took {solver_time} to solve. The "
@@ -925,6 +940,18 @@ def schedule(
                         task_placement.execution_strategy,
                     )
             else:
+                # There were no Placements from the Scheduler. Inform the Simulator.
+                self._previously_satisfied_choose_exprs = set([])
+                for task in tasks_to_be_scheduled:
+                    placements.append(
+                        Placement.create_task_placement(
+                            task=task,
+                            placement_time=None,
+                            worker_pool_id=None,
+                            worker_id=None,
+                            execution_strategy=None,
+                        )
+                    )
                 # There were no Placements from the Scheduler. We just skip the
                 # placement of any of the tasks, and wait for the next invocation.
                 self._logger.warning(f"[{sim_time.time}] Failed to place any tasks.")
@@ -1139,6 +1166,8 @@ def construct_task_strl(
                 scheduled_discretization.to(EventTime.Unit.US).time,
                 task_remaining_time.to(EventTime.Unit.US).time,
             )
+            if task.state == TaskState.SCHEDULED:
+                task_allocation_expression.setPreviouslySatisfied(True)
             self._logger.debug(
                 f"[{current_time.time}] Generated an AllocationExpression for "
                 f"task {task.unique_name} in state {task.state} starting at "
@@ -1282,6 +1311,11 @@ def construct_task_strl(
                             placement_hint,
                         )
                     )
+                    if (
+                        task_choose_expressions[-1].discriptiveName
+                        in self._previously_satisfied_choose_exprs
+                    ):
+                        task_choose_expressions[-1].setPreviouslySatisfied(True)
                     choice_placement_times_and_rewards.append(
                         (placement_time.time, reward_for_this_placement)
                     )