Do not scale learning rate of optimizer. (sql-machine-learning#2312)

* Don't scale learning rate

* Remove unused variables

* Add an ut for training_process in AllReduceTrainer

* Pre-commit

* Fix unit test

elasticdl/python/tests/allreduce_trainer_test.py

@@ -34,6 +34,13 @@ def setUp(self):
         model.loss = loss
         self._trainer = AllReduceTrainer(master_client, "", model)
 
+    def test_training_process(self):
+        self._trainer.init_horovod_if_needed()
+        features = tf.constant([[0.5], [0.6], [0.7]])
+        labels = tf.constant([[1.0], [0.0], [1.0]])
+        loss = self._trainer._training_process(features, labels)
+        self.assertIsNotNone(loss)
+
     def test_train_minibatch(self):
         self._trainer.init_horovod_if_needed()
         features = tf.constant([[0.5], [0.6], [0.7]])

elasticdl/python/worker/allreduce_trainer.py

@@ -47,22 +47,9 @@ def __init__(self, master_client, master_addr, model):
         self._rendezvous_id = None
         self._need_broadcast = True
         self._var_created = False
-        self._world_size = None
-        self._set_optimizer(model.optimizer)
+        self._optimizer = model.optimizer
         self._set_horovod_env()
 
-    def _set_optimizer(self, optimizer):
-        self._optimizer = optimizer
-        self._lr = optimizer._hyper["learning_rate"]
-        self._optimizer.lr = self._get_learning_rate
-
-    def _get_learning_rate(self):
-        scaler = 1 if self._world_size is None else self._world_size
-        lr = self._lr
-        if callable(lr):
-            lr = lr()
-        return lr * scaler
-
     @tf.function
     def _training_process(self, features, labels):
         with tf.GradientTape() as tape:
@@ -143,7 +130,6 @@ def init_horovod_if_needed(self):
             hvd.shutdown()
             hvd.init()
             os.environ[HorovodEnv.ELASTIC] = str(1)
-            self._world_size = hvd.size()
             self._rendezvous_id = rank_response.rendezvous_id
             self._need_broadcast = True