prepare for 1.2.1 release

README.md

@@ -1,6 +1,7 @@
 # LaunchPad
 [![Build Status](https://circleci.com/gh/stanfordmlgroup/LaunchPad.svg?style=svg&circle-token=00b64008c5dc07a73a311815b5ca0e935291dbb3)](https://circleci.com/gh/stanfordmlgroup/LaunchPad)
 [![CodeFactor](https://www.codefactor.io/repository/github/stanfordmlgroup/launchpad/badge)](https://www.codefactor.io/repository/github/stanfordmlgroup/launchpad)
+[![PyPI version](https://badge.fury.io/py/launch-pad.svg)](https://badge.fury.io/py/launch-pad)
 [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/) <br>
 LaunchPad is a light-weighted Slurm job launcher designed for hyper-parameter search.
 

launchpad/__init__.py

@@ -1 +1,36 @@
-from .launchpad import main
+import yaml
+from subprocess import check_call, check_output
+import uuid
+import fire
+import time
+import os
+import pandas as pd
+import logging
+import shutil
+
+from .job import Job
+from .util import Config, Args
+
+
+def run(config="config.yaml",
+         run=False):
+    if os.path.isdir(config):
+        config = os.path.join(config, "config.yaml")
+
+    _config = Config(config)
+
+    if isinstance(run, str):
+        _config.meta['run'] = run
+
+    job = Job(_config)
+
+    if run:
+        job.run()
+    else:
+        job.compile()
+
+def main():
+    fire.Fire(run)
+
+if __name__ == "__main__":
+    fire.Fire(run)

launchpad/job/base.py

@@ -48,14 +48,24 @@ def _get_metrics_path(self):
             raise ValueError("[metrics_path] has not been set up!")
         metrics_path = metrics_path.format(**self._hp)
         return metrics_path
-
-    def _get_exec_line(self):
-        executor, script_path = self._meta.script.split()
+
+    def _parse_script(self):
+        script_items = tuple(self._meta.script.split())
+        if len(script_items) == 2:
+            executor, script_path = script_items 
+            args = []
+        else:
+            executor, script_path, args = script_items 
+            args = [args]
         config_path = self._meta.config_path
         script_path = os.path.abspath(os.path.join(os.path.dirname(config_path),
-                                      script_path))
+                                  script_path))
+        return executor, script_path, args
+
+    def _get_exec_line(self):
+        executor, script_path, args = self._parse_script()
         self._code_dir = os.path.dirname(script_path)
-        self._exec_line = " ".join([executor, script_path] \
+        self._exec_line = " ".join([executor, script_path] + args \
                 + [f"--{k} {v}" for k, v in self._hp.items()])
 
     def _get_exp_name(self): 

launchpad/job/master_job.py

@@ -65,5 +65,7 @@ def print_state(self, idx, job):
         print(f"Current State: {colorful_state(state)}") 
         if state == "Running":
             print(f"Slurm job ID: {job._id}")
-            print("Latest metrics: ")
-            print(f"{job.get_metrics()}")
+            metrics = job.get_metrics()
+            if metrics is not None:
+                print("Latest metrics: ")
+                print(f"{job.get_metrics()}")

launchpad/job/nni_job.py

@@ -126,7 +126,11 @@ def _compile_slurm_job(self):
     def _compile_hp(self):
         hp_json_dict = {}
         for k, v in self._hp.items():
-            hp_json_dict[k] = {"_type": "choice", "_value": v}
+            if isinstance(v, list):
+                hp_json_dict[k] = {"_type": "choice", "_value": v}
+            else:
+                hp_json_dict[k] = {"_type": v['type'], "_value": v['value']}
+
         with open(self._nni_hp_path, 'w') as f:
             json.dump(hp_json_dict, f)
         print(f"Dump HP json file to [{self._nni_hp_path}].")
@@ -163,9 +167,6 @@ def _get_exp_name(self):
         self._exp_name = self._meta.prefix
 
     def _get_exec_line(self):
-        executor, script_path = self._meta.script.split()
-        config_path = self._meta.config_path
-        script_path = os.path.abspath(os.path.join(os.path.dirname(config_path),
-                                      script_path))
+        executor, script_path, args = self._parse_script()
         self._code_dir = os.path.dirname(script_path)
-        self._exec_line = " ".join([executor, script_path])
+        self._exec_line = " ".join([executor, script_path] + args)

launchpad/job/slurm_job.py

@@ -57,8 +57,8 @@ def get_state(self):
         try:
             s = self.get_info()
             if s is not None:
-                state = s[4]
-                self._id = s[0]
+                state = s['state']
+                self._id = s['id']
         except FileNotFoundError:
             pass # e.g. squeue not installed
 
@@ -80,7 +80,11 @@ def get_state(self):
 
     @ttl_cache(ttl=5)
     def get_metrics(self):
-        return pd.read_csv(self._get_metrics_path()).tail(1)
+        metrics_path = self._meta.get("metrics_path", None)
+        if metrics_path is None:
+            return None
+        metrics_path = metrics_path.format(**self._hp)
+        return pd.read_csv(metrics_path).tail(1)
 
     def run(self): 
         self.compile()
@@ -91,13 +95,6 @@ def run(self):
     def cancel(self):
         check_output(["scancel", "-n", self._exp_name])
 
-    def _get_metrics_path(self):
-        metrics_path = self._meta.get("metrics_path", None)
-        if metrics_path is None:
-            raise ValueError("[metrics_path] has not been set up!")
-        metrics_path = metrics_path.format(**self._hp)
-        return metrics_path
-
     def _get_sbatch_config(self):
         return "\n".join(
                 [f"#SBATCH --{k}={v}" for k, v in self._sbatch.items()])

setup.py

@@ -21,11 +21,11 @@
 
 
 def version():
-    return "1.1.2"
+    return "1.2.1"
 
 
 setup(
-    name='launchpad',
+    name='launch-pad',
     version=version(),
     description='A tool that automatically compile and launch slurm jobs \
                  based on a YAML configuration file.',
@@ -47,9 +47,8 @@ def version():
     ],
     keywords='automation, sbatch',
     install_requires=INSTALL_REQUIRES,
-    #packages=find_packages(),
-    py_modules=['launchpad'],
-    package_data={'launchpad': ['*.sh']},
+    packages=find_packages(),
+    data_files=[('slurm_script', ['launchpad/job/scripts/sbatch_template.sh'])],
     zip_safe=False,
     include_package_data=True,
     entry_points={'console_scripts': ['lp = launchpad:main']},