[Python] Move all python code together (husky-team#21)

Ref husky-team#18 & husky-team#19.

README.md

@@ -45,20 +45,24 @@ First, run `PyHuskyMaster`:
 
     ./PyHuskyMaster --conf /path/to/your/conf
 
-Second, run `PyHuskyDaemon`:
+Second, set `$PYTHONPATH` environment value:
 
-In the single-machine environment, use the following,
+    export PYTHONPATH=$PYHUSKY_ROOT/python
+
+Run `PyHuskyDaemon` in the single-machine environment:
 
     ./PyHuskyDaemon --conf /path/to/your/conf
 
-In the distributed environment, use the following to execute workers on all machines,
+Run `PyHuskyMaster` in the distributed environment, copy `$PYHUSKY_ROOT/python` to worker machines and set `$PYTHONPATH` like the second step in worker machines at first.
+
+Then, use `scritps/exec.sh` to execute `PyHuskyDaemon` on all worker machines:
 
-    ./exec.sh PyHuskyDaemon --conf /path/to/your/conf
+    ./scritps/exec.sh PyHuskyDaemon --conf /path/to/your/conf
 
-Then, start a `python` terminal in `$PYHUSKY_ROOT`:
+Last, start a `python` terminal in `$PYHUSKY_ROOT` on your master machine:
 
 ```python
-    >>> import frontend as ph
+    >>> import pyhusky.frontend as ph
     >>> ph.env.pyhusky_start("master", xxxxx)
     >>> words = ["hello", "world", "hello", "husky"]
     >>> word_list = ph.env.parallelize(words)

backend/pythonconnector.cpp

@@ -80,7 +80,7 @@ void PythonConnector::start_python_proc(int lid, int gid, int num_workers) {
     std::string master_port = std::to_string(Context::get_config().get_master_port());
     std::string proc_id = std::to_string(Context::get_worker_info().get_process_id());
     extern_py_proc =
-        popen(("python -m backend.python.python_backend " + std::to_string(lid) + " " + std::to_string(gid) + " " +
+        popen(("python -m pyhusky.backend.python_backend " + std::to_string(lid) + " " + std::to_string(gid) + " " +
                proc_id + " " + std::to_string(num_workers) + " " + master_port + " > /tmp/log-pyhusky-" + master_port +
                "-proc-" + proc_id + "-" + std::to_string(lid))
                   .c_str(),

backend/python/globalvar.py → python/pyhusky/backend/globalvar.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import zmq
 import sys
+import zmq
 
 class OperationParam:
     data_str = "data"

backend/library/functional.py → python/pyhusky/backend/library/functional.py

@@ -12,17 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-import json
 import cPickle
 import cloudpickle
-from backend.python.globalvar import GlobalVar, GlobalSocket, OperationParam, GlobalN2NSocket
-from backend.python.serializers import Serializer
-from frontend.binstream import BinStream
+import json
+import sys
 
 from itertools import groupby, islice
 from operator import itemgetter
 
+from pyhusky.backend.globalvar import GlobalVar, GlobalSocket, OperationParam, GlobalN2NSocket
+from pyhusky.common.binstream import BinStream
+from pyhusky.common.serializers import Serializer
+
 def register_all():
     # register actions
     Load.register()

backend/library/linear_regression.py → python/pyhusky/backend/library/linear_regression.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import cPickle
-from backend.python.globalvar import GlobalVar, GlobalSocket, OperationParam
-
+import json
 import sys
 
+from pyhusky.backend.globalvar import GlobalVar, GlobalSocket, OperationParam
+
 def register_all():
     LinearModel.register()
 

backend/library/logistic_regression.py → python/pyhusky/backend/library/logistic_regression.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import cPickle
-from backend.python.globalvar import GlobalVar, GlobalSocket, OperationParam
+import json
+
+from pyhusky.backend.globalvar import GlobalVar, GlobalSocket, OperationParam
 
 def register_all():
     LogisticModel.register()

backend/library/svm.py → python/pyhusky/backend/library/svm.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import cPickle
-from backend.python.globalvar import GlobalVar, GlobalSocket, OperationParam
+import json
+
+from pyhusky.backend.globalvar import GlobalVar, GlobalSocket, OperationParam
 
 def register_all():
     SVMModel.register()

backend/python/processor.py → python/pyhusky/backend/processor.py

@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+import cPickle
+import cloudpickle
 import json
 import struct
-import cloudpickle
-import cPickle
-from frontend.binstream import BinStream
-from backend.python.globalvar import GlobalVar, GlobalSocket
+import sys
+
+from pyhusky.backend.globalvar import GlobalVar, GlobalSocket
+from pyhusky.common.binstream import BinStream
 
 def log_msg(msg):
     print msg

backend/python/python_backend.py → python/pyhusky/backend/python_backend.py

@@ -14,10 +14,10 @@
 
 import sys
 
-from backend.python.globalvar import GlobalSocket, GlobalVar, GlobalN2NSocket
-from backend.python.serializers import Serializer, MarshalSerializer, PickleSerializer, AutoSerializer, CompressedSerializer
-from backend.register import register_func
-from processor import handle_instr
+from pyhusky.backend.globalvar import GlobalSocket, GlobalVar, GlobalN2NSocket
+from pyhusky.backend.processor import handle_instr
+from pyhusky.backend.register import register_func
+from pyhusky.common.serializers import Serializer, MarshalSerializer, PickleSerializer, AutoSerializer, CompressedSerializer
 
 def log_msg(msg):
     print msg

backend/register.py → python/pyhusky/backend/register.py

@@ -12,20 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import backend.library.functional as functional
-# import backend.library.graph as graph
-# import backend.library.word as word
-import backend.library.linear_regression as LinearR
-import backend.library.logistic_regression as LogisticR
-# import backend.library.spca as SPCA
-# import backend.library.tfidf as tfidf
-# import backend.library.bm25 as BM25
-import backend.library.svm as SVM
+import pyhusky.backend.library.functional as functional
+import pyhusky.backend.library.linear_regression as LinearR
+import pyhusky.backend.library.logistic_regression as LogisticR
 
 def register_func():
     # register
-    # functional
     functional.register_all()
     LinearR.register_all()
-    # SVM.register_all()
     LogisticR.register_all()

frontend/binstream.py → python/pyhusky/common/binstream.py

@@ -16,7 +16,7 @@
 import struct
 import types
 
-from operation import Operation
+from pyhusky.common.operation import Operation
 
 class BinStream(object):
     def __init__(self):

frontend/operation.py → python/pyhusky/common/operation.py

@@ -29,6 +29,6 @@ def __init__(self, op_name, op_param, op_deps=[]):
         self.op_param = op_param
         self.op_deps = [dep for dep in op_deps if dep is not None]
         self.is_materialized = False
+
     def __repr__(self):
         return "<"+self.op_name+">"
-        # return "<"+self.op_name+">"+" ".join(str(x) for x in self.op_param)

backend/python/serializers.py → python/pyhusky/common/serializers.py

@@ -14,9 +14,9 @@
 
 import cPickle
 import marshal
+import msgpack
 import sys
 import zlib
-import msgpack
 
 if sys.version < '3':
     import cPickle as pickle

frontend/config.py → python/pyhusky/frontend/config.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import atexit
 import argparse
+import atexit
 import random
 import string
 
-import session
+from pyhusky.frontend import session
 
 disable_progress = False
 

frontend/datareceiver.py → python/pyhusky/frontend/datareceiver.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from binstream import BinStream
-from frontend.library.register import register
-from backend.python.serializers import Serializer
+from pyhusky.common.binstream import BinStream
+from pyhusky.common.serializers import Serializer
+from pyhusky.frontend.library.register import register
 
 class Receiver:
     receiver_map = dict()

frontend/env.py → python/pyhusky/frontend/env.py

@@ -14,12 +14,12 @@
 
 import cPickle
 
-import config
-import session
-from huskylist import PyHuskyList, HDFS, MongoDB
-from operation import Operation, OperationParam
-from datareceiver import Receiver
-from backend.python.serializers import Serializer, MarshalSerializer, PickleSerializer, AutoSerializer, CompressedSerializer
+from pyhusky.common.operation import Operation, OperationParam
+from pyhusky.common.serializers import Serializer, MarshalSerializer, PickleSerializer, AutoSerializer, CompressedSerializer
+from pyhusky.frontend import config
+from pyhusky.frontend import session
+from pyhusky.frontend.datareceiver import Receiver
+from pyhusky.frontend.huskylist import PyHuskyList, HDFS, MongoDB
 
 # three types of lists:
 # HuskyList, PyHuskyList, HuskyObjList

frontend/huskylist.py → python/pyhusky/frontend/huskylist.py

@@ -16,12 +16,10 @@
 import string
 import types
 
-import config
-import session
-from operation import Operation, OperationParam 
-from scheduler import compute, compute_collect
-
-import copy
+from pyhusky.common.operation import Operation, OperationParam 
+from pyhusky.frontend import config
+from pyhusky.frontend import session
+from pyhusky.frontend.scheduler import compute, compute_collect
 
 def gen_list_name():
     list_name_len = 8
@@ -235,4 +233,3 @@ def load(self, database=None, collection=None):
         }
         self.pending_op = Operation("Functional#load_py", param, [])
         return self
-

frontend/library/linear_regression.py → python/pyhusky/frontend/library/linear_regression.py

@@ -14,10 +14,10 @@
 
 import numpy as np
 
-from frontend.huskylist import HuskyList
-from frontend.huskylist import PyHuskyList
-from frontend.operation import Operation, OperationParam
-from frontend import scheduler
+from pyhusky.common.operation import Operation, OperationParam
+from pyhusky.frontend import scheduler
+from pyhusky.frontend.huskylist import HuskyList
+from pyhusky.frontend.huskylist import PyHuskyList
 
 class LinearRegressionModel(HuskyList):
     def __init__(self, n_feature = -1):

frontend/library/logistic_regression.py → python/pyhusky/frontend/library/logistic_regression.py

@@ -14,10 +14,10 @@
 
 import numpy as np
 
-from frontend.huskylist import HuskyList
-from frontend.huskylist import PyHuskyList
-from frontend.operation import Operation, OperationParam
-from frontend import scheduler
+from pyhusky.common.operation import Operation, OperationParam
+from pyhusky.frontend import scheduler
+from pyhusky.frontend.huskylist import HuskyList
+from pyhusky.frontend.huskylist import PyHuskyList
 
 class LogisticRegressionModel(HuskyList):
     def __init__(self, n_feature = -1):

python/pyhusky/frontend/library/register.py

@@ -0,0 +1,20 @@
+# Copyright 2016 Husky Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pyhusky.frontend.library.linear_regression_receiver import LinearRegressionModelReceiver
+from pyhusky.frontend.library.logistic_regression_receiver import LogisticRegressionModelReceiver
+
+def register(receiver_map):
+    LinearRegressionModelReceiver.register(receiver_map)
+    LogisticRegressionModelReceiver.register(receiver_map)

frontend/library/svm.py → python/pyhusky/frontend/library/svm.py

@@ -14,10 +14,10 @@
 
 import numpy as np
 
-from frontend.huskylist import HuskyList
-from frontend.huskylist import PyHuskyList
-from frontend.operation import Operation, OperationParam
-from frontend import scheduler
+from pyhusky.common.operation import Operation, OperationParam
+from pyhusky.frontend import scheduler
+from pyhusky.frontend.huskylist import HuskyList
+from pyhusky.frontend.huskylist import PyHuskyList
 
 class SVMModel(HuskyList):
     def __init__(self, n_feature = -1):

frontend/scheduler.py → python/pyhusky/frontend/scheduler.py

@@ -13,18 +13,17 @@
 # limitations under the License.
 
 import cPickle
+import random
 import struct
-import zmq
 import time
-import random
-
-import communication
-import config
-import session
-from binstream import BinStream
-from datareceiver import data_receiver
-from operation import Operation
+import zmq
 
+from pyhusky.common.binstream import BinStream
+from pyhusky.common.operation import Operation
+from pyhusky.frontend import communication
+from pyhusky.frontend import config
+from pyhusky.frontend import session
+from pyhusky.frontend.datareceiver import data_receiver
 
 """
 structure of binstream