Run code coverage during tests (#2295)

.coveragerc

@@ -0,0 +1,10 @@
+[run]
+parallel = true
+concurrency = multiprocessing
+omit =
+    */env/*
+    */tests/*
+    cover.py
+source =
+    src/
+

.gitignore

@@ -71,6 +71,7 @@ command_coverage.txt
 private_config.json
 scripts/smart_create_gen/config.ini
 test_results/
+resources/
 
 # Code coverage
 .coverage

.travis.yml

@@ -4,8 +4,10 @@ python:
   - "2.7"
   - "3.5"
   - "3.6"
+env:
+  - CODE_COVERAGE="True"
 install:
   - pip install -qqq virtualenv # used by package_verify script
   - python scripts/dev_setup.py
-script: 
+script:
   - ./scripts/build.sh

scripts/automation/tests/nose_helper.py

@@ -3,15 +3,13 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-from datetime import datetime
-
 
 # pylint: disable=too-many-arguments
 def get_nose_runner(report_folder, parallel=True, process_timeout=600, process_restart=True,
-                    xunit_report=False, exclude_integration=True, code_coverage=False):
+                    xunit_report=False, exclude_integration=True):
     """Create a nose execution method"""
 
-    def _run_nose(name, working_dir):
+    def _run_nose(test_folders):
         import nose
         import os.path
 
@@ -20,40 +18,26 @@ def _run_nose(name, working_dir):
                 or not os.path.isdir(report_folder):
             raise ValueError('Report folder {} does not exist'.format(report_folder))
 
-        arguments = ['-w', working_dir, '-v']
+        arguments = [__file__, '-v']
         if parallel:
             arguments += ['--processes=-1', '--process-timeout={}'.format(process_timeout)]
             if process_restart:
                 arguments += ['--process-restartworker']
 
         if xunit_report:
-            log_file = os.path.join(report_folder, name + '-report.xml')
-            arguments += ['--with-xunit', '--xunit-file', log_file]
+            test_report = os.path.join(report_folder, 'nosetests-report.xml')
+            arguments += ['--with-xunit', '--xunit-file', test_report]
         else:
-            log_file = ''
+            test_report = ''
 
         if exclude_integration:
             arguments += ['--ignore-files=integration*']
 
-        if code_coverage:
-            # coverage_config = os.path.join(os.path.dirname(__file__), '.coveragerc')
-            # coverage_folder = os.path.join(report_folder, 'code_coverage')
-            # make_dirs(coverage_folder)
-            # if not os.path.exists(coverage_folder) or not os.path.isdir(coverage_folder):
-            #     raise Exception('Failed to create folder {} for code coverage result'
-            #                     .format(coverage_folder))
-
-            arguments += ['--with-coverage']
-
-        debug_file = os.path.join(report_folder, name + '-debug.log')
+        debug_file = os.path.join(report_folder, 'test-debug.log')
         arguments += ['--debug-log={}'.format(debug_file)]
+        arguments.extend(test_folders)
 
-        print('\n')
-        print('<<< Run {} >>>'.format(name))
-        start = datetime.now()
         result = nose.run(argv=arguments)
-        end = datetime.now()
-
-        return result, start, end, log_file
+        return result, test_report
 
     return _run_nose

scripts/automation/tests/run.py

@@ -29,19 +29,12 @@ def run_tests(modules, parallel, run_live):
         os.environ['AZURE_CLI_TEST_RUN_LIVE'] = 'True'
 
     # run tests
-    passed = True
-    module_results = []
-    for name, _, test_path in modules:
-        result, start, end, _ = run_nose(name, test_path)
-        passed &= result
-        record = (name, start.strftime('%H:%M:%D'), str((end - start).total_seconds()),
-                  'Pass' if result else 'Fail')
+    test_folders = [test_path for _, _, test_path in modules]
+    result, test_result = run_nose(test_folders)
 
-        module_results.append(record)
+    print('Test report: {}'.format(test_result))
 
-    print_records(module_results, title='test results')
-
-    return passed
+    return result
 
 
 if __name__ == '__main__':

scripts/build.sh

@@ -10,6 +10,7 @@ set -e
 scripts_root=$(cd $(dirname $0); pwd)
 
 export PYTHONPATH=$PATHONPATH:./src
+
 python -m azure.cli -h
 
 # PyLint does not yet support Python 3.6 https://github.com/PyCQA/pylint/issues/1241
@@ -21,7 +22,17 @@ else
     check_style --ci;
 fi
 
-run_tests
+if [ "$CODE_COVERAGE" == "True" ]; then
+    echo "Run tests with code coverage."
+    pip install -qqq coverage codecov
+    coverage run -m automation.tests.run
+
+    coverage combine
+    coverage report
+    codecov
+else
+    python -m automation.tests.run
+fi
 
 if [[ "$CI" == "true" ]]; then
     $scripts_root/package_verify.sh