Merge remote-tracking branch 'origin/actions-configuration' into main

.github/workflows/actions.yml

@@ -2,22 +2,31 @@ name: Auto-Test-Generator
 on:
   workflow_dispatch:
   pull_request:
-    branches:
-      - main
+    types: [opened, reopened]
+env:
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  REPO_URL: ${{ github.repositoryUrl }}
 jobs:
   generate:
     name: Changelog Generator
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
-      - run: ls -la
-      - name: Create Changelog
-        uses: ./
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v2
+          fetch-depth: 0
+          ref: ${{ github.head_ref }}
+      - uses: actions/cache@v2
         with:
-          name: changelog
-          path: /home/runner/work/_temp/_github_home/CHANGELOG.md
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{hashFiles('environment.yml') }}
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: diploma
+          environment-file: environment.yml
+          python-version: 3.9
+          auto-activate-base: false
+          use-only-tar-bz2: true
+      - run: python -m nltk.downloader punkt stopwords
+      - run: python generate_tests_code.py
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4

eval_main.py

@@ -33,11 +33,11 @@ def main():
                                    directory_for_generation="get_tests_imports_fixes")
         saver.goto_commit()
         for prompt in commit.construct_prompt():
-            # try:
-            tests = generator.generate(prompt)
-            save_and_eval(tests, saver, bleu, compiler, commit)
-            # except Exception as e:
-            #     print(f"Tests for {prompt} not saved", e)
+            try:
+                tests = generator.generate(prompt)
+                save_and_eval(tests, saver, bleu, compiler, commit)
+            except Exception as e:
+                print(f"Tests for {prompt} not saved", e)
         saver.commit_files()
         saver.clean_state()
     bleu.export('blue_pydriller_import_fixes.csv')

generate_tests_code.py

@@ -1,47 +1,35 @@
-import re
-import shlex
-import subprocess
-
-
-def get_commit_log():
-    output = subprocess.check_output(shlex.split('git log --pretty=%s --color'),
-                                     stderr=subprocess.STDOUT)
-    output = output.decode('ascii')
-    output = output.split('\n')
-    return output
-
-
-def strip_commits(commits):
-    output = []
-    for line in commits:
-        if re.findall(r"(feat|fix|refactor|test|cli)", line):
-            output.append(line)
-    return output
-
-
-def overwrite_changelog(commits):
-    print("Going to write the following commits: \n{}".format(commits))
-    with open("./CHANGELOG.md", "w+") as f:
-        f.write("# CHANGELOG \n\n ## Features:\n\n")
-        for feat in commits:
-            if re.findall(r"^feat", feat):
-                f.write("* {}\n".format(feat))
-        f.write("\n## Bugs:\n\n")
-        for fix in commits:
-            if re.findall(r"^fix", fix):
-                f.write("* {}\n".format(fix))
-        f.write("\n## Others:")
-        for other in commits:
-            if re.findall(r"^(refactor|test|cli)", other):
-                f.write("* {}\n".format(other))
-        f.write("\n\n\n> Generated with Github actions")
-
-
-def main():
-    commits = get_commit_log()
-    commits = strip_commits(commits)
-    overwrite_changelog(commits)
+import uuid
+from pathlib import Path
+import os
+
+from generation.CodeCleanser import CodeCleanser
+from generation.Compilable import Compilable
+from generation.GeneratedTestSaver import GeneratedTestSaver
+from generation.Generator import Generator
+from generation.ImportResolver import ImportResolver
+from history_scanner.GitHistoryDataSetParser import GitHistoryDataSetParser
+
+
+def generate_test(branch):
+    path = Path('./')
+    setup_command = ''
+    parser = GitHistoryDataSetParser(str(path.absolute()), branch=f'{branch}', only_last_commit=True)
+    data = parser.get_parsed_data()
+
+    generator = Generator(CodeCleanser(str(path.absolute()), setup_command, ImportResolver(str(path)), Compilable()))
+
+    for commit in data:
+        saver = GeneratedTestSaver(str(path.absolute()), commit.commit_id, main_branch='master',
+                                   directory_for_generation=f'generated_tests_for_{commit.commit_id}')
+        for prompt in commit.construct_prompt():
+            try:
+                tests = generator.generate(prompt)
+                for test in tests:
+                    saver.save_test_file(test, f'{uuid.uuid4()}')
+            except Exception as e:
+                print(f"Tests for {prompt} not saved", e)
 
 
 if __name__ == "__main__":
-    print("Hello World")
+    branch_name = os.getenv("GITHUB_HEAD_REF")
+    generate_test(branch_name)

history_scanner/GitHistoryDataSetParser.py

@@ -9,29 +9,42 @@
 
 
 class GitHistoryDataSetParser:
-    def __init__(self, repository_path="./", branch="main"):
+    def __init__(self, repository_path="./", branch="main", only_last_commit=False):
         self.commit_msg_tokens = []
-        self.git_repo = Repository(repository_path, only_in_branch=branch, only_modifications_with_file_types=[".py"],
-                                   include_deleted_files=False)
         self.parsed_data = []
         self.stemmer = SnowballStemmer(language='english')
+        self.only_last_commit = only_last_commit
+        if only_last_commit:
+            self.git_repo = Repository(repository_path, only_in_branch=branch,
+                                       only_modifications_with_file_types=[".py"],
+                                       include_deleted_files=False,
+                                       order='reverse')
+        else:
+            self.git_repo = Repository(repository_path, only_in_branch=branch,
+                                       only_modifications_with_file_types=[".py"],
+                                       include_deleted_files=False)
 
     def parse_data(self):
-        self.parsed_data = list(filter(None, [self.process_commit(commit)
-                                              for commit in self.git_repo.traverse_commits()]))
+        if self.only_last_commit:
+            commits = [next(self.git_repo.traverse_commits())]
+            self.parsed_data = list(filter(None, [self.process_commit(commit)
+                                                  for commit in commits]))
+        else:
+            self.parsed_data = list(filter(None, [self.process_commit(commit)
+                                                  for commit in self.git_repo.traverse_commits()]))
 
     def get_parsed_data(self):
         if not self.parsed_data:
-            self.parsed_data = list(filter(None, [self.process_commit(commit)
-                                                  for commit in self.git_repo.traverse_commits()]))
+            self.parse_data()
         return self.parsed_data
 
     def process_commit(self, commit) -> CommitData:
+        print("Processing commit", commit.msg)
         commit_msg_tokens = word_tokenize(commit.msg)
         self.commit_msg_tokens = [self.stemmer.stem(w) for w in commit_msg_tokens]
 
         py_files, test_files = self.__get_edited_files_source(commit)
-        if len(py_files) > 0 and len(test_files) > 0:
+        if len(py_files) > 0 and (self.only_last_commit or len(test_files) > 0):
             commit_data = CommitData(commit.hash, commit.msg, py_files, test_files)
             print(commit_data)
             return commit_data

pip_requirements.txt

@@ -18,6 +18,7 @@ debugpy @ file:///C:/ci/debugpy_1637091961445/work
 decorator==4.4.2
 defusedxml @ file:///tmp/build/80754af9/defusedxml_1615228127516/work
 docker-pycreds==0.4.0
+docopt==0.6.2
 entrypoints @ file:///C:/ci/entrypoints_1649926621128/work
 et-xmlfile==1.0.1
 executing @ file:///opt/conda/conda-bld/executing_1646925071911/work
@@ -131,3 +132,4 @@ win-inet-pton @ file:///D:/bld/win_inet_pton_1635832858406/work
 win32-setctime==1.1.0
 wincertstore==0.2
 windb==0.0.2
+yarg==0.1.9