From fe9fd56edba6de8ca758fb6c1a63d254308b17a3 Mon Sep 17 00:00:00 2001 From: Prachi Manchanda Date: Tue, 8 Aug 2017 19:35:17 +0530 Subject: [PATCH] FileGlobs.py: Recursively extract .npmignore globs Looks for .npmignore in dirs and subdirs recursively and extract file globs Closes https://github.com/coala/coala-quickstart/issues/109 --- coala_quickstart/generation/FileGlobs.py | 18 +++- coala_quickstart/generation/Utilities.py | 30 +++++- tests/generation/FileGlobs.py | 100 +++++++++++++++++- .../file_globs_npmignore_testfiles/.coafile | 0 .../file_globs_npmignore_testfiles/.gitignore | 11 ++ .../other_folder/.npmignore | 5 + .../other_folder/new_file.c | 0 .../other_folder/test.html | 0 .../sample_data/data/.npmignore | 5 + .../sample_data/data/new_script.js | 0 .../sample_data/data/test.css | 0 .../sample_data/example.py | 0 12 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 tests/generation/file_globs_npmignore_testfiles/.coafile create mode 100644 tests/generation/file_globs_npmignore_testfiles/.gitignore create mode 100644 tests/generation/file_globs_npmignore_testfiles/other_folder/.npmignore create mode 100644 tests/generation/file_globs_npmignore_testfiles/other_folder/new_file.c create mode 100644 tests/generation/file_globs_npmignore_testfiles/other_folder/test.html create mode 100644 tests/generation/file_globs_npmignore_testfiles/sample_data/data/.npmignore create mode 100644 tests/generation/file_globs_npmignore_testfiles/sample_data/data/new_script.js create mode 100644 tests/generation/file_globs_npmignore_testfiles/sample_data/data/test.css create mode 100644 tests/generation/file_globs_npmignore_testfiles/sample_data/example.py diff --git a/coala_quickstart/generation/FileGlobs.py b/coala_quickstart/generation/FileGlobs.py index fbbf45f..44f60da 100644 --- a/coala_quickstart/generation/FileGlobs.py +++ b/coala_quickstart/generation/FileGlobs.py @@ -1,7 +1,8 @@ import os from coalib.parsing.Globbing import glob_escape -from coala_quickstart.generation.Utilities import get_gitignore_glob +from coala_quickstart.generation.Utilities import ( + get_gitignore_glob, get_npmignore_glob) from coala_utils.Question import ask_question from coala_quickstart.Strings import GLOB_HELP from coalib.collecting.Collectors import collect_files @@ -30,6 +31,21 @@ def get_project_files(log_printer, printer, project_dir, non_interactive=False): color="green") ignore_globs = get_gitignore_glob(project_dir) + npmignore_dir_list = [] + + for dir_name, subdir_name, files in os.walk(project_dir): + if(os.path.isfile(os.path.join(dir_name, ".npmignore"))): + npmignore_dir_list += [dir_name] + + if(npmignore_dir_list): + printer.print("The contents of your .npmignore file for the project " + "will be automatically loaded as files to ignore.", + color="green") + if ignore_globs is None: + ignore_globs = get_npmignore_glob(project_dir, npmignore_dir_list) + else: + ignore_globs += get_npmignore_glob(project_dir, npmignore_dir_list) + if non_interactive and not ignore_globs: ignore_globs = [] diff --git a/coala_quickstart/generation/Utilities.py b/coala_quickstart/generation/Utilities.py index 506a82c..8670c9b 100644 --- a/coala_quickstart/generation/Utilities.py +++ b/coala_quickstart/generation/Utilities.py @@ -18,11 +18,12 @@ def is_glob_exp(line): return sum(1 for x in results) != 0 -def parse_gitignore_line(line): +def parse_ignore_line(line): """ - Parses the line from ``.gitignore`` and returns a list of globs. + Parses the line from ``.gitignore`` and ``.npmignore`` + and returns a list of globs. - :param line: A line from the project's ``.gitignore`` file. + :param line: A line from the project's ``.gitignore`` or ``.npmignore`` file :return: A list of glob expressions translated to the syntax used in coala globbing. """ @@ -72,10 +73,31 @@ def get_gitignore_glob(project_dir, filename=".gitignore"): with open(gitignore, "r") as file: for line in file: - for glob in parse_gitignore_line(line): + for glob in parse_ignore_line(line): yield os.path.join(project_dir, glob) +def get_npmignore_glob(project_dir, npmignore_dir_list, filename=".npmignore"): + """ + Generates a list of glob expressions equivalent to the + contents of the user's project's ``.npmignore`` file. + + :param project_dir: + The user's project directory. + :param npmignore_dir_list: + A list of directories in project containing .npmignore + :return: + A list generator of glob expressions generated from the + ``.npmignore`` file. + """ + for dir_name in npmignore_dir_list: + npmignore = os.path.join(dir_name, filename) + with open(npmignore, "r") as file: + for line in file: + for glob in parse_ignore_line(line): + yield os.path.join(dir_name, glob) + + def split_by_language(project_files): """ Splits the given files based on language. This ignores unknown extensions. diff --git a/tests/generation/FileGlobs.py b/tests/generation/FileGlobs.py index b814bd4..b5c7abd 100644 --- a/tests/generation/FileGlobs.py +++ b/tests/generation/FileGlobs.py @@ -6,7 +6,8 @@ from coala_utils.ContextManagers import ( simulate_console_inputs, suppress_stdout) from coala_quickstart.generation.FileGlobs import get_project_files -from coala_quickstart.generation.Utilities import get_gitignore_glob +from coala_quickstart.generation.Utilities import ( + get_gitignore_glob, get_npmignore_glob) from coalib.collecting.Collectors import collect_files @@ -102,6 +103,103 @@ def test_get_project_files_gitignore(self): os.remove(".gitignore") os.chdir(orig_cwd) + def test_get_project_files_npmignore(self): + orig_cwd = os.getcwd() + os.chdir(os.path.dirname(os.path.realpath(__file__))) + os.makedirs("file_globs_npmignore_testfiles", exist_ok=True) + os.chdir("file_globs_npmignore_testfiles") + + with open(".gitignore", "w") as f: + f.write(""" +# Start of gitignore +buildtest +ignore.c +/testignore +/upload.c +/*.py +*.pyc +__pycache__ +# End of gitignore + """) + + os.makedirs("other_folder", exist_ok=True) + os.chdir("other_folder") + with open('.npmignore', "w") as file: + file.write(""" +#Start of npmignore +*.html +#End of npmignore + """) + os.chdir(os.path.dirname(os.path.realpath(__file__))) + os.chdir("file_globs_npmignore_testfiles") + os.makedirs("sample_data", exist_ok=True) + os.chdir("sample_data") + os.makedirs("data", exist_ok=True) + os.chdir("data") + with open('.npmignore', "w") as file: + file.write(""" +#Start of npmignore +*.css +#End of npmignore + """) + os.chdir(os.path.dirname(os.path.realpath(__file__))) + os.chdir("file_globs_npmignore_testfiles") + files = [os.path.join("src", "main.c"), + os.path.join("src", "main.h"), + os.path.join("src", "lib", "ssl.c"), + os.path.join("src", "tests", "main.c"), + os.path.join("src", "abc.py"), + os.path.join("src", "upload.c"), + os.path.join("other_folder", "new_file.c"), + os.path.join("sample_data", "data", "new_script.js"), + os.path.join("sample_data", "example.py"), + ".coafile"] + ignored_files = [os.path.join("buildtest", "main.c"), + os.path.join("testignore", "run.c"), + os.path.join("src", "build", "main.c"), + "ignore.c", + os.path.join("src", "ignore.c"), + "glob2.py", + "upload.c", + os.path.join("src", "abc.pyc"), + os.path.join("other_folder", "test.html"), + os.path.join("sample_data", "data", "test.css"), + "run.pyc"] + + for file in files + ignored_files: + os.makedirs(os.path.dirname(os.path.abspath(file)), exist_ok=True) + open(file, "w").close() + files += [".gitignore"] + files += [os.path.join("other_folder", ".npmignore")] + files += [os.path.join("sample_data", "data", ".npmignore")] + + gitignore_dir_list = [os.path.join(os.getcwd())] + npmignore_dir_list = [os.path.join(os.getcwd(), "other_folder"), + os.path.join(os.getcwd(), "sample_data", "data")] + + globs = list(get_gitignore_glob(os.getcwd(), gitignore_dir_list)) + globs += list(get_npmignore_glob(os.getcwd(), npmignore_dir_list)) + + returned_files = collect_files( + [os.path.join(os.getcwd(), "**")], + self.log_printer, + ignored_file_paths=globs) + files = [os.path.normcase(os.path.abspath(file)) for file in files] + ignored_files = [os.path.abspath(file) for file in ignored_files] + self.maxDiff = None + self.assertEqual(sorted(files), sorted(returned_files)) + + with suppress_stdout(): + self.assertEqual( + sorted(get_project_files( + self.log_printer, self.printer, os.getcwd())[0]), + sorted(files)) + + os.remove(os.path.join("other_folder", ".npmignore")) + os.remove(os.path.join("sample_data", "data", ".npmignore")) + os.remove(".gitignore") + os.chdir(orig_cwd) + def test_get_project_files_ci_mode(self): orig_cwd = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__)) + diff --git a/tests/generation/file_globs_npmignore_testfiles/.coafile b/tests/generation/file_globs_npmignore_testfiles/.coafile new file mode 100644 index 0000000..e69de29 diff --git a/tests/generation/file_globs_npmignore_testfiles/.gitignore b/tests/generation/file_globs_npmignore_testfiles/.gitignore new file mode 100644 index 0000000..4fcc474 --- /dev/null +++ b/tests/generation/file_globs_npmignore_testfiles/.gitignore @@ -0,0 +1,11 @@ + +# Start of gitignore +buildtest +ignore.c +/testignore +/upload.c +/*.py +*.pyc +__pycache__ +# End of gitignore + \ No newline at end of file diff --git a/tests/generation/file_globs_npmignore_testfiles/other_folder/.npmignore b/tests/generation/file_globs_npmignore_testfiles/other_folder/.npmignore new file mode 100644 index 0000000..1a4a63f --- /dev/null +++ b/tests/generation/file_globs_npmignore_testfiles/other_folder/.npmignore @@ -0,0 +1,5 @@ + +#Start of npmignore +*.html +#End of npmignore + \ No newline at end of file diff --git a/tests/generation/file_globs_npmignore_testfiles/other_folder/new_file.c b/tests/generation/file_globs_npmignore_testfiles/other_folder/new_file.c new file mode 100644 index 0000000..e69de29 diff --git a/tests/generation/file_globs_npmignore_testfiles/other_folder/test.html b/tests/generation/file_globs_npmignore_testfiles/other_folder/test.html new file mode 100644 index 0000000..e69de29 diff --git a/tests/generation/file_globs_npmignore_testfiles/sample_data/data/.npmignore b/tests/generation/file_globs_npmignore_testfiles/sample_data/data/.npmignore new file mode 100644 index 0000000..319ce18 --- /dev/null +++ b/tests/generation/file_globs_npmignore_testfiles/sample_data/data/.npmignore @@ -0,0 +1,5 @@ + +#Start of npmignore +*.css +#End of npmignore + \ No newline at end of file diff --git a/tests/generation/file_globs_npmignore_testfiles/sample_data/data/new_script.js b/tests/generation/file_globs_npmignore_testfiles/sample_data/data/new_script.js new file mode 100644 index 0000000..e69de29 diff --git a/tests/generation/file_globs_npmignore_testfiles/sample_data/data/test.css b/tests/generation/file_globs_npmignore_testfiles/sample_data/data/test.css new file mode 100644 index 0000000..e69de29 diff --git a/tests/generation/file_globs_npmignore_testfiles/sample_data/example.py b/tests/generation/file_globs_npmignore_testfiles/sample_data/example.py new file mode 100644 index 0000000..e69de29