Migrate to pyproject.toml and modernize project configuration, fix some issues

.github/workflows/test.yml

@@ -12,30 +12,55 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+
+    - name: Set up JDK 21
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: '21'
+
+    - name: Create and activate virtualenv
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+        python -m pip install --upgrade pip
+
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip setuptools wheel
-        pip install pytest pytest-xdist # Testing packages
-        python setup.py install_egg_info # Workaround https://github.com/pypa/pip/issues/4537
-        pip install -e . # Run pytest
+        source venv/bin/activate
+        pip install setuptools wheel build pytest pytest-xdist
+        python -m build --sdist --wheel
+        pip install dist/*.whl
+
+    - name: Verify installed packages
+      run: |
+        source venv/bin/activate
+        pip list
+
     - name: Import language_tool_python
       run: |
+        source venv/bin/activate
         printf "import language_tool_python\n" | python
+
     - name: Test with pytest
       run: |
+        source venv/bin/activate
         pytest -vx --dist=loadfile -n auto
-    #- name: Run command-line tests
-    #  run: |
-    #    ./tests/test_local.bash  # Test command-line with local server
-    #    ./tests/test_remote.bash # Test command-line with remote server
+
+    - name: Run additional tests in bash scripts for Ubuntu
+      run: |
+        source venv/bin/activate
+        bash tests/test_local.bash
+        bash tests/test_remote.bash

.gitignore

@@ -8,3 +8,12 @@ __pycache__/
 build/
 dist/
 language_tool_python/LanguageTool-*/
+language_tool_python-*/
+pytest-cache-files-*/
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

MANIFEST.in

@@ -1,9 +1,12 @@
-exclude Makefile
-include COPYING
-include README.rst
 include run_doctest.py
-include test.bash
-include test.py
-include test_remote.bash
-include requirements.txt
-exclude language_tool_python/Language-Tool-*
+include tests/test_local.bash
+include tests/test_remote.bash
+prune pytest-cache-files-*/
+prune env/
+prune .env
+prune .venv
+prune env/
+prune venv/
+prune ENV/
+prune env.bak/
+prune venv.bak/

Makefile

@@ -15,4 +15,4 @@ check:
 		./language_tool_python \
 		$(wildcard ./language_tool_python/*.py) \
 		$(wildcard *.py)
-	python setup.py --long-description | rstcheck -
+	python extract_long_description.py | rstcheck -

README.md

@@ -264,7 +264,7 @@ The default download path is `~/.cache/language_tool_python/`. The LanguageTool
 
 ## Prerequisites
 
-- [Python 3.6+](https://www.python.org)
+- [Python 3.9+](https://www.python.org)
 - [LanguageTool](https://www.languagetool.org) (Java 8.0 or higher)
 
 The installation process should take care of downloading LanguageTool (it may

build_and_publish.sh

@@ -1,4 +1,5 @@
-rm -rf build/ dist/
-python setup.py sdist bdist_wheel
+rm -rf build/ dist/ *.egg-info/
+python -m pip install --upgrade pip setuptools wheel build
+python -m build
 twine check dist/*
 twine upload dist/* --verbose

extract_long_description.py

@@ -0,0 +1,13 @@
+import toml
+import os
+
+with open("pyproject.toml", "rb") as f:
+    pyproject = toml.loads(f.read().decode('utf-8'))
+
+readme_path = pyproject["project"]["readme"]
+
+if os.path.exists(readme_path):
+    with open(readme_path, "r", encoding="utf-8") as readme_file:
+        print(readme_file.read())
+else:
+    raise FileNotFoundError(f"{readme_path} not found.")

language_tool_python/__main__.py

@@ -4,12 +4,17 @@
 import locale
 import re
 import sys
+from importlib.metadata import version
+import toml
 
 from .server import LanguageTool
 from .utils import LanguageToolError
 
-import pkg_resources
-__version__ = pkg_resources.require("language_tool_python")[0].version
+try:
+    __version__ = version("language_tool_python")
+except PackageNotFoundError:
+    with open("pyproject.toml", "rb") as f:
+        __version__ = toml.loads(f.read().decode('utf-8'))["project"]["version"]
 
 
 def parse_args():
@@ -33,6 +38,8 @@ def parse_args():
     parser.add_argument('--enabled-only', action='store_true',
                         help='disable all rules except those specified in '
                              '--enable')
+    parser.add_argument('-p', '--picky', action='store_true',
+                        help='If set, additional rules will be activated.')
     parser.add_argument(
         '--version', action='version',
         version='%(prog)s {}'.format(__version__),
@@ -77,14 +84,6 @@ def get_text(filename, encoding, ignore):
     return text
 
 
-def print_unicode(text):
-    """Print in a portable manner."""
-    if sys.version_info[0] < 3:
-        text = text.encode('utf-8')
-
-    print(text)
-
-
 def main():
     args = parse_args()
 
@@ -109,44 +108,30 @@ def main():
             if args.remote_port is not None:
                 remote_server += ':{}'.format(args.remote_port)
         lang_tool = LanguageTool(
+            language=args.language,
             motherTongue=args.mother_tongue,
             remote_server=remote_server,
         )
-        guess_language = None
 
         try:
             text = get_text(filename, encoding, ignore=args.ignore_lines)
         except UnicodeError as exception:
             print('{}: {}'.format(filename, exception), file=sys.stderr)
             continue
 
-        if args.language:
-            if args.language.lower() == 'auto':
-                try:
-                    from guess_language import guess_language
-                except ImportError:
-                    print('guess_language is unavailable.', file=sys.stderr)
-                    return 1
-                else:
-                    language = guess_language(text)
-                    print('Detected language: {}'.format(language),
-                          file=sys.stderr)
-                    if not language:
-                        return 1
-                    lang_tool.language = language
-            else:
-                lang_tool.language = args.language
-
         if not args.spell_check:
             lang_tool.disable_spellchecking()
 
         lang_tool.disabled_rules.update(args.disable)
         lang_tool.enabled_rules.update(args.enable)
         lang_tool.enabled_rules_only = args.enabled_only
 
+        if args.picky:
+            lang_tool.picky = True
+
         try:
             if args.apply:
-                print_unicode(lang_tool.correct(text))
+                print(lang_tool.correct(text))
             else:
                 for match in lang_tool.check(text):
                     rule_id = match.ruleId
@@ -162,7 +147,7 @@ def main():
                     if replacement_text and not message.endswith(('.', '?')):
                         message += '; suggestions: ' + replacement_text
 
-                    print_unicode('{}: {}: {}'.format(
+                    print('{}: {}: {}'.format(
                         filename,
                         rule_id,
                         message))

language_tool_python/config_file.py

@@ -5,12 +5,12 @@
 import tempfile
 
 ALLOWED_CONFIG_KEYS = { 
-    'maxTextLength', 'maxTextHardLength', 'secretTokenKey', 'maxCheckTimeMillis', 'maxErrorsPerWordRate',
-    'maxSpellingSuggestions', 'maxCheckThreads', 'cacheSize', 'cacheTTLSeconds', 'cacheSize', 'requestLimit',
+    'maxTextLength', 'maxTextHardLength', 'maxCheckTimeMillis', 'maxErrorsPerWordRate',
+    'maxSpellingSuggestions', 'maxCheckThreads', 'cacheSize', 'cacheTTLSeconds', 'requestLimit',
     'requestLimitInBytes', 'timeoutRequestLimit', 'requestLimitPeriodInSeconds', 'languageModel',
-    'word2vecModel', 'fasttextModel', 'fasttextBinary', 'maxWorkQueueSize', 'rulesFile', 'warmUp',
+    'fasttextModel', 'fasttextBinary', 'maxWorkQueueSize', 'rulesFile',
     'blockedReferrers', 'premiumOnly', 'disabledRuleIds', 'pipelineCaching', 'maxPipelinePoolSize',
-    'pipelineCaching', 'pipelineExpireTimeInSeconds', 'pipelinePrewarming'
+    'pipelineExpireTimeInSeconds', 'pipelinePrewarming'
 }
 class LanguageToolConfig:
     config: Dict[str, Any]
@@ -19,6 +19,15 @@ def __init__(self, config: Dict[str, Any]):
         assert set(config.keys()) <= ALLOWED_CONFIG_KEYS, f"unexpected keys in config: {set(config.keys()) - ALLOWED_CONFIG_KEYS}"
         assert len(config), "config cannot be empty"
         self.config = config
+
+        if 'disabledRuleIds' in self.config:
+            self.config['disabledRuleIds'] = ','.join(self.config['disabledRuleIds'])
+        if 'blockedReferrers' in self.config:
+            self.config['blockedReferrers'] = ','.join(self.config['blockedReferrers'])
+        for key in ["pipelineCaching", "premiumOnly", "pipelinePrewarming"]:
+            if key in self.config:
+                self.config[key] = str(bool(self.config[key])).lower()
+
         self.path = self._create_temp_file()
 
     def _create_temp_file(self) -> str:

language_tool_python/download_lt.py

@@ -31,7 +31,7 @@
 BASE_URL = os.environ.get('LTP_DOWNLOAD_HOST', 'https://www.languagetool.org/download/')
 FILENAME = 'LanguageTool-{version}.zip'
 
-LTP_DOWNLOAD_VERSION = '6.4'
+LTP_DOWNLOAD_VERSION = '6.5'
 
 JAVA_VERSION_REGEX = re.compile(
     r'^(?:java|openjdk) version "(?P<major1>\d+)(|\.(?P<major2>\d+)\.[^"]+)"',
@@ -114,8 +114,9 @@ def http_get(url, out_file, proxies=None):
     total = int(content_length) if content_length is not None else None
     if req.status_code == 403:  # Not found on AWS
         raise Exception('Could not find at URL {}.'.format(url))
+    version = re.search(r'(\d+\.\d+)', url).group(1)
     progress = tqdm.tqdm(unit="B", unit_scale=True, total=total,
-                         desc=f'Downloading LanguageTool {LTP_DOWNLOAD_VERSION}')
+                         desc=f'Downloading LanguageTool {version}')
     for chunk in req.iter_content(chunk_size=1024):
         if chunk:  # filter out keep-alive new chunks
             progress.update(len(chunk))