Initial import (#1)

* Scaffolding + ~ import from transformers

* move more things around

* Fixes/deletes from copy/pastes

* GH Actions

.github/workflows/python-quality.yml

@@ -0,0 +1,24 @@
+name: Python quality
+
+on:
+  push:
+    branches:
+      - "*"
+
+jobs:
+  check_code_quality:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install .[dev]
+    - run: black --check tests src
+    - run: isort --check-only tests src
+    - run: flake8 tests src

.github/workflows/python-tests.yml

@@ -0,0 +1,31 @@
+name: Python tests
+
+on:
+  push:
+    branches:
+      - "*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9"]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - run: |
+        git config --global user.email "[email protected]"
+        git config --global user.name "ci"
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install .[testing]
+    - name: pytest
+      run: RUN_GIT_LFS_TESTS=1 pytest -sv ./tests/

.gitignore

@@ -127,3 +127,4 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+.vscode/

Makefile

@@ -0,0 +1,18 @@
+.PHONY: quality style test
+
+
+check_dirs := tests src
+
+
+quality:
+	black --check $(check_dirs)
+	isort --check-only $(check_dirs)
+	flake8 $(check_dirs)
+
+style:
+	black $(check_dirs)
+	isort $(check_dirs)
+
+test:
+	pytest -sv ./tests/
+

setup.cfg

@@ -0,0 +1,50 @@
+[isort]
+default_section = FIRSTPARTY
+ensure_newline_before_comments = True
+force_grid_wrap = 0
+include_trailing_comma = True
+known_first_party = huggingface_hub
+known_third_party =
+    absl
+    conllu
+    datasets
+    elasticsearch
+    fairseq
+    faiss-cpu
+    fastprogress
+    fire
+    fugashi
+    git
+    h5py
+    matplotlib
+    nltk
+    numpy
+    packaging
+    pandas
+    PIL
+    psutil
+    pytest
+    pytorch_lightning
+    rouge_score
+    sacrebleu
+    seqeval
+    sklearn
+    streamlit
+    tensorboardX
+    tensorflow
+    tensorflow_datasets
+    timeout_decorator
+    torch
+    torchtext
+    torchvision
+    torch_xla
+    tqdm
+
+line_length = 88
+lines_after_imports = 2
+multi_line_output = 3
+use_parentheses = True
+
+[flake8]
+ignore = E203, E501, E741, W503, W605
+max-line-length = 88

setup.py

@@ -0,0 +1,67 @@
+from setuptools import find_packages, setup
+
+
+def get_version() -> str:
+    rel_path = "src/huggingface_hub/__init__.py"
+    with open(rel_path, "r") as fp:
+        for line in fp.read().splitlines():
+            if line.startswith("__version__"):
+                delim = '"' if '"' in line else "'"
+                return line.split(delim)[1]
+    raise RuntimeError("Unable to find version string.")
+
+
+install_requires = [
+    "filelock",
+    "requests",
+    "tqdm",
+]
+
+extras = {}
+
+extras["testing"] = [
+    "pytest",
+]
+
+extras["quality"] = [
+    "black>=20.8b1",
+    "isort>=5.5.4",
+    "flake8>=3.8.3",
+]
+
+extras["all"] = extras["testing"] + extras["quality"]
+
+extras["dev"] = extras["all"]
+
+
+setup(
+    name="huggingface_hub",
+    version=get_version(),
+    author="Hugging Face, Inc.",
+    author_email="[email protected]",
+    description="Client library to download and publish models on the huggingface.co hub",
+    long_description=open("README.md", "r", encoding="utf-8").read(),
+    long_description_content_type="text/markdown",
+    keywords="model-hub machine-learning models natural-language-processing deep-learning pytorch pretrained-models",
+    license="Apache",
+    url="https://github.com/huggingface/huggingface_hub",
+    package_dir={"": "src"},
+    packages=find_packages("src"),
+    extras_require=extras,
+    entry_points={
+        "console_scripts": [
+            "huggingface-cli=huggingface_hub.commands.huggingface_cli:main"
+        ]
+    },
+    python_requires=">=3.6.0",
+    install_requires=install_requires,
+    classifiers=[
+        "Intended Audience :: Developers",
+        "Intended Audience :: Education",
+        "Intended Audience :: Science/Research",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ],
+)

src/huggingface_hub/__init__.py

@@ -0,0 +1,22 @@
+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.1.0"
+
+from .file_download import HUGGINGFACE_CO_URL_TEMPLATE, cached_download, hf_hub_url
+from .hf_api import HfApi, HfFolder

src/huggingface_hub/commands/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from argparse import ArgumentParser
+
+
+class BaseHuggingfaceCLICommand(ABC):
+    @staticmethod
+    @abstractmethod
+    def register_subcommand(parser: ArgumentParser):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def run(self):
+        raise NotImplementedError()

src/huggingface_hub/commands/huggingface_cli.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from argparse import ArgumentParser
+
+from huggingface_hub.commands.lfs import LfsCommands
+from huggingface_hub.commands.user import UserCommands
+
+
+def main():
+    parser = ArgumentParser(
+        "huggingface-cli", usage="huggingface-cli <command> [<args>]"
+    )
+    commands_parser = parser.add_subparsers(help="huggingface-cli command helpers")
+
+    # Register commands
+    UserCommands.register_subcommand(commands_parser)
+    LfsCommands.register_subcommand(commands_parser)
+
+    # Let's go
+    args = parser.parse_args()
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        exit(1)
+
+    # Run
+    service = args.func(args)
+    service.run()
+
+
+if __name__ == "__main__":
+    main()