Skip to content

Commit

Permalink
Do not send staging token to production (#2811)
Browse files Browse the repository at this point in the history
* Do not send staging token to production

* more fixed tests

* lint

* fix last test
  • Loading branch information
Wauplin authored Jan 30, 2025
1 parent 1033d0a commit 2fdc6f4
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 55 deletions.
8 changes: 0 additions & 8 deletions tests/test_cache_no_symlinks.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from huggingface_hub.constants import CONFIG_NAME, HF_HUB_CACHE
from huggingface_hub.file_download import are_symlinks_supported

from .testing_constants import TOKEN
from .testing_utils import DUMMY_MODEL_ID, with_production_testing


Expand Down Expand Up @@ -55,7 +54,6 @@ def test_download_no_symlink_new_file(self, mock_are_symlinks_supported: Mock) -
filename=CONFIG_NAME,
cache_dir=self.cache_dir,
local_files_only=False,
use_auth_token=TOKEN,
)
)
# Not a symlink !
Expand All @@ -74,7 +72,6 @@ def test_download_no_symlink_existing_file(self, mock_are_symlinks_supported: Mo
filename=CONFIG_NAME,
cache_dir=self.cache_dir,
local_files_only=False,
use_auth_token=TOKEN,
)
)
self.assertTrue(filepath.is_symlink())
Expand All @@ -92,7 +89,6 @@ def test_download_no_symlink_existing_file(self, mock_are_symlinks_supported: Mo
filename=CONFIG_NAME,
cache_dir=self.cache_dir,
local_files_only=False,
use_auth_token=TOKEN,
)
)
# File exist but is not a symlink
Expand All @@ -116,23 +112,20 @@ def test_scan_and_delete_cache_no_symlinks(self, mock_are_symlinks_supported: Mo
DUMMY_MODEL_ID,
filename=CONFIG_NAME,
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)

# Download README.md from main
hf_hub_download(
DUMMY_MODEL_ID,
filename="README.md",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)

# Download config.json from older revision
hf_hub_download(
DUMMY_MODEL_ID,
filename=CONFIG_NAME,
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
revision=OLDER_REVISION,
)

Expand All @@ -144,7 +137,6 @@ def test_scan_and_delete_cache_no_symlinks(self, mock_are_symlinks_supported: Mo
DUMMY_MODEL_ID,
filename="merges.txt",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
revision=OLDER_REVISION,
)

Expand Down
12 changes: 6 additions & 6 deletions tests/test_file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,18 +191,18 @@ def test_file_not_found_locally_and_network_disabled(self):
)

def test_private_repo_and_file_cached_locally(self):
api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
repo_id = api.create_repo(repo_id=repo_name(), private=True).repo_id
api.upload_file(path_or_fileobj=b"content", path_in_repo=constants.CONFIG_NAME, repo_id=repo_id)
api = HfApi(endpoint=ENDPOINT_STAGING)
repo_id = api.create_repo(repo_id=repo_name(), private=True, token=TOKEN).repo_id
api.upload_file(path_or_fileobj=b"content", path_in_repo="config.json", repo_id=repo_id, token=TOKEN)

with SoftTemporaryDirectory() as tmpdir:
# Download a first time with token => file is cached
filepath_1 = hf_hub_download(DUMMY_MODEL_ID, filename=constants.CONFIG_NAME, cache_dir=tmpdir, token=TOKEN)
filepath_1 = api.hf_hub_download(repo_id, filename="config.json", cache_dir=tmpdir, token=TOKEN)

# Download without token => return cached file
filepath_2 = hf_hub_download(DUMMY_MODEL_ID, filename=constants.CONFIG_NAME, cache_dir=tmpdir)
filepath_2 = api.hf_hub_download(repo_id, filename="config.json", cache_dir=tmpdir, token=False)

self.assertEqual(filepath_1, filepath_2)
assert filepath_1 == filepath_2

def test_file_cached_and_read_only_access(self):
"""Should works if file is already cached and user has read-only permission.
Expand Down
47 changes: 6 additions & 41 deletions tests/test_utils_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
_try_delete_path,
)

from .testing_constants import TOKEN
from .testing_utils import (
rmtree_with_retry,
with_production_testing,
Expand Down Expand Up @@ -64,48 +63,19 @@ class TestValidCacheUtils(unittest.TestCase):
def setUp(self) -> None:
"""Setup a clean cache for tests that will remain valid in all tests."""
# Download latest main
snapshot_download(
repo_id=MODEL_ID,
repo_type="model",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)
snapshot_download(repo_id=MODEL_ID, repo_type="model", cache_dir=self.cache_dir)

# Download latest commit which is same as `main`
snapshot_download(
repo_id=MODEL_ID,
revision=REPO_A_MAIN_HASH,
repo_type="model",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)
snapshot_download(repo_id=MODEL_ID, revision=REPO_A_MAIN_HASH, repo_type="model", cache_dir=self.cache_dir)

# Download the first commit
snapshot_download(
repo_id=MODEL_ID,
revision=REPO_A_OTHER_HASH,
repo_type="model",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)
snapshot_download(repo_id=MODEL_ID, revision=REPO_A_OTHER_HASH, repo_type="model", cache_dir=self.cache_dir)

# Download from a PR
snapshot_download(
repo_id=MODEL_ID,
revision="refs/pr/1",
repo_type="model",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)
snapshot_download(repo_id=MODEL_ID, revision="refs/pr/1", repo_type="model", cache_dir=self.cache_dir)

# Download a Dataset repo from "main"
snapshot_download(
repo_id=DATASET_ID,
revision="main",
repo_type="dataset",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)
snapshot_download(repo_id=DATASET_ID, revision="main", repo_type="dataset", cache_dir=self.cache_dir)

@unittest.skipIf(os.name == "nt", "Windows cache is tested separately")
def test_scan_cache_on_valid_cache_unix(self) -> None:
Expand Down Expand Up @@ -357,12 +327,7 @@ class TestCorruptedCacheUtils(unittest.TestCase):
def setUp(self) -> None:
"""Setup a clean cache for tests that will get corrupted/modified in tests."""
# Download latest main
snapshot_download(
repo_id=MODEL_ID,
repo_type="model",
cache_dir=self.cache_dir,
use_auth_token=TOKEN,
)
snapshot_download(repo_id=MODEL_ID, repo_type="model", cache_dir=self.cache_dir)

self.repo_path = self.cache_dir / MODEL_PATH
self.refs_path = self.repo_path / "refs"
Expand Down

0 comments on commit 2fdc6f4

Please sign in to comment.