Un check/filtered clone

outpostkit/repository/_loaders/transformers/constants.py

@@ -0,0 +1,14 @@
+PT_WEIGHTS_NAME = "pytorch_model.bin"
+PT_WEIGHTS_INDEX_NAME = "pytorch_model.bin.index.json"
+TF2_WEIGHTS_NAME = "tf_model.h5"
+TF2_WEIGHTS_INDEX_NAME = "tf_model.h5.index.json"
+TF_WEIGHTS_NAME = "model.ckpt"
+FLAX_WEIGHTS_NAME = "flax_model.msgpack"
+FLAX_WEIGHTS_INDEX_NAME = "flax_model.msgpack.index.json"
+SAFE_WEIGHTS_NAME = "model.safetensors"
+SAFE_WEIGHTS_INDEX_NAME = "model.safetensors.index.json"
+CONFIG_NAME = "config.json"
+FEATURE_EXTRACTOR_NAME = "preprocessor_config.json"
+IMAGE_PROCESSOR_NAME = FEATURE_EXTRACTOR_NAME
+PROCESSOR_NAME = "processor_config.json"
+GENERATION_CONFIG_NAME = "generation_config.json"

outpostkit/repository/_loaders/transformers/download.py

@@ -0,0 +1,83 @@
+import os
+from typing import Optional
+
+from outpostkit._types.repository import REPOSITORY_TYPES
+from outpostkit._utils import save_file_at_path_from_response, split_full_name
+from outpostkit.client import Client
+from outpostkit.repository import RepositoryAtRef
+
+
+def load_local_file_if_present(file_path: str):
+    if os.path.isfile(file_path):
+        with open(file_path) as file:
+            # Perform operations to load the file
+            # For example, you can read its contents:
+            file_contents = file.read()
+            return file_contents
+    else:
+        raise FileNotFoundError(f"The file '{file_path}' does not exist.")
+
+
+def is_file_present_locally(file_path: str):
+    if not os.path.isfile(file_path):
+        raise FileNotFoundError(f"The file '{file_path}' does not exist.")
+
+
+def download_file_from_repo(
+    repo_type: REPOSITORY_TYPES,
+    full_name: str,
+    file_path: str,
+    store_dir: str,
+    client: Optional[Client],
+    ref: str = "HEAD",
+):
+    try:
+        (repo_entity, repo_name) = split_full_name(full_name)
+    except ValueError:
+        raise FileNotFoundError(
+            f"Invalid {repo_type} repository fullName or path {full_name}"
+        ) from None
+
+    if client is None:
+        client = Client()
+    repo = RepositoryAtRef(
+        entity=repo_entity,
+        name=repo_name,
+        ref=ref,
+        repo_type=repo_type,
+        client=client,
+    )
+    get_file_resp = repo.download_blob(file_path, raw=True)
+    file_loc = os.path.join(store_dir, file_path)
+    save_file_at_path_from_response(get_file_resp, file_loc)
+    return file_loc
+
+
+def get_file(
+    full_name_or_dir: str,
+    repo_type: REPOSITORY_TYPES,
+    file_path: str,
+    store_dir: str,
+    ref: str = "HEAD",
+    token: Optional[str] = None,
+    client: Optional[Client] = None,
+    **kwargs,
+) -> str:
+    subfolder = kwargs.pop("subfolder")
+    if subfolder is not None:
+        file_path = os.path.join(subfolder, file_path)
+    if token and not Client:
+        client = Client(api_token=token)
+    if os.path.isdir(full_name_or_dir):
+        file_loc = os.path.join(full_name_or_dir, file_path)
+        is_file_present_locally(file_loc)
+        return file_loc
+    else:
+        return download_file_from_repo(
+            repo_type=repo_type,
+            store_dir=store_dir,
+            ref=ref,
+            client=client,
+            file_path=file_path,
+            full_name=full_name_or_dir,
+        )

outpostkit/repository/_loaders/transformers/peft.py

@@ -0,0 +1,42 @@
+from typing import Optional
+
+from outpostkit.client import Client
+from outpostkit.exceptions import OutpostHTTPException
+from outpostkit.logger import init_outpost_logger
+from outpostkit.repository._loaders.transformers.download import get_file
+
+ADAPTER_CONFIG_NAME = "adapter_config.json"
+ADAPTER_WEIGHTS_NAME = "adapter_model.bin"
+ADAPTER_SAFE_WEIGHTS_NAME = "adapter_model.safetensors"
+
+logger = init_outpost_logger(__name__)
+
+
+def find_adapter_config_file(
+    full_name_or_dir: str,
+    store_dir: str,
+    ref: str = "HEAD",
+    token: Optional[str] = None,
+    client: Optional[Client] = None,
+    **kwargs,
+) -> Optional[str]:
+    adapter_cached_filename = None
+    try:
+        adapter_cached_filename = get_file(
+            full_name_or_dir=full_name_or_dir,
+            file_path=ADAPTER_CONFIG_NAME,
+            repo_type="model",
+            store_dir=store_dir,
+            ref=ref,
+            token=token,
+            client=client,
+            **kwargs,
+        )
+    except FileNotFoundError:
+        pass
+    except OutpostHTTPException as e:
+        if e.code == 404:
+            logger.warn("Could not find PEFT config file. continuing...")
+        else:
+            raise e
+    return adapter_cached_filename