Merge pull request #101 from NERSC/symlink

Fix support for symlinks

src/sfapi_client/_async/paths.py

@@ -5,6 +5,7 @@
 from base64 import b64decode
 from contextlib import asynccontextmanager
 import tempfile
+import re
 
 from .._models import (
     DirectoryEntry as PathBase,
@@ -22,6 +23,9 @@ def _is_no_such(error: SfApiError):
     return "No such file or directory" in error.message
 
 
+SYMLINK_REGEX = r"^(\/.*) ->"
+
+
 class AsyncRemotePath(PathBase):
     """
     RemotePath is used to model a remote path, it takes inspiration from
@@ -133,7 +137,16 @@ async def is_file(self) -> bool:
         """
         :return: Returns True if path is a file, False otherwise.
         """
-        return not await self.is_dir()
+        if self.perms is None:
+            await self.update()
+
+        return self.perms[0] == "-"
+
+    async def is_symlink(self) -> bool:
+        if self.perms is None:
+            await self.update()
+
+        return self.perms[0] == "l"
 
     async def download(self, binary=False) -> IO[AnyStr]:
         """
@@ -188,15 +201,22 @@ def _to_remote_path(path, entry):
 
             return p
 
-        # Special case for listing file
+        # Special case for listing file or symlink
         if len(directory_listing_response.entries) == 1:
             entry = directory_listing_response.entries[0]
-            # The API can add an extra /
-            path = entry.name
-            if entry.name.startswith("//"):
-                path = path[1:]
-            filename = PurePosixPath(path).name
-            entry.name = filename
+            # symlink
+            if entry.perms[0] == "l":
+                if match := re.search(SYMLINK_REGEX, entry.name):
+                    entry.name = match.group(1)
+            # file
+            else:
+                # The API can add an extra /
+                path = entry.name
+                if entry.name.startswith("//"):
+                    path = path[1:]
+                filename = PurePosixPath(path).name
+                entry.name = filename
+
             paths.append(_to_remote_path(path, entry))
         else:
             for entry in directory_listing_response.entries:
@@ -209,6 +229,10 @@ def _to_remote_path(path, entry):
                 elif directory and entry.name == ".":
                     entry.name = PurePosixPath(path).name
                     return [_to_remote_path(path, entry)]
+                # Special case for symlink
+                elif entry.perms[0] == "l":
+                    if match := re.search(SYMLINK_REGEX, entry.name):
+                        entry.name = match.group(1)
 
                 paths.append(_to_remote_path(f"{path}/{entry.name}", entry))
 
@@ -260,9 +284,9 @@ async def upload(self, file: BytesIO) -> "AsyncRemotePath":
             if not _is_no_such(ex):
                 raise
 
-            # Check if the parent is a directory ( as in we are creating a new file ),
+            # Check if the parent is a directory or symlink ( as in we are creating a new file ),
             # if not re raise the original exception
-            if not await self.parent.is_dir():
+            if not (await self.parent.is_dir() or await self.parent.is_symlink()):
                 raise
             else:
                 upload_path = str(self._path)
@@ -301,9 +325,9 @@ async def open(self, mode: str) -> IO[AnyStr]:
             if not _is_no_such(ex):
                 raise
 
-            # Check if the parent is a directory ( as in we are creating a new file ),
+            # Check if the parent is a directory or symlink ( as in we are creating a new file ),
             # if not re raise the original exception
-            if not await self.parent.is_dir():
+            if not (await self.parent.is_dir() or await self.parent.is_symlink()):
                 raise
 
         valid_modes_chars = set("rwb")

src/sfapi_client/_sync/paths.py

@@ -5,6 +5,7 @@
 from base64 import b64decode
 from contextlib import contextmanager
 import tempfile
+import re
 
 from .._models import (
     DirectoryEntry as PathBase,
@@ -22,6 +23,9 @@ def _is_no_such(error: SfApiError):
     return "No such file or directory" in error.message
 
 
+SYMLINK_REGEX = r"^(\/.*) ->"
+
+
 class RemotePath(PathBase):
     """
     RemotePath is used to model a remote path, it takes inspiration from
@@ -133,7 +137,16 @@ def is_file(self) -> bool:
         """
         :return: Returns True if path is a file, False otherwise.
         """
-        return not self.is_dir()
+        if self.perms is None:
+            self.update()
+
+        return self.perms[0] == "-"
+
+    def is_symlink(self) -> bool:
+        if self.perms is None:
+            self.update()
+
+        return self.perms[0] == "l"
 
     def download(self, binary=False) -> IO[AnyStr]:
         """
@@ -188,15 +201,22 @@ def _to_remote_path(path, entry):
 
             return p
 
-        # Special case for listing file
+        # Special case for listing file or symlink
         if len(directory_listing_response.entries) == 1:
             entry = directory_listing_response.entries[0]
-            # The API can add an extra /
-            path = entry.name
-            if entry.name.startswith("//"):
-                path = path[1:]
-            filename = PurePosixPath(path).name
-            entry.name = filename
+            # symlink
+            if entry.perms[0] == "l":
+                if match := re.search(SYMLINK_REGEX, entry.name):
+                    entry.name = match.group(1)
+            # file
+            else:
+                # The API can add an extra /
+                path = entry.name
+                if entry.name.startswith("//"):
+                    path = path[1:]
+                filename = PurePosixPath(path).name
+                entry.name = filename
+
             paths.append(_to_remote_path(path, entry))
         else:
             for entry in directory_listing_response.entries:
@@ -209,6 +229,10 @@ def _to_remote_path(path, entry):
                 elif directory and entry.name == ".":
                     entry.name = PurePosixPath(path).name
                     return [_to_remote_path(path, entry)]
+                # Special case for symlink
+                elif entry.perms[0] == "l":
+                    if match := re.search(SYMLINK_REGEX, entry.name):
+                        entry.name = match.group(1)
 
                 paths.append(_to_remote_path(f"{path}/{entry.name}", entry))
 
@@ -260,9 +284,9 @@ def upload(self, file: BytesIO) -> "RemotePath":
             if not _is_no_such(ex):
                 raise
 
-            # Check if the parent is a directory ( as in we are creating a new file ),
+            # Check if the parent is a directory or symlink ( as in we are creating a new file ),
             # if not re raise the original exception
-            if not self.parent.is_dir():
+            if not (self.parent.is_dir() or self.parent.is_symlink()):
                 raise
             else:
                 upload_path = str(self._path)
@@ -301,9 +325,9 @@ def open(self, mode: str) -> IO[AnyStr]:
             if not _is_no_such(ex):
                 raise
 
-            # Check if the parent is a directory ( as in we are creating a new file ),
+            # Check if the parent is a directory or symlink ( as in we are creating a new file ),
             # if not re raise the original exception
-            if not self.parent.is_dir():
+            if not (self.parent.is_dir() or self.parent.is_symlink()):
                 raise
 
         valid_modes_chars = set("rwb")

tests/test_paths_async.py

@@ -295,3 +295,54 @@ async def test_file_open_write_new(
         # Now check that the content has changed
         async with remote_file.open("r") as fp:
             assert file_contents in fp.read()
+
+
+@pytest.mark.asyncio
+async def test_ls_symlink(
+    async_authenticated_client, test_machine, test_job_path, test_tmp_dir
+):
+    async with async_authenticated_client as client:
+        test_job = Path(test_job_path)
+        test_job_directory = test_job.parent
+        machine = await client.compute(test_machine)
+        link_path = Path(test_tmp_dir) / "link"
+        try:
+            await machine.run(f"ln -s {test_job_directory} {link_path}")
+            [path] = await machine.ls(link_path, directory=True)
+            assert str(path) == str(link_path)
+            assert await path.is_symlink()
+        finally:
+            await machine.run(f"rm -f {link_path}")
+
+
+@pytest.mark.asyncio
+async def test_ls_symlink_upload(
+    async_authenticated_client, test_machine, test_job_path, test_tmp_dir
+):
+    async with async_authenticated_client as client:
+        test_job = Path(test_job_path)
+        test_job_directory = test_job.parent
+        machine = await client.compute(test_machine)
+        link_path = Path(test_tmp_dir) / "link"
+        try:
+            await machine.run(f"ln -s {test_job_directory} {link_path}")
+            [path] = await machine.ls(link_path, directory=True)
+            assert str(path) == str(link_path)
+            assert await path.is_symlink()
+
+            # Create empty file
+            random_name = "".join(
+                random.choices(string.ascii_lowercase + string.digits, k=10)
+            )
+            remote_file = path / f"{random_name}.txt"
+
+            # Now write to the file
+            file_contents = "symlink"
+            async with remote_file.open("wb") as fp:
+                fp.write(file_contents.encode())
+
+            # Now check that the content has changed
+            async with remote_file.open("r") as fp:
+                assert file_contents in fp.read()
+        finally:
+            await machine.run(f"rm -rf {link_path}")