Added support for relative paths inside proxies in generated index.yaml

sonatype-nexus-community · Apr 30, 2020 · bf36cbc · bf36cbc
1 parent 37a6370
commit bf36cbc
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 17 deletions.
diff --git a/...a/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriter.java b/...a/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriter.java
@@ -58,16 +58,18 @@ public class IndexYamlAbsoluteUrlRewriter
   private StorageFacet storageFacet;
 
   public TempBlob removeUrlsFromIndexYamlAndWriteToTempBlob(final TempBlob index,
-                                                            final Repository repository)
+                                                            final Repository repository, 
+                                                            final URI remoteUrl)
   {
     storageFacet = repository.facet(StorageFacet.class);
 
-    return new StreamCopier<>(outputStream -> updateUrls(index.get(), outputStream),
+    return new StreamCopier<>(outputStream -> updateUrls(index.get(), outputStream, remoteUrl),
         this::createTempBlob).read();
   }
 
   private void updateUrls(final InputStream is,
-                          final OutputStream os)
+                          final OutputStream os,
+                          final URI remoteUrl)
   {
     try (Reader reader = new InputStreamReader(is);
          Writer writer = new OutputStreamWriter(os)) {
@@ -78,7 +80,7 @@ private void updateUrls(final InputStream is,
         if (event instanceof ScalarEvent) {
           ScalarEvent scalarEvent = (ScalarEvent) event;
           if (rewrite) {
-            event = maybeSetAbsoluteUrlAsRelative(scalarEvent);
+            event = maybeSetAbsoluteUrlAsRelative(scalarEvent, remoteUrl);
           }
           else if (URLS.equals(scalarEvent.getValue())) {
             rewrite = true;
@@ -98,19 +100,32 @@ else if (event instanceof CollectionEndEvent) {
     }
   }
 
-  private Event maybeSetAbsoluteUrlAsRelative(ScalarEvent scalarEvent) {
+  private Event maybeSetAbsoluteUrlAsRelative(ScalarEvent scalarEvent, URI remoteUrl) {
     String oldUrl = scalarEvent.getValue();
     try {
       URI uri = new URIBuilder(oldUrl).build();
       if (uri.isAbsolute()) {
-        String fileName = uri.getPath();
-        // Rewrite absolute paths to relative
-        if (!fileName.isEmpty()) {
+
+        // There are a couple possibilities for algorithms to remove here.
+        // Frequently, this is will be relative to the remote. If so, relativize
+        // explicity against the remote such that any path elements in the remote are stripped
+        // out.
+        String fileName = null;
+        String remoteStr = remoteUrl.toString();
+        if (oldUrl.startsWith(remoteStr) && oldUrl.length() > remoteStr.length()) {
+          fileName = oldUrl.substring(remoteStr.length(), oldUrl.length());
+        }
+        else {
+          // Otherwise, we just strip out the host.
+          fileName = uri.getPath();
+        }
+
+        // Strip leading slash if it exists.
+        if (fileName.startsWith("/")) {
           fileName = Paths.get(fileName).getFileName().toString();
         }
-        scalarEvent = new ScalarEvent(scalarEvent.getAnchor(), scalarEvent.getTag(),
-            scalarEvent.getImplicit(), fileName, scalarEvent.getStartMark(),
-            scalarEvent.getEndMark(), scalarEvent.getStyle());
+        scalarEvent = new ScalarEvent(scalarEvent.getAnchor(), scalarEvent.getTag(), scalarEvent.getImplicit(),
+            fileName, scalarEvent.getStartMark(), scalarEvent.getEndMark(), scalarEvent.getStyle());
       }
     }
     catch (URISyntaxException ex) {

diff --git a/.../src/main/java/org/sonatype/repository/helm/internal/orient/proxy/HelmProxyFacetImpl.java b/.../src/main/java/org/sonatype/repository/helm/internal/orient/proxy/HelmProxyFacetImpl.java
@@ -121,7 +121,7 @@ protected Content store(final Context context, final Content content) throws IOE
   private Content putMetadata(final String path, final Content content, final AssetKind assetKind) throws IOException {
     StorageFacet storageFacet = facet(StorageFacet.class);
     try (TempBlob tempBlob = storageFacet.createTempBlob(content.openInputStream(), HASH_ALGORITHMS)) {
-      try (TempBlob newTempBlob = indexYamlAbsoluteUrlRewriter.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, getRepository()) ) {
+      try (TempBlob newTempBlob = indexYamlAbsoluteUrlRewriter.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, getRepository(), getRemoteUrl()) ) {
         return saveMetadataAsAsset(path, newTempBlob, content, assetKind);
       }
     }

diff --git a/...g/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriterTest.java b/...g/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriterTest.java
@@ -16,6 +16,7 @@
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.net.URI;
 
 import org.sonatype.goodies.testsupport.TestSupport;
 import org.sonatype.nexus.blobstore.api.Blob;
@@ -27,9 +28,15 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mock;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.events.CollectionEndEvent;
+import org.yaml.snakeyaml.events.CollectionStartEvent;
+import org.yaml.snakeyaml.events.Event;
+import org.yaml.snakeyaml.events.ScalarEvent;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.either;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.startsWith;
@@ -42,6 +49,9 @@
 public class IndexYamlAbsoluteUrlRewriterTest
     extends TestSupport
 {
+  private URI remoteRepositoryUri = URI.create("https://path.to.repo.infrastructure/pathtorepo/");
+
+
   private static final String INDEX_YAML = "index.yaml";
 
   private static final String INDEX_YAML_NO_ABSOLUTE_URLS = "indexWithRelativeUrls.yaml";
@@ -74,25 +84,58 @@ public void setUp() throws Exception {
   @Test
   public void checkCustomUrls() throws Exception {
     setupIndexMock(INDEX_YAML_WITH_CUSTOM_URL);
-    TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository);
+    TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri);
     assertThat(newTempBlob.get(), is(instanceOf(InputStream.class)));
     checkThatAbsoluteUrlRemoved(newTempBlob.get());
   }
 
   @Test
   public void removeUrlsFromIndexYaml() throws Exception {
     setupIndexMock(INDEX_YAML);
-    TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository);
+    TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri);
     assertThat(newTempBlob.get(), is(instanceOf(InputStream.class)));
     checkThatAbsoluteUrlRemoved(newTempBlob.get());
   }
 
   @Test
   public void doNotModifyUrlsWhenAlreadyRelative() throws Exception {
     setupIndexMock(INDEX_YAML_NO_ABSOLUTE_URLS);
-    TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository);
+    TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri);
     assertThat(newTempBlob.get(), is(instanceOf(InputStream.class)));
-    checkThatAbsoluteUrlRemoved(newTempBlob.get());
+    checkRelativeUrlsInSet(newTempBlob.get());
+  }
+
+  // The expected URL strings based on indexWithRelativeUrls.yaml
+  private static final String[] expectedConvertedURLs =
+      {"acs-engine-autoscaler-2.1.3.tgz", 
+          "charts/acs-engine-autoscaler-2.1.2.tgz", 
+          "acs-engine-autoscaler-2.1.1.tgz",
+          "pathwithinrepo/some-application-7.5.0.tgz"};
+
+  private void checkRelativeUrlsInSet(final InputStream is) throws Exception {
+    try (BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+      Yaml yaml = new Yaml();
+      int urlIndex = 0;
+      boolean inUrl = false;
+      for (Event yamlEvent : yaml.parse(reader)) {
+        if (yamlEvent instanceof ScalarEvent) {
+          ScalarEvent scalarEvent = (ScalarEvent) yamlEvent;
+          if (inUrl) {
+            assertThat(scalarEvent.getValue(), equalTo(expectedConvertedURLs[urlIndex]));
+            urlIndex++;
+          }
+          else if ("urls".equals(scalarEvent.getValue())) {
+            inUrl = true;
+          }
+        }
+        else if (yamlEvent instanceof CollectionStartEvent) {
+          // NOOP
+        }
+        else if (yamlEvent instanceof CollectionEndEvent) {
+          inUrl = false;
+        }
+      }
+    }
   }
 
   private void checkThatAbsoluteUrlRemoved(final InputStream is) throws Exception {
@@ -119,7 +162,7 @@ private void checkThatAbsoluteUrlRemoved(final InputStream is) throws Exception
   @Test
   public void ensureNoExclamationMarksInYaml() throws Exception {
     setupIndexMock(INDEX_YAML);
-    TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository);
+    underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri);
     BufferedReader reader = new BufferedReader(new InputStreamReader(tempBlob.get()));
     String line;
     while ((line = reader.readLine()) != null) {
@@ -134,6 +177,7 @@ private void setupIndexMock(final String indexYamlName) throws Exception {
     when(tempBlob.getBlob()).thenReturn(mock(Blob.class));
   }
 
+  @SuppressWarnings("unchecked")
   private void setupRepositoryMock() {
     when(repository.facet(StorageFacet.class)).thenReturn(storageFacet);
     when(storageFacet.createTempBlob(any(InputStream.class), any(Iterable.class))).thenAnswer(args -> {

diff --git a/...esources/org/sonatype/repository/helm/internal/orient/metadata/indexWithRelativeUrls.yaml b/...esources/org/sonatype/repository/helm/internal/orient/metadata/indexWithRelativeUrls.yaml
@@ -52,4 +52,18 @@ entries:
     urls:
     - acs-engine-autoscaler-2.1.1.tgz
     version: 2.1.1
+  internal-power-app:
+  - apiVersion: v1
+    appVersion: 7.5.0
+    created: 2018-02-13T23:19:19.729122896Z
+    description: Dummy chart to prove parsing logic
+    digest: 39e66eb53c310529bd9dd19776f8ba662e063a4ebd51fc5ec9f2267e2e073e3e
+    icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png
+    maintainers:
+    - email: [email protected]
+      name: Tory McKeag
+    name: digital-energy-market
+    urls:
+    - https://path.to.repo.infrastructure/pathtorepo/pathwithinrepo/some-application-7.5.0.tgz
+    version: 0.0.1   
 generated: 2018-03-27T20:26:09.598062444Z