From bf36cbcdebf58716554ffccb59eab36e7d284f39 Mon Sep 17 00:00:00 2001 From: Tory McKeag Date: Thu, 30 Apr 2020 12:48:30 -0700 Subject: [PATCH] Added support for relative paths inside proxies in generated index.yaml --- .../IndexYamlAbsoluteUrlRewriter.java | 37 +++++++++---- .../orient/proxy/HelmProxyFacetImpl.java | 2 +- .../IndexYamlAbsoluteUrlRewriterTest.java | 54 +++++++++++++++++-- .../metadata/indexWithRelativeUrls.yaml | 14 +++++ 4 files changed, 90 insertions(+), 17 deletions(-) diff --git a/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriter.java b/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriter.java index f12c0476..a34a34e4 100644 --- a/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriter.java +++ b/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriter.java @@ -58,16 +58,18 @@ public class IndexYamlAbsoluteUrlRewriter private StorageFacet storageFacet; public TempBlob removeUrlsFromIndexYamlAndWriteToTempBlob(final TempBlob index, - final Repository repository) + final Repository repository, + final URI remoteUrl) { storageFacet = repository.facet(StorageFacet.class); - return new StreamCopier<>(outputStream -> updateUrls(index.get(), outputStream), + return new StreamCopier<>(outputStream -> updateUrls(index.get(), outputStream, remoteUrl), this::createTempBlob).read(); } private void updateUrls(final InputStream is, - final OutputStream os) + final OutputStream os, + final URI remoteUrl) { try (Reader reader = new InputStreamReader(is); Writer writer = new OutputStreamWriter(os)) { @@ -78,7 +80,7 @@ private void updateUrls(final InputStream is, if (event instanceof ScalarEvent) { ScalarEvent scalarEvent = (ScalarEvent) event; if (rewrite) { - event = maybeSetAbsoluteUrlAsRelative(scalarEvent); + event = maybeSetAbsoluteUrlAsRelative(scalarEvent, remoteUrl); } else if (URLS.equals(scalarEvent.getValue())) { rewrite = true; @@ -98,19 +100,32 @@ else if (event instanceof CollectionEndEvent) { } } - private Event maybeSetAbsoluteUrlAsRelative(ScalarEvent scalarEvent) { + private Event maybeSetAbsoluteUrlAsRelative(ScalarEvent scalarEvent, URI remoteUrl) { String oldUrl = scalarEvent.getValue(); try { URI uri = new URIBuilder(oldUrl).build(); if (uri.isAbsolute()) { - String fileName = uri.getPath(); - // Rewrite absolute paths to relative - if (!fileName.isEmpty()) { + + // There are a couple possibilities for algorithms to remove here. + // Frequently, this is will be relative to the remote. If so, relativize + // explicity against the remote such that any path elements in the remote are stripped + // out. + String fileName = null; + String remoteStr = remoteUrl.toString(); + if (oldUrl.startsWith(remoteStr) && oldUrl.length() > remoteStr.length()) { + fileName = oldUrl.substring(remoteStr.length(), oldUrl.length()); + } + else { + // Otherwise, we just strip out the host. + fileName = uri.getPath(); + } + + // Strip leading slash if it exists. + if (fileName.startsWith("/")) { fileName = Paths.get(fileName).getFileName().toString(); } - scalarEvent = new ScalarEvent(scalarEvent.getAnchor(), scalarEvent.getTag(), - scalarEvent.getImplicit(), fileName, scalarEvent.getStartMark(), - scalarEvent.getEndMark(), scalarEvent.getStyle()); + scalarEvent = new ScalarEvent(scalarEvent.getAnchor(), scalarEvent.getTag(), scalarEvent.getImplicit(), + fileName, scalarEvent.getStartMark(), scalarEvent.getEndMark(), scalarEvent.getStyle()); } } catch (URISyntaxException ex) { diff --git a/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/proxy/HelmProxyFacetImpl.java b/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/proxy/HelmProxyFacetImpl.java index f79401c9..d602ee4e 100644 --- a/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/proxy/HelmProxyFacetImpl.java +++ b/nexus-repository-helm/src/main/java/org/sonatype/repository/helm/internal/orient/proxy/HelmProxyFacetImpl.java @@ -121,7 +121,7 @@ protected Content store(final Context context, final Content content) throws IOE private Content putMetadata(final String path, final Content content, final AssetKind assetKind) throws IOException { StorageFacet storageFacet = facet(StorageFacet.class); try (TempBlob tempBlob = storageFacet.createTempBlob(content.openInputStream(), HASH_ALGORITHMS)) { - try (TempBlob newTempBlob = indexYamlAbsoluteUrlRewriter.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, getRepository()) ) { + try (TempBlob newTempBlob = indexYamlAbsoluteUrlRewriter.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, getRepository(), getRemoteUrl()) ) { return saveMetadataAsAsset(path, newTempBlob, content, assetKind); } } diff --git a/nexus-repository-helm/src/test/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriterTest.java b/nexus-repository-helm/src/test/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriterTest.java index ebc9a464..be922944 100644 --- a/nexus-repository-helm/src/test/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriterTest.java +++ b/nexus-repository-helm/src/test/java/org/sonatype/repository/helm/internal/orient/metadata/IndexYamlAbsoluteUrlRewriterTest.java @@ -16,6 +16,7 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URI; import org.sonatype.goodies.testsupport.TestSupport; import org.sonatype.nexus.blobstore.api.Blob; @@ -27,9 +28,15 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mock; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.events.CollectionEndEvent; +import org.yaml.snakeyaml.events.CollectionStartEvent; +import org.yaml.snakeyaml.events.Event; +import org.yaml.snakeyaml.events.ScalarEvent; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.either; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.startsWith; @@ -42,6 +49,9 @@ public class IndexYamlAbsoluteUrlRewriterTest extends TestSupport { + private URI remoteRepositoryUri = URI.create("https://path.to.repo.infrastructure/pathtorepo/"); + + private static final String INDEX_YAML = "index.yaml"; private static final String INDEX_YAML_NO_ABSOLUTE_URLS = "indexWithRelativeUrls.yaml"; @@ -74,7 +84,7 @@ public void setUp() throws Exception { @Test public void checkCustomUrls() throws Exception { setupIndexMock(INDEX_YAML_WITH_CUSTOM_URL); - TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository); + TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri); assertThat(newTempBlob.get(), is(instanceOf(InputStream.class))); checkThatAbsoluteUrlRemoved(newTempBlob.get()); } @@ -82,7 +92,7 @@ public void checkCustomUrls() throws Exception { @Test public void removeUrlsFromIndexYaml() throws Exception { setupIndexMock(INDEX_YAML); - TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository); + TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri); assertThat(newTempBlob.get(), is(instanceOf(InputStream.class))); checkThatAbsoluteUrlRemoved(newTempBlob.get()); } @@ -90,9 +100,42 @@ public void removeUrlsFromIndexYaml() throws Exception { @Test public void doNotModifyUrlsWhenAlreadyRelative() throws Exception { setupIndexMock(INDEX_YAML_NO_ABSOLUTE_URLS); - TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository); + TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri); assertThat(newTempBlob.get(), is(instanceOf(InputStream.class))); - checkThatAbsoluteUrlRemoved(newTempBlob.get()); + checkRelativeUrlsInSet(newTempBlob.get()); + } + + // The expected URL strings based on indexWithRelativeUrls.yaml + private static final String[] expectedConvertedURLs = + {"acs-engine-autoscaler-2.1.3.tgz", + "charts/acs-engine-autoscaler-2.1.2.tgz", + "acs-engine-autoscaler-2.1.1.tgz", + "pathwithinrepo/some-application-7.5.0.tgz"}; + + private void checkRelativeUrlsInSet(final InputStream is) throws Exception { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is))) { + Yaml yaml = new Yaml(); + int urlIndex = 0; + boolean inUrl = false; + for (Event yamlEvent : yaml.parse(reader)) { + if (yamlEvent instanceof ScalarEvent) { + ScalarEvent scalarEvent = (ScalarEvent) yamlEvent; + if (inUrl) { + assertThat(scalarEvent.getValue(), equalTo(expectedConvertedURLs[urlIndex])); + urlIndex++; + } + else if ("urls".equals(scalarEvent.getValue())) { + inUrl = true; + } + } + else if (yamlEvent instanceof CollectionStartEvent) { + // NOOP + } + else if (yamlEvent instanceof CollectionEndEvent) { + inUrl = false; + } + } + } } private void checkThatAbsoluteUrlRemoved(final InputStream is) throws Exception { @@ -119,7 +162,7 @@ private void checkThatAbsoluteUrlRemoved(final InputStream is) throws Exception @Test public void ensureNoExclamationMarksInYaml() throws Exception { setupIndexMock(INDEX_YAML); - TempBlob newTempBlob = underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository); + underTest.removeUrlsFromIndexYamlAndWriteToTempBlob(tempBlob, repository, remoteRepositoryUri); BufferedReader reader = new BufferedReader(new InputStreamReader(tempBlob.get())); String line; while ((line = reader.readLine()) != null) { @@ -134,6 +177,7 @@ private void setupIndexMock(final String indexYamlName) throws Exception { when(tempBlob.getBlob()).thenReturn(mock(Blob.class)); } + @SuppressWarnings("unchecked") private void setupRepositoryMock() { when(repository.facet(StorageFacet.class)).thenReturn(storageFacet); when(storageFacet.createTempBlob(any(InputStream.class), any(Iterable.class))).thenAnswer(args -> { diff --git a/nexus-repository-helm/src/test/resources/org/sonatype/repository/helm/internal/orient/metadata/indexWithRelativeUrls.yaml b/nexus-repository-helm/src/test/resources/org/sonatype/repository/helm/internal/orient/metadata/indexWithRelativeUrls.yaml index adeccdc0..17a69ba9 100644 --- a/nexus-repository-helm/src/test/resources/org/sonatype/repository/helm/internal/orient/metadata/indexWithRelativeUrls.yaml +++ b/nexus-repository-helm/src/test/resources/org/sonatype/repository/helm/internal/orient/metadata/indexWithRelativeUrls.yaml @@ -52,4 +52,18 @@ entries: urls: - acs-engine-autoscaler-2.1.1.tgz version: 2.1.1 + internal-power-app: + - apiVersion: v1 + appVersion: 7.5.0 + created: 2018-02-13T23:19:19.729122896Z + description: Dummy chart to prove parsing logic + digest: 39e66eb53c310529bd9dd19776f8ba662e063a4ebd51fc5ec9f2267e2e073e3e + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: tory.mckeag@ge.com + name: Tory McKeag + name: digital-energy-market + urls: + - https://path.to.repo.infrastructure/pathtorepo/pathwithinrepo/some-application-7.5.0.tgz + version: 0.0.1 generated: 2018-03-27T20:26:09.598062444Z