Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: More explicit Provenance Handling #9421

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion helper-cli/src/main/kotlin/utils/Extensions.kt
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,9 @@ internal fun OrtResult.getLicenseFindingsById(
packageConfigurationProvider.getPackageConfigurations(id, provenance).flatMap { it.licenseFindingCurations }
}

getScanResultsForId(id).filter { it.provenance is KnownProvenance }.map {
getScanResultsForId(id).filter {
it.provenance is RepositoryProvenance || it.provenance is ArtifactProvenance
}.map {
// If a VCS path curation has been applied after the scanning stage, it is possible to apply that
// curation without re-scanning in case the new VCS path is a subdirectory of the scanned VCS path.
// So, filter by VCS path to enable the user to see the effect on the detected license with a shorter
Expand Down
2 changes: 1 addition & 1 deletion model/src/main/kotlin/ScannerRun.kt
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ data class ScannerRun(

init {
scanResults.forEach { scanResult ->
require(scanResult.provenance is KnownProvenance) {
require(scanResult.provenance is RepositoryProvenance || scanResult.provenance is ArtifactProvenance) {
"Found a scan result with an unknown provenance, which is not allowed."
}

Expand Down
1 change: 1 addition & 0 deletions model/src/main/kotlin/config/PackageConfiguration.kt
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
is UnknownProvenance -> false
is ArtifactProvenance -> sourceArtifactUrl != null && sourceArtifactUrl == provenance.sourceArtifact.url
is RepositoryProvenance -> vcs != null && vcs.matches(provenance)
else -> false

Check warning on line 80 in model/src/main/kotlin/config/PackageConfiguration.kt

View check run for this annotation

Codecov / codecov/patch

model/src/main/kotlin/config/PackageConfiguration.kt#L80

Added line #L80 was not covered by tests
}
}
}
Expand Down
6 changes: 4 additions & 2 deletions model/src/main/kotlin/licenses/LicenseInfoResolver.kt
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,26 @@

package org.ossreviewtoolkit.model.licenses

import org.ossreviewtoolkit.model.ArtifactProvenance
import java.util.concurrent.ConcurrentHashMap

import org.ossreviewtoolkit.model.CopyrightFinding
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.KnownProvenance
import org.ossreviewtoolkit.model.LicenseSource
import org.ossreviewtoolkit.model.Provenance
import org.ossreviewtoolkit.model.RepositoryProvenance
import org.ossreviewtoolkit.model.TextLocation
import org.ossreviewtoolkit.model.UnknownProvenance
import org.ossreviewtoolkit.model.config.CopyrightGarbage
import org.ossreviewtoolkit.model.config.LicenseFilePatterns
import org.ossreviewtoolkit.model.config.PathExclude
import org.ossreviewtoolkit.model.utils.FileArchiver
import org.ossreviewtoolkit.model.utils.FindingCurationMatcher
import org.ossreviewtoolkit.model.utils.FindingsMatcher
import org.ossreviewtoolkit.model.utils.PathLicenseMatcher
import org.ossreviewtoolkit.model.utils.prependedPath
import org.ossreviewtoolkit.utils.ort.createOrtTempDir
import org.ossreviewtoolkit.utils.spdx.SpdxSingleLicenseExpression

Check warning

Code scanning / detekt

Reports files that do not follow ORT's order for imports Warning

Imports in file '/home/runner/work/ort/ort/model/src/main/kotlin/licenses/LicenseInfoResolver.kt' are not sorted alphabetically or single blank lines are missing between different top-level packages

class LicenseInfoResolver(
private val provider: LicenseInfoProvider,
Expand Down Expand Up @@ -264,7 +264,9 @@

when (provenance) {
is UnknownProvenance -> return@forEach
is KnownProvenance -> if (!archiver.unarchive(archiveDir, provenance)) return@forEach
is RepositoryProvenance -> if (!archiver.unarchive(archiveDir, provenance)) return@forEach
is ArtifactProvenance -> if (!archiver.unarchive(archiveDir, provenance)) return@forEach
else -> return@forEach

Check warning on line 269 in model/src/main/kotlin/licenses/LicenseInfoResolver.kt

View check run for this annotation

Codecov / codecov/patch

model/src/main/kotlin/licenses/LicenseInfoResolver.kt#L269

Added line #L269 was not covered by tests
}

val directory = (provenance as? RepositoryProvenance)?.vcsInfo?.path.orEmpty()
Expand Down
1 change: 1 addition & 0 deletions model/src/main/kotlin/utils/FileProvenanceFileStorage.kt
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
val key = when (this) {
is ArtifactProvenance -> "${sourceArtifact.url}${sourceArtifact.hash.value}"
is RepositoryProvenance -> "${vcsInfo.type}${vcsInfo.url}$resolvedRevision"
else -> ""

Check warning on line 84 in model/src/main/kotlin/utils/FileProvenanceFileStorage.kt

View check run for this annotation

Codecov / codecov/patch

model/src/main/kotlin/utils/FileProvenanceFileStorage.kt#L84

Added line #L84 was not covered by tests
}

return HashAlgorithm.SHA1.calculate(key.toByteArray())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,4 +120,5 @@
is ArtifactProvenance -> "source-artifact|${sourceArtifact.url}|${sourceArtifact.hash}"
// The trailing "|" is kept for backward compatibility because there used to be an additional parameter.
is RepositoryProvenance -> "vcs|${vcsInfo.type}|${vcsInfo.url}|$resolvedRevision|"
else -> ""

Check warning on line 123 in model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt

View check run for this annotation

Codecov / codecov/patch

model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt#L123

Added line #L123 was not covered by tests
}
2 changes: 2 additions & 0 deletions model/src/main/kotlin/utils/PurlExtensions.kt
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@
}

is UnknownProvenance -> PurlExtras()

else -> PurlExtras()

Check warning on line 110 in model/src/main/kotlin/utils/PurlExtensions.kt

View check run for this annotation

Codecov / codecov/patch

model/src/main/kotlin/utils/PurlExtensions.kt#L110

Added line #L110 was not covered by tests
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,7 @@
}

private fun EM.provenanceLink(provenance: Provenance?) {
if (provenance is ArtifactProvenance) {

Check notice on line 690 in plugins/reporters/static-html/src/main/kotlin/StaticHtmlReporter.kt

View workflow job for this annotation

GitHub Actions / qodana-scan

Cascade 'if' can be replaced with 'when'

Cascade 'if' should be replaced with 'when'
+" (from "
a(href = provenance.sourceArtifact.url) { +"artifact" }
+")"
Expand All @@ -695,6 +695,8 @@
+" (from "
a(href = provenance.vcsInfo.url) { +"VCS" }
+")"
} else {
+""
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import io.kotest.core.listeners.TestListener
import io.kotest.core.spec.style.WordSpec
import io.kotest.matchers.shouldBe

import org.ossreviewtoolkit.model.KnownProvenance
import org.ossreviewtoolkit.model.RepositoryProvenance
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
Expand Down Expand Up @@ -78,6 +77,6 @@ private fun createRepositoryProvenance(
) = RepositoryProvenance(vcsInfo, resolvedRevision)

private fun createNestedProvenance(
root: KnownProvenance,
root: RepositoryProvenance,
subRepositories: Map<String, RepositoryProvenance> = emptyMap()
) = NestedProvenance(root, subRepositories)
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
return when (provenance) {
is ArtifactProvenance -> NestedProvenance(root = provenance, subRepositories = emptyMap())
is RepositoryProvenance -> resolveNestedRepository(provenance)
else -> NestedProvenance(root = provenance, subRepositories = emptyMap())

Check warning on line 53 in scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt

View check run for this annotation

Codecov / codecov/patch

scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt#L53

Added line #L53 was not covered by tests
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB

requireEmptyVcsPath(provenance)

if (provenance !is KnownProvenance) {
if (provenance !is RepositoryProvenance && provenance !is ArtifactProvenance) {
throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.")
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ class ProvenanceBasedPostgresStorage(

requireEmptyVcsPath(provenance)

if (provenance !is KnownProvenance) {
if (provenance !is RepositoryProvenance && provenance !is ArtifactProvenance) {
throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.")
}

Expand Down
4 changes: 3 additions & 1 deletion utils/test/src/main/kotlin/Utils.kt
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,9 @@ fun readOrtResult(file: File) = file.mapper().readValue<OrtResult>(patchExpected
fun scannerRunOf(vararg pkgScanResults: Pair<Identifier, List<ScanResult>>): ScannerRun {
val pkgScanResultsWithKnownProvenance = pkgScanResults.associate { (id, scanResultsForId) ->
id to scanResultsForId.map { scanResult ->
scanResult.takeIf { scanResult.provenance is KnownProvenance } ?: scanResult.copy(
scanResult.takeIf {
scanResult.provenance is RepositoryProvenance || scanResult.provenance is ArtifactProvenance
} ?: scanResult.copy(
provenance = ArtifactProvenance(
sourceArtifact = RemoteArtifact(
url = id.toPurl(),
Expand Down
Loading