Skip to content

Commit

Permalink
refactor(vcs): Enable to provide VCS-specific configuration options
Browse files Browse the repository at this point in the history
While VCS implementations are already plugins, they are not yet
configurable. VCS implementations require common configurations
(e.g., `revision`, `recursive`) and should support also
VCS-specific configurations if they are consumed via their API.
This allows to add functionality to individual VCS implementations
without the need to implement them for all of them.

Fixes oss-review-toolkit#8556.

Signed-off-by: Wolfgang Klenk <[email protected]>
  • Loading branch information
wkl3nk committed Nov 13, 2024
1 parent f8a0c39 commit baca8ab
Show file tree
Hide file tree
Showing 21 changed files with 427 additions and 81 deletions.
6 changes: 5 additions & 1 deletion cli/src/funTest/kotlin/AnalyzerFunTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit
import org.ossreviewtoolkit.analyzer.Analyzer
import org.ossreviewtoolkit.analyzer.PackageManagerFactory
import org.ossreviewtoolkit.analyzer.analyze
import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
Expand All @@ -52,7 +53,10 @@ class AnalyzerFunTest : WordSpec({
revision = "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa"
)
)
val outputDir = tempdir().also { GitRepo().download(pkg, it) }
val outputDir = tempdir().also {
GitRepo.Factory().create(VersionControlSystemConfiguration())
.download(pkg, it)
}

val result = analyze(outputDir, packageManagers = emptySet()).toYaml()

Expand Down
8 changes: 6 additions & 2 deletions downloader/src/main/kotlin/Downloader.kt
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,9 @@ class Downloader(private val config: DownloaderConfiguration) {
var applicableVcs: VersionControlSystem? = null

if (pkg.vcsProcessed.type != VcsType.UNKNOWN) {
applicableVcs = VersionControlSystem.forType(pkg.vcsProcessed.type)
applicableVcs = VersionControlSystem.forType(
pkg.vcsProcessed.type, config.getCaseInsensitiveVersionControlSystems()
)
logger.info {
applicableVcs?.let {
"Detected VCS type '${it.type}' from type name '${pkg.vcsProcessed.type}'."
Expand All @@ -241,7 +243,9 @@ class Downloader(private val config: DownloaderConfiguration) {
}

if (applicableVcs == null) {
applicableVcs = VersionControlSystem.forUrl(pkg.vcsProcessed.url)
applicableVcs = VersionControlSystem.forUrl(
pkg.vcsProcessed.url, config.getCaseInsensitiveVersionControlSystems()
)
logger.info {
applicableVcs?.let {
"Detected VCS type '${it.type}' from URL ${pkg.vcsProcessed.url}."
Expand Down
93 changes: 53 additions & 40 deletions downloader/src/main/kotlin/VersionControlSystem.kt
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@ import java.io.IOException

import org.apache.logging.log4j.kotlin.logger

import org.ossreviewtoolkit.downloader.VersionControlSystemFactory.Companion.ALL
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
import org.ossreviewtoolkit.model.config.LicenseFilePatterns
import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
import org.ossreviewtoolkit.model.orEmpty
import org.ossreviewtoolkit.utils.common.CommandLineTool
import org.ossreviewtoolkit.utils.common.Plugin
import org.ossreviewtoolkit.utils.common.collectMessages
import org.ossreviewtoolkit.utils.common.uppercaseFirstChar
import org.ossreviewtoolkit.utils.ort.ORT_REPO_CONFIG_FILENAME
Expand All @@ -44,22 +45,23 @@ abstract class VersionControlSystem(
* the version control system is available.
*/
private val commandLineTool: CommandLineTool? = null
) : Plugin {
) {
companion object {
/**
* All [version control systems][VersionControlSystem] available in the classpath, sorted by their priority.
*/
val ALL by lazy {
Plugin.getAll<VersionControlSystem>().toList().sortedByDescending { (_, vcs) -> vcs.priority }.toMap()
}

/**
* Return the applicable VCS for the given [vcsType], or null if none is applicable.
*/
fun forType(vcsType: VcsType) =
ALL.values.find {
it.isAvailable() && it.isApplicableType(vcsType)
fun forType(
vcsType: VcsType,
versionControlSystemsConfiguration: Map<String, VersionControlSystemConfiguration> = emptyMap()
) = ALL.values.filter { vcsFactory -> vcsFactory.type == vcsType.toString() }
.map { vcsFactory ->
// If there is a configuration for the VCS type, use it, otherwise create
// the VCS with an empty configuration.
versionControlSystemsConfiguration[vcsFactory.type]?.let { vcsConfig ->
vcsFactory.create(options = vcsConfig.options, secrets = emptyMap())
} ?: vcsFactory.create(options = emptyMap(), secrets = emptyMap())
}
.firstOrNull { vcs -> vcs.isAvailable() }

/**
* A map to cache the [VersionControlSystem], if any, for previously queried URLs. This helps to speed up
Expand All @@ -72,8 +74,10 @@ abstract class VersionControlSystem(
* Return the applicable VCS for the given [vcsUrl], or null if none is applicable.
*/
@Synchronized
fun forUrl(vcsUrl: String) =
// Do not use getOrPut() here as it cannot handle null values, also see
fun forUrl(
vcsUrl: String,
versionControlSystemsConfiguration: Map<String, VersionControlSystemConfiguration> = emptyMap()
) = // Do not use getOrPut() here as it cannot handle null values, also see
// https://youtrack.jetbrains.com/issue/KT-21392.
if (vcsUrl in urlToVcsMap) {
urlToVcsMap[vcsUrl]
Expand All @@ -82,12 +86,18 @@ abstract class VersionControlSystem(
when (val type = VcsHost.parseUrl(vcsUrl).type) {
VcsType.UNKNOWN -> {
// ...then eventually try to determine the type also dynamically.
ALL.values.find {
it.isAvailable() && it.isApplicableUrl(vcsUrl)
}
ALL.values
.map { vcsFactory ->
// If there is a configuration for the VCS type, use it, otherwise create
// the VCS with an empty configuration.
versionControlSystemsConfiguration[vcsFactory.type]
?.let { vcsConfig ->
vcsFactory.create(options = vcsConfig.options, secrets = emptyMap())
} ?: vcsFactory.create(options = emptyMap(), secrets = emptyMap())
}.firstOrNull { vcs -> vcs.isAvailable() && vcs.isApplicableUrl(vcsUrl) }
}

else -> forType(type)
else -> forType(type, versionControlSystemsConfiguration)
}.also {
urlToVcsMap[vcsUrl] = it
}
Expand All @@ -109,28 +119,31 @@ abstract class VersionControlSystem(
return if (absoluteVcsDirectory in dirToVcsMap) {
dirToVcsMap[absoluteVcsDirectory]
} else {
ALL.values.asSequence().mapNotNull {
if (it is CommandLineTool && !it.isInPath()) {
null
} else {
it.getWorkingTree(absoluteVcsDirectory)
}
}.find {
try {
it.isValid()
} catch (e: IOException) {
e.showStackTrace()

logger.debug {
"Exception while validating ${it.vcsType} working tree, treating it as non-applicable: " +
e.collectMessages()
ALL.values.asSequence()
.map { vcsFactory -> vcsFactory.create(options = emptyMap(), secrets = emptyMap()) }
.mapNotNull {
if (it is CommandLineTool && !it.isInPath()) {
null
} else {
it.getWorkingTree(absoluteVcsDirectory)
}

false
}.find {
try {
it.isValid()
} catch (e: IOException) {
e.showStackTrace()

logger.debug {
"Exception while validating ${it.vcsType} working tree, " +
"treating it as non-applicable: " +
e.collectMessages()
}

false
}
}.also {
dirToVcsMap[absoluteVcsDirectory] = it
}
}.also {
dirToVcsMap[absoluteVcsDirectory] = it
}
}
}

Expand Down Expand Up @@ -165,9 +178,9 @@ abstract class VersionControlSystem(
}

/**
* The priority in which this VCS should be probed. A higher value means a higher priority.
* The type of CVS that is supported by this VCS plugin.
*/
protected open val priority: Int = 0
abstract val type: String

/**
* A list of symbolic names that point to the latest revision.
Expand Down
49 changes: 49 additions & 0 deletions downloader/src/main/kotlin/VersionControlSystemFactory.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.downloader

import org.ossreviewtoolkit.utils.common.Plugin
import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory

/**
* An abstract class to be implemented by factories for [version contral systems][VersionControlSystem].
* The constructor parameter [type] denotes which VCS type is supported by this plugin.
* The constructor parameter [priority] is used to determine the order in which the VCS plugins are used.
*/
abstract class VersionControlSystemFactory<CONFIG>(override val type: String, val priority: Int) :
TypedConfigurablePluginFactory<CONFIG, VersionControlSystem> {
companion object {
/**
* All [version control system factories][VersionControlSystemFactory] available in the classpath,
* associated by their names, sorted by priority.
*/
val ALL by lazy {
Plugin.getAll<VersionControlSystemFactory<*>>()
.toList()
.sortedByDescending { (_, vcsFactory) -> vcsFactory.priority }
.toMap()
}
}
}

/**
* A base class for specific version control system configurations.
*/
open class VersionControlSystemConfiguration
7 changes: 5 additions & 2 deletions downloader/src/test/kotlin/VersionControlSystemTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git
import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitConfiguration
import org.ossreviewtoolkit.utils.common.CommandLineTool

class VersionControlSystemTest : WordSpec({
Expand Down Expand Up @@ -87,7 +88,8 @@ class VersionControlSystemTest : WordSpec({

every { workingTree.guessRevisionName(any(), any()) } returns "v1.6.0"

Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
Git.Factory().create(GitConfiguration())
.getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
"v1.6.0"
)
}
Expand All @@ -110,7 +112,8 @@ class VersionControlSystemTest : WordSpec({
every { workingTree.listRemoteBranches() } returns listOf("main")
every { workingTree.listRemoteTags() } returns emptyList()

Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
Git.Factory().create(GitConfiguration())
.getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
"master",
"main"
)
Expand Down
22 changes: 22 additions & 0 deletions integrations/schemas/ort-configuration-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,28 @@
"items": {
"$ref": "#/definitions/SourceCodeOrigins"
}
},
"versionControlSystems": {
"type": "object",
"properties": {
"Git": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"submoduleHistoryDepth": {
"type": "integer",
"minimum": 1
},
"updateNestedSubmodules": {
"type": "boolean"
}
}
}
}
}
}
}
}
},
Expand Down
28 changes: 27 additions & 1 deletion model/src/main/kotlin/config/DownloaderConfiguration.kt
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,35 @@ data class DownloaderConfiguration(
* Configuration of the considered source code origins and their priority order. This must not be empty and not
* contain any duplicates.
*/
val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT)
val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT),

/**
* Version control system specific configurations. The key needs to match VCS type,
* e.g. "Git" for the Git version control system.
*/
val versionControlSystems: Map<String, VersionControlSystemConfiguration> = emptyMap()
) {
/**
* A copy of [versionControlSystems] with case-insensitive keys.
*/
private val versionControlSystemsCaseInsensitive: Map<String, VersionControlSystemConfiguration> =
versionControlSystems.toSortedMap(String.CASE_INSENSITIVE_ORDER)

init {
sourceCodeOrigins.requireNotEmptyNoDuplicates()

val duplicateVersionControlSystems =
versionControlSystems.keys - versionControlSystemsCaseInsensitive.keys.toSet()

require(duplicateVersionControlSystems.isEmpty()) {
"The following version control systems have duplicate configuration: " +
"${duplicateVersionControlSystems.joinToString()}."
}
}

/**
* Get a [VersionControlSystemConfiguration] from [versionControlSystems].
* The difference to accessing the map directly is that VCS type can be case-insensitive.
*/
fun getCaseInsensitiveVersionControlSystems() = versionControlSystemsCaseInsensitive
}
35 changes: 35 additions & 0 deletions model/src/main/kotlin/config/VersionControlSystemConfiguration.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.model.config

import com.fasterxml.jackson.annotation.JsonInclude

import org.ossreviewtoolkit.utils.common.Options

/**
* The configuration for a Version Control System (VCS).
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
data class VersionControlSystemConfiguration(
/**
* Custom configuration options. See the documentation of the respective class for available options.
*/
val options: Options = emptyMap()
)
11 changes: 11 additions & 0 deletions model/src/main/resources/reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,17 @@ ort:

sourceCodeOrigins: [VCS, ARTIFACT]

# Optional VCS-specific configuration options.
versionControlSystems:
Git:
options:
# Depth of the commit history to fetch when updating submodules
submoduleHistoryDepth: 10

# A flag to control whether nested submodules should be updated (true), or if only the submodules
# on the first layer should be considered (false).
updateNestedSubmodules: true

scanner:
skipConcluded: true
skipExcluded: true
Expand Down
Loading

0 comments on commit baca8ab

Please sign in to comment.