From 8115929b38c88ff24368df0361a18740ec441544 Mon Sep 17 00:00:00 2001 From: "kshakir (Khalid Shakir) (he/him)" Date: Tue, 30 Mar 2021 17:06:47 -0400 Subject: [PATCH] Build updates incl. scala 2.12.12 and adoptopenjdk 11-hs BT-125 BT-126 (#6194) --- .circleci/config.yml | 11 -- .gitignore | 1 + .sdkmanrc | 3 + .travis.yml | 25 +-- CHANGELOG.md | 5 + centaur/run_tests_parallel.sh | 15 -- centaur/test_cromwell.sh | 10 +- codegen_java/build.sbt | 7 +- codegen_java/project/Publishing.scala | 4 +- codegen_java/project/build.properties | 2 +- .../core/path/MappedPathBuilder.scala | 31 --- .../core/path/MappedPathBuilderSpec.scala | 88 --------- docs/Releases.md | 3 +- docs/WOMtool.md | 2 +- docs/developers/Building.md | 3 +- docs/developers/Centaur.md | 2 +- project/ContinuousIntegration.scala | 4 +- project/Merging.scala | 6 +- project/Publishing.scala | 11 +- project/Settings.scala | 31 ++- project/Testing.scala | 12 +- project/Version.scala | 18 +- project/build.properties | 2 +- project/plugins.sbt | 5 - project/project/build.properties | 2 +- publish/docker-setup.sh | 28 ++- scripts/docker-develop/Dockerfile | 42 ++-- .../impl/MetadataDatabaseAccessSpec.scala | 2 +- src/ci/bin/test-deadlock.sh | 4 +- src/ci/bin/test.inc.sh | 180 +++++++++++++----- src/ci/bin/testCheckPublish.sh | 2 +- src/ci/bin/testDockerScripts.sh | 2 +- src/ci/bin/testHoricromtalDeadlock.sh | 2 - src/ci/bin/testMetadataComparisonPython.sh | 14 +- .../docker-compose/cromwell-test/Dockerfile | 3 +- .../cromwell-test/docker-setup.sh | 83 ++++++-- wom/src/main/scala/wom/views/GraphPrint.scala | 29 +-- .../womtool/WomtoolJsonCommandSpec.scala | 3 +- .../scala/womtool/WomtoolValidateSpec.scala | 22 ++- 39 files changed, 378 insertions(+), 341 deletions(-) create mode 100644 .sdkmanrc delete mode 100755 centaur/run_tests_parallel.sh delete mode 100644 core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala delete mode 100644 core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala diff --git a/.circleci/config.yml b/.circleci/config.yml index 9829949e974..32882143ed7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,16 +4,6 @@ orbs: slack: circleci/slack@4.2.1 build-tools: circleci/build-tools@2.9.0 -commands: - install_adoptopenjdk: - description: "Installing adoptopenjdk 8 and setting it as default Java" - steps: - - run: wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add - - - run: sudo add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ - - run: sudo apt update - - run: sudo apt install adoptopenjdk-8-hotspot - - run: sudo update-java-alternatives --set adoptopenjdk-8-hotspot-amd64 - jobs: test: parameters: @@ -41,7 +31,6 @@ jobs: if [[ -z "${CI_PULL_REQUEST}" ]] && [[ "${BUILD_TYPE}" != "sbt" ]] ; then circleci-agent step halt fi - - install_adoptopenjdk - checkout - run: name: Custom step - configure GIT identity diff --git a/.gitignore b/.gitignore index 9eb2de8d647..77b2ff1457a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *~ .DS_Store .artifactory +.bsp .idea/* **/.idea/* .ensime_cache/* diff --git a/.sdkmanrc b/.sdkmanrc new file mode 100644 index 00000000000..59f306d13ad --- /dev/null +++ b/.sdkmanrc @@ -0,0 +1,3 @@ +# Enable auto-env through the sdkman_auto_env config +# Add key=value pairs of SDKs to use below +java=11.0.10.hs-adpt diff --git a/.travis.yml b/.travis.yml index 21ab778c719..3c098c36611 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,10 @@ -sudo: required -dist: xenial +os: linux +dist: focal services: - docker -language: scala -scala: - - 2.12.9 +language: minimal git: depth: false -jdk: - - openjdk8 cache: directories: - $HOME/.ivy2/cache @@ -21,10 +17,7 @@ before_cache: - find $HOME/.coursier/cache -name "ivydata-*.properties" -print -delete - find $HOME/.sbt -name "*.lock" -print -delete env: - global: - - PYTHON3_HOME=/opt/python/3.7.1 - - PATH=$PYTHON3_HOME/bin\:$PATH - matrix: + jobs: # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked - >- BUILD_TYPE=centaurAws @@ -101,6 +94,16 @@ env: BUILD_TYPE=dockerScripts - >- BUILD_TYPE=sbt + BUILD_SBT_INCLUDE=engine + - >- + BUILD_TYPE=sbt + BUILD_SBT_INCLUDE=server + - >- + BUILD_TYPE=sbt + BUILD_SBT_INCLUDE=services + - >- + BUILD_TYPE=sbt + BUILD_SBT_EXCLUDE='engine|server|services' - >- BUILD_TYPE=dbms - >- diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c43fb7e670..17cab5308ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## 60 Release Notes +### Java 11 + +As of this version, a distribution of Java 11 is required to run Cromwell. Cromwell is developed, tested, and +containerized using [AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/). + ### Hybrid metadata storage ("carboniting") removed Carboniting functionality has been removed from Cromwell. diff --git a/centaur/run_tests_parallel.sh b/centaur/run_tests_parallel.sh deleted file mode 100755 index bafdcd0089c..00000000000 --- a/centaur/run_tests_parallel.sh +++ /dev/null @@ -1,15 +0,0 @@ -# -# This approach allows for the running of scalatests that mixin the -# trait 'ParallelTestExecution' to be executed in parallel with a -# throttle to the number of threads. Simply running 'sbt test' -# has that throttle hardcoded to be 2*cores -# - -# optional parameter for parallelism, defaults to 3 -THREADS=${1:-3} - -echo "Running tests with ${THREADS}-way parallelism" - -sbt --warn centaur/it:compile -CP=$(sbt "export centaur/it:dependencyClasspath" --error) -java -cp $CP org.scalatest.tools.Runner -R centaur/target/scala-2.12/it-classes -oD -PS${THREADS} diff --git a/centaur/test_cromwell.sh b/centaur/test_cromwell.sh index e96bacba469..44f9d22447b 100755 --- a/centaur/test_cromwell.sh +++ b/centaur/test_cromwell.sh @@ -111,7 +111,7 @@ if [[ -n ${CROMWELL_BRANCH} ]]; then git checkout "${CROMWELL_BRANCH}" git pull echo "Building Cromwell" - sbt --warn assembly >> "${ASSEMBLY_LOG}" 2>&1 + sbt -Dsbt.supershell=false --warn assembly >> "${ASSEMBLY_LOG}" 2>&1 cd .. # This is the "branch" logic but sets the CROMWELL_JAR to be used in either the "branch" or "jar" use cases. # Note that this may not be necessary in the docker-compose use case. @@ -124,11 +124,11 @@ cd "${RUN_DIR}" TEST_STATUS="failed" if [[ "${CENTAUR_SBT_COVERAGE}" == "true" ]]; then - sbt --warn coverage centaur/it:compile - CP=$(sbt --warn coverage "export centaur/it:dependencyClasspath" -error) + sbt -Dsbt.supershell=false --warn coverage centaur/it:compile + CP=$(sbt -no-colors --error coverage "export centaur/it:dependencyClasspath") else - sbt --warn centaur/it:compile - CP=$(sbt --warn "export centaur/it:dependencyClasspath" -error) + sbt -Dsbt.supershell=false --warn centaur/it:compile + CP=$(sbt -no-colors --error "export centaur/it:dependencyClasspath") fi # Add the it-classes folder to the classpath to ensure logback configuration files are picked up. diff --git a/codegen_java/build.sbt b/codegen_java/build.sbt index 6e66efa5ec3..dfb7300d55d 100644 --- a/codegen_java/build.sbt +++ b/codegen_java/build.sbt @@ -6,10 +6,10 @@ lazy val root = (project in file(".")). Seq(organization := "org.broadinstitute.cromwell", name := "cromwell-client", version := createVersion("0.1"), - scalaVersion := "2.12.8", + scalaVersion := "2.12.12", scalacOptions ++= Seq("-feature"), - javacOptions in compile ++= Seq("-Xlint:deprecation"), - publishArtifact in (Compile, packageDoc) := false, + compile / javacOptions ++= Seq("-Xlint:deprecation"), + Compile / packageDoc / publishArtifact := false, resolvers += Resolver.mavenLocal, updateOptions := updateOptions.value.withGigahorse(false), libraryDependencies ++= Seq( @@ -25,4 +25,3 @@ lazy val root = (project in file(".")). "com.novocode" % "junit-interface" % "0.10" % "test" )) ++ publishSettings:_* ) - \ No newline at end of file diff --git a/codegen_java/project/Publishing.scala b/codegen_java/project/Publishing.scala index 57e680adbbe..6c3de9881fc 100644 --- a/codegen_java/project/Publishing.scala +++ b/codegen_java/project/Publishing.scala @@ -24,8 +24,8 @@ import Artifactory._ //priority over the local package cache. see here: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 Seq( publishTo := Option(artifactoryResolver(false)), - publishArtifact in Compile := true, - publishArtifact in Test := true, + Compile / publishArtifact := true, + Test / publishArtifact := true, credentials += artifactoryCredentials ) diff --git a/codegen_java/project/build.properties b/codegen_java/project/build.properties index 2e6e3d24608..dbae93bcfd5 100644 --- a/codegen_java/project/build.properties +++ b/codegen_java/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.6 \ No newline at end of file +sbt.version=1.4.9 diff --git a/core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala b/core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala deleted file mode 100644 index ade950cb851..00000000000 --- a/core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala +++ /dev/null @@ -1,31 +0,0 @@ -package cromwell.core.path - -import java.nio.file.FileSystems - -import scala.util.Try - -class MappedPathBuilder(prefix: String, mappedRoot: String) extends PathBuilder { - override def name: String = s"$prefix mapped to $mappedRoot" - - override def build(pathAsString: String): Try[Path] = Try { - if (pathAsString.startsWith(prefix)) { - val fileSystem = FileSystems.getDefault - val root = fileSystem.getPath(mappedRoot) - val path = fileSystem.getPath(mappedRoot, pathAsString.stripPrefix(prefix)) - MappedPath(prefix, root, path) - - } else { - throw new IllegalArgumentException(s"$pathAsString must start with $prefix") - } - } -} - -case class MappedPath(prefix: String, mappedRoot: NioPath, nioPath: NioPath) extends Path { - override protected def newPath(nioPath: NioPath): Path = MappedPath(prefix, mappedRoot, nioPath) - - override def pathAsString: String = nioPath.toString - - override def pathWithoutScheme: String = mappedRoot.relativize(nioPath).toString - - def prefixedPathAsString: String = prefix + pathWithoutScheme -} diff --git a/core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala b/core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala deleted file mode 100644 index e206a090e82..00000000000 --- a/core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala +++ /dev/null @@ -1,88 +0,0 @@ -package cromwell.core.path - -import org.scalatest.Suite -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.matchers.should.Matchers -import org.scalatest.prop.Tables.Table - -class MappedPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers with PathBuilderSpecUtils { - behavior of "MappedPathBuilder" - - it should behave like truncateCommonRoots(prefixedPathBuilder, pathsToTruncate) - - goodPaths foreach { goodPath => - it should behave like buildGoodPath(prefixedPathBuilder, goodPath) - } - - badPaths foreach { badPath => - it should behave like buildBadPath(prefixedPathBuilder, badPath) - } - - private def pathsToTruncate = Table( - ("context", "file", "relative"), - ("gs://bucket", "gs://bucket/path/to/file", "path/to/file"), - ("gs://bucket/path/to/my/dir", "gs://bucket/path/to/my/dir/file", "file"), - ("gs://bucket/path/to/my/dir", "gs://bucket/path/to/my/dir//file", "file"), - ("gs://bucket/path/to/my//dir", "gs://bucket/path/to/my/dir/file", "file"), - ("gs://bucket/path/to/my//dir", "gs://bucket/path/to/my/dir//file", "file"), - ("gs://bucket/path/to/my/dir", "gs://bucket/path/./to/my/dir/file", "./to/my/dir/file"), - ("gs://bucket/path/to/my/dir/with/file", "gs://bucket/path/to/other/dir/with/file", "other/dir/with/file") - ) - - private def goodPaths = Seq( - GoodPath( - description = "a prefixed path", - path = "gs://hello/world", - normalize = false, - pathAsString = "/my/mapped/dir/hello/world", - pathWithoutScheme = "hello/world", - parent = "/my/mapped/dir/hello", - getParent = "/my/mapped/dir/hello", - root = "/", - name = "world", - getFileName = "world", - getNameCount = 5, - isAbsolute = true), - - GoodPath( - description = "an empty prefixed path", - path = "gs://", - normalize = false, - pathAsString = "/my/mapped/dir", - pathWithoutScheme = "", - parent = "/my/mapped", - getParent = "/my/mapped", - root = "/", - name = "dir", - getFileName = "dir", - getNameCount = 3, - isAbsolute = true), - - GoodPath( - description = "a prefixed path starting with .", - path = "gs://./hello/world", - normalize = false, - pathAsString = "/my/mapped/dir/./hello/world", - pathWithoutScheme = "./hello/world", - parent = "/my/mapped/dir/hello", - getParent = "/my/mapped/dir/./hello", - root = "/", - name = "world", - getFileName = "world", - getNameCount = 6, - isAbsolute = true) - ) - - private def badPaths = Seq( - BadPath("an empty path", "", " must start with gs://"), - BadPath("a https path", "https://hello/world", "https://hello/world must start with gs://"), - BadPath("a file uri path", "file:///hello/world", "file:///hello/world must start with gs://"), - BadPath("a relative file path", "hello/world", "hello/world must start with gs://"), - BadPath("an absolute file path", "/hello/world", "/hello/world must start with gs://"), - BadPath("an mapped directory path", "/my/mapped/dir/hello/world", "/my/mapped/dir/hello/world must start with gs://") - ) - - private lazy val prefixedPathBuilder = { - new MappedPathBuilder("gs://", "/my/mapped/dir") - } -} diff --git a/docs/Releases.md b/docs/Releases.md index ea2b297c860..1a446c90bc2 100644 --- a/docs/Releases.md +++ b/docs/Releases.md @@ -18,7 +18,8 @@ Mac users with Homebrew can also get Cromwell with the command `brew install cro This documentation frequently refers to a "Cromwell jar" with a name like `cromwell-.jar`. This is the main artifact in Cromwell releases that contains all executable Cromwell code and default configuration. -[Java 8](http://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html) is required to run Cromwell. +A distribution of Java 11 is required to run Cromwell. Cromwell is developed, tested, and containerized using +[AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/). For users running a Cromwell server [a docker image](https://hub.docker.com/r/broadinstitute/cromwell) has been made available. diff --git a/docs/WOMtool.md b/docs/WOMtool.md index 00b1f40abd2..751e19b3a17 100644 --- a/docs/WOMtool.md +++ b/docs/WOMtool.md @@ -6,7 +6,7 @@ The following is the toolchain used for development of womtool. Other versions * [Scala 2.12](http://www.scala-lang.org/) * [SBT 1.x](https://www.scala-sbt.org/) -* [Java 8](http://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html) +* [AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/) * [Git](https://git-scm.com/) ## Building diff --git a/docs/developers/Building.md b/docs/developers/Building.md index 0adfb040e7b..9322f160120 100644 --- a/docs/developers/Building.md +++ b/docs/developers/Building.md @@ -5,7 +5,7 @@ features or fixes, the following are required to build Cromwell from source: * [Scala 2.12](http://www.scala-lang.org/) * [SBT 1.x](https://www.scala-sbt.org/) -* [Java 8](http://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html) +* [AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/) * [Git](https://git-scm.com/) You can also use the [development image](https://github.com/broadinstitute/cromwell/tree/develop/scripts/docker-develop), and build a development container to work inside: @@ -47,4 +47,3 @@ $ sbt server/docker ``` This will build and tag a Docker image with a name like `broadinstitute/cromwell:-SNAP`. - diff --git a/docs/developers/Centaur.md b/docs/developers/Centaur.md index 422b2bf96e2..795a8e24453 100644 --- a/docs/developers/Centaur.md +++ b/docs/developers/Centaur.md @@ -16,7 +16,7 @@ There are two ways to invoke the integration tests: * `sbt "centaur/it:test"` - compiles and run via sbt directly, simple but also has the problem of running 2x cores tests in parallel which can overwhelm your Cromwell server if running in a development environment -* `run_tests_parallel.sh [THREADS]` - runs the same tests with an enforced parallelism limit. Defaults to `3` if not specified +* `src/ci/bin/testCentaurLocal.sh` - runs the same tests using the continuous integration pipeline configuration ### Tags diff --git a/project/ContinuousIntegration.scala b/project/ContinuousIntegration.scala index ef38e7d8569..321ac0a69ab 100644 --- a/project/ContinuousIntegration.scala +++ b/project/ContinuousIntegration.scala @@ -50,10 +50,10 @@ object ContinuousIntegration { def aggregateSettings(rootProject: Project): Seq[Setting[_]] = List( // Before compiling, check if the expected projects are aggregated so that they will be compiled-and-tested too. - compile in Compile := { + Compile / compile := { streams.value.log // make sure logger is loaded validateAggregatedProjects(rootProject, state.value) - (compile in Compile).value + (Compile / compile).value }, ) diff --git a/project/Merging.scala b/project/Merging.scala index 9d46dee3232..964d342a234 100644 --- a/project/Merging.scala +++ b/project/Merging.scala @@ -23,7 +23,7 @@ object Merging { case "maven" :: "com.google.guava" :: xs => MergeStrategy.first case _ => - val oldStrategy = (assemblyMergeStrategy in assembly).value + val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) } case x@PathList("OSGI-INF", path@_*) => @@ -33,7 +33,7 @@ object Merging { case "l10n" :: "bundle.properties" :: Nil => MergeStrategy.concat case _ => - val oldStrategy = (assemblyMergeStrategy in assembly).value + val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) } case "asm-license.txt" | "module-info.class" | "overview.html" | "cobertura.properties" => @@ -41,7 +41,7 @@ object Merging { case PathList("mime.types") => MergeStrategy.last case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value + val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) } } diff --git a/project/Publishing.scala b/project/Publishing.scala index 62f4f53c71f..f93d443a22d 100644 --- a/project/Publishing.scala +++ b/project/Publishing.scala @@ -5,7 +5,6 @@ import sbt.Keys._ import sbt._ import sbtassembly.AssemblyPlugin.autoImport._ import sbtdocker.DockerPlugin.autoImport._ -import ContinuousIntegration._ import sbtdocker.Instruction import scala.sys.process._ @@ -36,10 +35,10 @@ object Publishing { val versionsCsv = if (Version.isSnapshot) version.value else s"$cromwellVersion,${version.value}" sys.env.getOrElse("CROMWELL_SBT_DOCKER_TAGS", versionsCsv).split(",") }, - imageNames in docker := dockerTags.value map { tag => + docker / imageNames := dockerTags.value map { tag => ImageName(namespace = Option("broadinstitute"), repository = name.value, tag = Option(tag)) }, - dockerfile in docker := { + docker / dockerfile := { // The assembly task generates a fat JAR file val artifact: File = assembly.value val artifactTargetPath = s"/app/${artifact.name}" @@ -47,7 +46,7 @@ object Publishing { val additionalDockerInstr: Seq[Instruction] = dockerCustomSettings.value new Dockerfile { - from("us.gcr.io/broad-dsp-gcr-public/base/jre:8-debian") + from("us.gcr.io/broad-dsp-gcr-public/base/jre:11-debian") expose(8000) add(artifact, artifactTargetPath) runRaw(s"ln -s $artifactTargetPath /app/$projectName.jar") @@ -94,11 +93,11 @@ object Publishing { additionalDockerInstr.foreach(addInstruction) } }, - buildOptions in docker := BuildOptions( + docker / buildOptions := BuildOptions( cache = false, removeIntermediateContainers = BuildOptions.Remove.Always ), - dockerCustomSettings in ThisBuild := Nil, // setting the default value + ThisBuild / dockerCustomSettings := Nil, // setting the default value ) def dockerPushSettings(pushEnabled: Boolean): Seq[Setting[_]] = { diff --git a/project/Settings.scala b/project/Settings.scala index 2978e3c27fd..9196e5eb154 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -10,7 +10,6 @@ import sbt._ import sbtassembly.AssemblyPlugin import sbtassembly.AssemblyPlugin.autoImport._ import sbtdocker.{DockerPlugin, Instruction, Instructions} -import sbtrelease.ReleasePlugin object Settings { @@ -47,7 +46,6 @@ object Settings { "-Ybackend-parallelism", "3", "-Ycache-plugin-class-loader:last-modified", "-Ycache-macro-class-loader:last-modified", - "-target:jvm-1.8", "-encoding", "UTF-8" ) @@ -88,23 +86,24 @@ object Settings { ) lazy val assemblySettings = Seq( - assemblyJarName in assembly := name.value + "-" + version.value + ".jar", - test in assembly := {}, - assemblyMergeStrategy in assembly := customMergeStrategy.value, - logLevel in assembly := - sys.env.get("CROMWELL_SBT_ASSEMBLY_LOG_LEVEL").flatMap(Level.apply).getOrElse((logLevel in assembly).value) + assembly / assemblyJarName := name.value + "-" + version.value + ".jar", + assembly / test := {}, + assembly / assemblyMergeStrategy := customMergeStrategy.value, + assembly / logLevel := + sys.env.get("CROMWELL_SBT_ASSEMBLY_LOG_LEVEL").flatMap(Level.apply).getOrElse((assembly / logLevel).value) ) - val Scala2_12Version = "2.12.9" + // 2.12.13 blocked on the release of sbt-scoverage 1.6.2 https://github.com/scoverage/sbt-scoverage/issues/319 + val Scala2_12Version = "2.12.12" val ScalaVersion = Scala2_12Version - val sharedSettings = ReleasePlugin.projectSettings ++ + val sharedSettings = cromwellVersionWithGit ++ artifactorySettings ++ List( organization := "org.broadinstitute", scalaVersion := ScalaVersion, resolvers ++= commonResolvers, // Don't run tasks in parallel, especially helps in low CPU environments like Travis - parallelExecution in Global := false, - concurrentRestrictions in Global ++= List( + Global / parallelExecution := false, + Global / concurrentRestrictions ++= List( // Don't run any other tasks while running tests, especially helps in low CPU environments like Travis Tags.exclusive(Tags.Test), // Only run tests on one sub-project at a time, especially helps in low CPU environments like Travis @@ -113,10 +112,10 @@ object Settings { dependencyOverrides ++= cromwellDependencyOverrides, scalacOptions ++= baseSettings ++ warningSettings ++ consoleHostileSettings, // http://stackoverflow.com/questions/31488335/scaladoc-2-11-6-fails-on-throws-tag-with-unable-to-find-any-member-to-link#31497874 - scalacOptions in(Compile, doc) ++= baseSettings ++ List("-no-link-warnings"), + Compile / doc / scalacOptions ++= baseSettings ++ List("-no-link-warnings"), // No console-hostile options, otherwise the console is effectively unusable. // https://github.com/sbt/sbt/issues/1815 - scalacOptions in(Compile, console) --= consoleHostileSettings, + Compile / console / scalacOptions --= consoleHostileSettings, addCompilerPlugin(paradisePlugin), excludeDependencies ++= List( "org.typelevel" % "simulacrum-scalafix-annotations_2.12", @@ -152,7 +151,7 @@ object Settings { ) ) - val swaggerUiSettings = List(resourceGenerators in Compile += writeSwaggerUiVersionConf) + val swaggerUiSettings = List(Compile / resourceGenerators += writeSwaggerUiVersionConf) val backendSettings = List(addCompilerPlugin(kindProjectorPlugin)) val engineSettings = swaggerUiSettings val cromiamSettings = swaggerUiSettings @@ -182,7 +181,7 @@ object Settings { if (integrationTests) addIntegrationTestSettings else identity, _ .disablePlugins(AssemblyPlugin) - .settings(resourceGenerators in Compile += writeProjectVersionConf) + .settings(Compile / resourceGenerators += writeProjectVersionConf) .settings(customSettings) ) @@ -210,7 +209,7 @@ object Settings { }, _ .settings(assemblySettings) - .settings(resourceGenerators in Compile += writeProjectVersionConf) + .settings(Compile / resourceGenerators += writeProjectVersionConf) .settings(customSettings) ) diff --git a/project/Testing.scala b/project/Testing.scala index 7868d52d958..f5544540489 100644 --- a/project/Testing.scala +++ b/project/Testing.scala @@ -94,22 +94,22 @@ object Testing { val testSettings = List( libraryDependencies ++= testDependencies.map(_ % Test), // `test` (or `assembly`) - Run most tests - testOptions in Test ++= Seq(TestReportArgs) ++ filterTestArgs, + Test / testOptions ++= Seq(TestReportArgs) ++ filterTestArgs, // `alltests:test` - Run all tests - testOptions in AllTests := (testOptions in Test).value.diff(filterTestArgs), + AllTests / testOptions := (Test / testOptions).value.diff(filterTestArgs), // Add scalameter as a test framework in the CromwellBenchmarkTest scope - testFrameworks in CromwellBenchmarkTest += new TestFramework("org.scalameter.ScalaMeterFramework"), + CromwellBenchmarkTest / testFrameworks += new TestFramework("org.scalameter.ScalaMeterFramework"), // Don't execute benchmarks in parallel - parallelExecution in CromwellBenchmarkTest := false, + CromwellBenchmarkTest / parallelExecution := false, // Make sure no secrets are commited to git minnieKenny := { val log = streams.value.log val args = spaceDelimited("").parsed minnieKennySingleRunner.runOnce(log, args) }, - test in Test := { + Test / test := { minnieKenny.toTask("").value - (test in Test).value + (Test / test).value }, ) diff --git a/project/Version.scala b/project/Version.scala index 6d38ee23ff3..c88f43414a7 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -18,38 +18,38 @@ object Version { // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = Seq( - git.versionProperty in ThisBuild := "project.version", - git.baseVersion in ThisBuild := cromwellVersion, - version in ThisBuild := + ThisBuild / git.versionProperty := "project.version", + ThisBuild / git.baseVersion := cromwellVersion, + ThisBuild / version := makeVersion( versionProperty = git.versionProperty.value, baseVersion = git.baseVersion.?.value, headCommit = git.gitHeadCommit.value), - shellPrompt in ThisBuild := { state => "%s| %s> ".format(GitCommand.prompt.apply(state), cromwellVersion) } + ThisBuild / shellPrompt := { state => "%s| %s> ".format(GitCommand.prompt.apply(state), cromwellVersion) } ) val writeProjectVersionConf: Def.Initialize[Task[Seq[File]]] = Def.task { - writeVersionConf(name.value, (resourceManaged in Compile).value, version.value) + writeVersionConf(name.value, (Compile / resourceManaged).value, version.value) } val writeSwaggerUiVersionConf: Def.Initialize[Task[Seq[File]]] = Def.task { - writeVersionConf("swagger-ui", (resourceManaged in Compile).value, swaggerUiVersion) + writeVersionConf("swagger-ui", (Compile / resourceManaged).value, swaggerUiVersion) } /** * Writes a version.conf compatible with cromwell-common's VersionUtil. Returns the written file wrapped in a Seq to - * make it compatible for appending to `resourceGenerators in Compile`. + * make it compatible for appending to `Compile / resourceGenerators`. * * Ex: * {{{ - * resourceGenerators in Compile += writeVersionConf(name.value, (resourceManaged in Compile).value, version.value) + * Compile / resourceGenerators += writeVersionConf(name.value, (Compile / resourceManaged).value, version.value) * }}} * * For a project named "my-project", writes a conf named "my-project-version.conf" containing * "my.project.version = [version]" * * @param projectName Name of the project - * @param directory The managed resource directory, usually `(resourceManaged in Compile).value` + * @param directory The managed resource directory, usually `(Compile / resourceManaged).value` * @param version The version to write * @return The written file */ diff --git a/project/build.properties b/project/build.properties index e3de7827bdf..f0f04096225 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # scala-steward:off -sbt.version=1.2.8 +sbt.version=1.4.9 diff --git a/project/plugins.sbt b/project/plugins.sbt index 0b9824f8a76..48204e15836 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,9 +1,4 @@ addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.8.0") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0") -addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") -addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.13") -addSbtPlugin("io.get-coursier" % "sbt-coursier" % "2.0.3") -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1") diff --git a/project/project/build.properties b/project/project/build.properties index 9fac47b1c26..f0f04096225 100644 --- a/project/project/build.properties +++ b/project/project/build.properties @@ -1,2 +1,2 @@ # scala-steward:off -sbt.version=1.2.8 +sbt.version=1.4.9 diff --git a/publish/docker-setup.sh b/publish/docker-setup.sh index 1dae8b0c553..d537171662f 100755 --- a/publish/docker-setup.sh +++ b/publish/docker-setup.sh @@ -10,18 +10,32 @@ apt-get install \ curl \ git \ gnupg \ - openjdk-8-jdk \ + wget \ -y --no-install-recommends +# setup install for adoptopenjdk +# https://adoptopenjdk.net/installation.html#linux-pkg-deb +wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - +echo "deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb $( + grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 + ) main" | + tee /etc/apt/sources.list.d/adoptopenjdk.list + # Install jq 1.6 to ensure --rawfile is supported curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/bin/jq chmod +x /usr/bin/jq -# Install sbt via https://www.scala-sbt.org/1.0/docs/Installing-sbt-on-Linux.html -echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 apt-get update -apt-get install sbt -y --no-install-recommends +apt-get install \ + adoptopenjdk-11-hotspot \ + -y --no-install-recommends + +# sbt launcher non-deb package installation instructions adapted from +# - https://github.com/sbt/sbt/releases/tag/v1.4.9 +# - https://github.com/broadinstitute/scala-baseimage/pull/4/files +curl --location --fail --silent --show-error "https://github.com/sbt/sbt/releases/download/v1.4.9/sbt-1.4.9.tgz" | + tar zxf - -C /usr/share +update-alternatives --install /usr/bin/sbt sbt /usr/share/sbt/bin/sbt 1 -# Update sbt -sbt sbtVersion +# Update sbt launcher +sbt -Dsbt.supershell=false -Dsbt.rootdir=true sbtVersion diff --git a/scripts/docker-develop/Dockerfile b/scripts/docker-develop/Dockerfile index d7748d01f08..56fbb0dcfc5 100644 --- a/scripts/docker-develop/Dockerfile +++ b/scripts/docker-develop/Dockerfile @@ -1,7 +1,10 @@ -FROM adoptopenjdk:8 +FROM ubuntu:focal + +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update -q && \ apt-get upgrade -qq && \ + apt-get install -y curl wget gnupg gnupg2 && \ rm -rf /var/lib/apt/lists/* # docker build -t vanessa/cromwell-dev . @@ -10,19 +13,33 @@ RUN apt-get update -q && \ # # Scala 2.12 # SBT 1.x -# Java 8 +# Java 11 # Git # Env variables -ENV SCALA_VERSION 2.12.9 -ENV SBT_VERSION 1.2.8 +ENV SCALA_VERSION 2.12.12 +ENV SBT_VERSION 1.4.9 + +# +## AdoptOpenJDK Hotspot +# +# https://adoptopenjdk.net/installation.html#linux-pkg-deb +RUN \ +wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - && \ +echo \ +"deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb $(grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2) main" | \ +tee /etc/apt/sources.list.d/adoptopenjdk.list && \ +apt-get update && \ +apt-get install -y adoptopenjdk-11-hotspot # ## Scala # RUN mkdir -p /home/pigman && \ - curl -fsL https://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz | tar xfz - -C /opt/ && \ + curl \ + --location --fail --silent --show-error \ + https://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz | tar xfz - -C /opt/ && \ echo >> /home/pigman/.bashrc && \ echo "export PATH=/opt/scala-$SCALA_VERSION/bin:$PATH" >> /home/pigman/.bashrc @@ -30,12 +47,15 @@ RUN mkdir -p /home/pigman && \ ## sbt # -RUN curl -L -o sbt-$SBT_VERSION.deb https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb && \ - dpkg -i sbt-$SBT_VERSION.deb && \ - rm sbt-$SBT_VERSION.deb && \ - apt-get update && \ - apt-get install sbt && \ - sbt sbtVersion +# non-deb package installation instructions adapted from +# - https://github.com/sbt/sbt/releases/tag/v1.4.9 +# - https://github.com/broadinstitute/scala-baseimage/pull/4/files +RUN curl \ + --location --fail --silent --show-error \ + "https://github.com/sbt/sbt/releases/download/v$SBT_VERSION/sbt-$SBT_VERSION.tgz" | \ + tar zxf - -C /usr/share && \ + update-alternatives --install /usr/bin/sbt sbt /usr/share/sbt/bin/sbt 1 && \ + sbt -Dsbt.supershell=false -Dsbt.rootdir=true sbtVersion # Instruct user to add code here during development RUN useradd pigman && \ diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala index 0bf2df68516..aab861d89dc 100644 --- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala +++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala @@ -132,7 +132,7 @@ class MetadataDatabaseAccessSpec extends AnyFlatSpec with CromwellTimeoutSpec wi def unorderedEvents(id: WorkflowId): Future[Vector[MetadataEvent]] = { val workflowKey = MetadataKey(id, jobKey = None, key = null) - val now = OffsetDateTime.now() + val now = OffsetDateTime.now().withNano(0) val yesterday = now.minusDays(1) val tomorrow = now.plusDays(1) diff --git a/src/ci/bin/test-deadlock.sh b/src/ci/bin/test-deadlock.sh index e3764cd182f..9cddbb65ade 100755 --- a/src/ci/bin/test-deadlock.sh +++ b/src/ci/bin/test-deadlock.sh @@ -2,12 +2,12 @@ set -euo pipefail -pip install docker-py +pip3 install docker-py cromwell_hosts_file="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/cromwell_hosts.txt" # Get a list of cromwell hosts. -python "${CROMWELL_BUILD_SCRIPTS_DIRECTORY}"/get_cromwell_hosts.py > "${cromwell_hosts_file}" +python3 "${CROMWELL_BUILD_SCRIPTS_DIRECTORY}"/get_cromwell_hosts.py > "${cromwell_hosts_file}" cromwell_hosts=() while IFS=$'\n' read -r line; do cromwell_hosts+=("$line") diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 9c2ef12c2f7..7d743071769 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -343,6 +343,9 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_SBT_COVERAGE_COMMAND="" fi + CROMWELL_BUILD_SBT_INCLUDE="${BUILD_SBT_INCLUDE:-}" + CROMWELL_BUILD_SBT_EXCLUDE="${BUILD_SBT_EXCLUDE:-}" + case "${CROMWELL_BUILD_TYPE}" in centaurPapiUpgradePapiV2alpha1*) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_v2beta_upgrade_application.conf" @@ -382,7 +385,6 @@ cromwell::private::create_build_variables() { hours_to_minutes=60 CROMWELL_BUILD_HEARTBEAT_MINUTES=$((20 * hours_to_minutes)) - export CROMWELL_BUILD_UNIT_TEST_EXCLUDE_TAGS export CROMWELL_BUILD_BACKEND_TYPE export CROMWELL_BUILD_BRANCH export CROMWELL_BUILD_CROMWELL_CONFIG @@ -412,21 +414,24 @@ cromwell::private::create_build_variables() { export CROMWELL_BUILD_OS_LINUX export CROMWELL_BUILD_PRIOR_VERSION_NUMBER export CROMWELL_BUILD_PROVIDER + export CROMWELL_BUILD_PROVIDER_CIRCLE export CROMWELL_BUILD_PROVIDER_JENKINS export CROMWELL_BUILD_PROVIDER_TRAVIS - export CROMWELL_BUILD_PROVIDER_CIRCLE export CROMWELL_BUILD_PROVIDER_UNKNOWN - export CROMWELL_BUILD_REQUIRES_SECURE export CROMWELL_BUILD_REQUIRES_PRIOR_VERSION + export CROMWELL_BUILD_REQUIRES_SECURE export CROMWELL_BUILD_RESOURCES_DIRECTORY export CROMWELL_BUILD_RESOURCES_SOURCES export CROMWELL_BUILD_ROOT_DIRECTORY export CROMWELL_BUILD_RUN_TESTS export CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND export CROMWELL_BUILD_SBT_COVERAGE_COMMAND + export CROMWELL_BUILD_SBT_EXCLUDE + export CROMWELL_BUILD_SBT_INCLUDE export CROMWELL_BUILD_SCRIPTS_DIRECTORY export CROMWELL_BUILD_TAG export CROMWELL_BUILD_TYPE + export CROMWELL_BUILD_UNIT_TEST_EXCLUDE_TAGS export CROMWELL_BUILD_URL export CROMWELL_BUILD_VAULT_EXECUTABLE export CROMWELL_BUILD_VAULT_ZIP @@ -772,8 +777,8 @@ cromwell::private::exec_test_script() { cromwell::private::stop_travis_defaults() { # https://stackoverflow.com/questions/27382295/how-to-stop-services-on-travis-ci-running-by-default#answer-27410479 - sudo /etc/init.d/mysql stop - sudo /etc/init.d/postgresql stop + sudo /etc/init.d/mysql stop || true + sudo /etc/init.d/postgresql stop || true } cromwell::private::delete_boto_config() { @@ -789,20 +794,58 @@ cromwell::private::delete_sbt_boot() { rm -rf ~/.sbt/boot/ } +cromwell::private::install_adoptopenjdk() { + # https://adoptopenjdk.net/installation.html#linux-pkg-deb + sudo apt-get install -y wget apt-transport-https gnupg + wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | + sudo apt-key add - + echo "deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb $( + grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 + ) main" | + sudo tee /etc/apt/sources.list.d/adoptopenjdk.list + sudo apt-get update + sudo apt-get install -y adoptopenjdk-11-hotspot + sudo update-java-alternatives --set adoptopenjdk-11-hotspot-amd64 +} + +cromwell::private::install_sbt_launcher() { + # Install sbt launcher + # Non-deb package installation instructions adapted from + # - https://github.com/sbt/sbt/releases/tag/v1.4.9 + # - https://github.com/broadinstitute/scala-baseimage/pull/4/files + curl --location --fail --silent --show-error "https://github.com/sbt/sbt/releases/download/v1.4.9/sbt-1.4.9.tgz" | + sudo tar zxf - -C /usr/share + sudo update-alternatives --install /usr/bin/sbt sbt /usr/share/sbt/bin/sbt 1 +} + +cromwell::private::install_docker_compose() { + # Install or upgrade docker-compose so that we get the correct exit codes + # https://docs.docker.com/compose/release-notes/#1230 + # https://docs.docker.com/compose/install/ + curl \ + --location --fail --silent --show-error \ + "https://github.com/docker/compose/releases/download/1.28.5/docker-compose-$(uname -s)-$(uname -m)" \ + > docker-compose + sudo mv docker-compose /usr/local/bin + sudo chmod +x /usr/local/bin/docker-compose +} + +cromwell::private::setup_pyenv_python_latest() { + # Make `python` whatever the most recent version of python installed + # Fixes cases where someone has set pyenv to override `python` to use an older `python2` instead of `python3` + pyenv global "$(pyenv versions --bare --skip-aliases | sort -t '.' -k1,1n -k2,2n -k3,3n | tail -n 1)" +} + cromwell::private::pip_install() { local pip_package pip_package="${1:?pip_install called without a package}"; shift if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]]; then - if [[ "${CROMWELL_BUILD_PROVIDER}" == "${CROMWELL_BUILD_PROVIDER_CIRCLE}" ]]; then - pip3 install "${pip_package}" "$@" - else - sudo -H "${PYTHON3_HOME}/bin/pip" install "${pip_package}" "$@" - fi + sudo -H "$(command -v pip3)" install "${pip_package}" "$@" elif [[ "${CROMWELL_BUILD_IS_VIRTUAL_ENV}" == "true" ]]; then - pip install "${pip_package}" "$@" + pip3 install "${pip_package}" "$@" else - pip install "${pip_package}" --user "$@" + pip3 install "${pip_package}" --user "$@" fi } @@ -1042,7 +1085,7 @@ cromwell::private::login_docker() { cromwell::private::render_secure_resources() { # Copy the CI resources, then render the secure resources using Vault - sbt --warn renderCiResources \ + sbt -Dsbt.supershell=false --warn renderCiResources \ || if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]]; then echo echo "Continuing without rendering secure resources." @@ -1061,20 +1104,11 @@ cromwell::private::render_secure_resources() { cromwell::private::copy_all_resources() { # Only copy the CI resources. Secure resources are not rendered. - sbt --warn copyCiResources + sbt -Dsbt.supershell=false --warn copyCiResources } cromwell::private::setup_secure_resources() { case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\ - "${CROMWELL_BUILD_PROVIDER_CIRCLE}") - # Try to login to vault, and if successful then use vault creds to login to docker. - # For those committers with vault access this avoids pull rate limits reported in BT-143. - cromwell::private::install_vault - cromwell::private::login_vault - cromwell::private::login_docker - cromwell::private::render_secure_resources - ;; "${CROMWELL_BUILD_PROVIDER_JENKINS}") # Jenkins secret resources should have already been rendered outside the CI's docker-compose container. cromwell::private::copy_all_resources @@ -1113,10 +1147,11 @@ cromwell::private::assemble_jars() { # shellcheck disable=SC2086 CROMWELL_SBT_ASSEMBLY_LOG_LEVEL=error \ sbt \ + -Dsbt.supershell=false \ --warn \ ${CROMWELL_BUILD_SBT_COVERAGE_COMMAND} \ - ${CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND} \ - -error + --error \ + ${CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND} } cromwell::private::setup_prior_version_resources() { @@ -1155,21 +1190,21 @@ cromwell::private::setup_prior_version_resources() { } cromwell::private::generate_code_coverage() { - sbt --warn coverageReport -warn - sbt --warn coverageAggregate -warn + sbt -Dsbt.supershell=false --warn coverageReport + sbt -Dsbt.supershell=false --warn coverageAggregate bash <(curl -s https://codecov.io/bash) > /dev/null || true } cromwell::private::publish_artifacts_only() { - CROMWELL_SBT_ASSEMBLY_LOG_LEVEL=warn sbt "$@" publish -warn + CROMWELL_SBT_ASSEMBLY_LOG_LEVEL=warn sbt -Dsbt.supershell=false --warn "$@" publish } cromwell::private::publish_artifacts_and_docker() { - CROMWELL_SBT_ASSEMBLY_LOG_LEVEL=warn sbt "$@" publish dockerBuildAndPush -warn + CROMWELL_SBT_ASSEMBLY_LOG_LEVEL=warn sbt -Dsbt.supershell=false --warn "$@" publish dockerBuildAndPush } cromwell::private::publish_artifacts_check() { - sbt --warn verifyArtifactoryCredentialsExist -warn + sbt -Dsbt.supershell=false --warn verifyArtifactoryCredentialsExist } # Some CI environments want to know when new docker images are published. They do not currently poll dockerhub but do @@ -1365,12 +1400,27 @@ cromwell::build::setup_common_environment() { case "${CROMWELL_BUILD_PROVIDER}" in "${CROMWELL_BUILD_PROVIDER_TRAVIS}") cromwell::private::stop_travis_defaults + # Try to login to vault, and if successful then use vault creds to login to docker. + # For those committers with vault access this avoids pull rate limits reported in BT-143. + cromwell::private::install_vault + cromwell::private::login_vault + cromwell::private::login_docker + cromwell::private::install_adoptopenjdk + cromwell::private::install_sbt_launcher + cromwell::private::install_docker_compose cromwell::private::delete_boto_config cromwell::private::delete_sbt_boot cromwell::private::upgrade_pip cromwell::private::start_docker_databases ;; "${CROMWELL_BUILD_PROVIDER_CIRCLE}") + # Try to login to vault, and if successful then use vault creds to login to docker. + # For those committers with vault access this avoids pull rate limits reported in BT-143. + cromwell::private::install_vault + cromwell::private::login_vault + cromwell::private::login_docker + cromwell::private::install_adoptopenjdk + cromwell::private::setup_pyenv_python_latest cromwell::private::start_docker_databases ;; "${CROMWELL_BUILD_PROVIDER_JENKINS}"|\ @@ -1403,20 +1453,6 @@ cromwell::build::setup_conformance_environment() { cromwell::private::add_exit_function cromwell::private::cat_conformance_log } -cromwell::build::setup_docker_environment() { - if [[ "${CROMWELL_BUILD_PROVIDER}" == "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ]]; then - # Upgrade docker-compose so that we get the correct exit codes - docker-compose -version - sudo rm /usr/local/bin/docker-compose - curl \ - -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" \ - > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - docker-compose -version - fi -} - cromwell::private::find_or_assemble_cromwell_jar() { cromwell::private::find_cromwell_jar if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]] || ! cromwell::private::exists_cromwell_jar; then @@ -1459,13 +1495,64 @@ cromwell::build::build_cromwell_docker() { cromwell:build::run_sbt_test() { # CROMWELL_BUILD_SBT_COVERAGE_COMMAND allows enabling or disabling `sbt coverage`. + # Note: sbt logging level now affects the test logging level: https://github.com/sbt/sbt/issues/4480 + # Globally leaving the sbt log level at info for now. + # Disabling the supershell to reduce log levels. + # Splitting the JVMs for compilation then scalatest-with-cromwell to reduce memory pressure. + # Splitting the JVMs for testing-by-sbt-project to also reduce memory pressure. + # The list of sbt projects is generated by parsing this `log.info()` output, with log color formatting turned off: + # https://github.com/sbt/sbt/blob/v1.4.9/main/src/main/scala/sbt/Main.scala#L759-L760 + # For more information on testing and memory see also: https://olegych.github.io/blog/sbt-fork.html + + # shellcheck disable=SC2086 + sbt \ + -Dsbt.supershell=false \ + ${CROMWELL_BUILD_SBT_COVERAGE_COMMAND} \ + test:compile + + local sbt_tests + + if [[ -n "${CROMWELL_BUILD_SBT_INCLUDE}" ]]; then + # Test only the projects specified + sbt_tests=$( + sbt -Dsbt.log.noformat=true projects | + grep -F $'[info] \t ' | + awk '{print $2}' | + grep -E "^(${CROMWELL_BUILD_SBT_INCLUDE})$" | + awk '{printf "%s/test ", $1}' \ + || true + ) + elif [[ -n "${CROMWELL_BUILD_SBT_EXCLUDE}" ]]; then + # Test all the projects except a few exclusions + sbt_tests=$( + sbt -Dsbt.log.noformat=true projects | + grep -F $'[info] \t ' | + awk '{print $2}' | + grep -v -E "^(${CROMWELL_BUILD_SBT_EXCLUDE})$" | + awk '{printf "%s/test ", $1}' \ + || true + ) + else + # Test all the projects + sbt_tests="test" + fi + + # Ensure we are testing something + if [[ -z "${sbt_tests}" ]]; then + echo "Error: Unable to retrieve list of sbt projects." >&2 + echo "CROMWELL_BUILD_SBT_INCLUDE='${CROMWELL_BUILD_SBT_INCLUDE}'" >&2 + echo "CROMWELL_BUILD_SBT_EXCLUDE='${CROMWELL_BUILD_SBT_EXCLUDE}'" >&2 + exit 1 + fi + + echo "Starting sbt ${sbt_tests}" # shellcheck disable=SC2086 sbt \ - -warn \ + -Dsbt.supershell=false \ -Dakka.test.timefactor=${CROMWELL_BUILD_UNIT_SPAN_SCALE_FACTOR} \ -Dbackend.providers.Local.config.filesystems.local.localization.0=copy \ ${CROMWELL_BUILD_SBT_COVERAGE_COMMAND} \ - test + ${sbt_tests} } cromwell::build::run_centaur() { @@ -1507,6 +1594,7 @@ cromwell::build::generate_code_coverage() { cromwell::build::publish_artifacts() { if [[ "${CROMWELL_BUILD_PROVIDER}" == "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ]] && \ [[ "${CROMWELL_BUILD_TYPE}" == "sbt" ]] && \ + [[ "${CROMWELL_BUILD_SBT_INCLUDE}" == "" ]] && \ [[ "${CROMWELL_BUILD_EVENT}" == "push" ]]; then if [[ "${CROMWELL_BUILD_BRANCH}" == "develop" ]]; then diff --git a/src/ci/bin/testCheckPublish.sh b/src/ci/bin/testCheckPublish.sh index 176d6edb825..60989795ed0 100755 --- a/src/ci/bin/testCheckPublish.sh +++ b/src/ci/bin/testCheckPublish.sh @@ -10,6 +10,6 @@ cromwell::build::setup_common_environment cromwell::build::pip_install mkdocs mkdocs build -s -sbt --warn checkRestApiDocs +package assembly dockerPushCheck +doc +sbt -Dsbt.supershell=false --warn checkRestApiDocs +package assembly dockerPushCheck +doc git secrets --scan-history diff --git a/src/ci/bin/testDockerScripts.sh b/src/ci/bin/testDockerScripts.sh index dc6f258d54b..e9f670bba54 100755 --- a/src/ci/bin/testDockerScripts.sh +++ b/src/ci/bin/testDockerScripts.sh @@ -24,4 +24,4 @@ docker run \ --volume "${CROMWELL_BUILD_ROOT_DIRECTORY}:${CROMWELL_BUILD_ROOT_DIRECTORY}" \ --workdir "${CROMWELL_BUILD_ROOT_DIRECTORY}" \ "${docker_tag}" \ - sbt --warn assembly + sbt -Dsbt.supershell=false --warn assembly diff --git a/src/ci/bin/testHoricromtalDeadlock.sh b/src/ci/bin/testHoricromtalDeadlock.sh index f4fe2029a5a..e629f449b04 100755 --- a/src/ci/bin/testHoricromtalDeadlock.sh +++ b/src/ci/bin/testHoricromtalDeadlock.sh @@ -10,8 +10,6 @@ source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh cromwell::build::setup_common_environment -cromwell::build::setup_docker_environment - cromwell::build::build_cromwell_docker # Turn off exit-on-error temporarily, as we expect an error diff --git a/src/ci/bin/testMetadataComparisonPython.sh b/src/ci/bin/testMetadataComparisonPython.sh index e24589c7c26..f12b9ee5fe7 100755 --- a/src/ci/bin/testMetadataComparisonPython.sh +++ b/src/ci/bin/testMetadataComparisonPython.sh @@ -24,11 +24,11 @@ docker run -it --rm \ -v "${CROMWELL_BUILD_ROOT_DIRECTORY}/scripts/metadata_comparison:/metadata_comparison" \ python:3 /bin/bash -c " -pip install --upgrade requests -pip install --upgrade google-api-python-client -pip install --upgrade google-cloud -pip install --upgrade google-cloud-storage -pip install --upgrade gitpython -pip install --upgrade python-dateutil -python -m unittest discover -v /metadata_comparison +pip3 install --upgrade requests +pip3 install --upgrade google-api-python-client +pip3 install --upgrade google-cloud +pip3 install --upgrade google-cloud-storage +pip3 install --upgrade gitpython +pip3 install --upgrade python-dateutil +python3 -m unittest discover -v /metadata_comparison " diff --git a/src/ci/docker-compose/cromwell-test/Dockerfile b/src/ci/docker-compose/cromwell-test/Dockerfile index d3237ba18a2..46f6d6bc843 100644 --- a/src/ci/docker-compose/cromwell-test/Dockerfile +++ b/src/ci/docker-compose/cromwell-test/Dockerfile @@ -1,6 +1,7 @@ -FROM adoptopenjdk:8 +FROM ubuntu:focal WORKDIR /cromwell-test/ COPY docker-setup.sh ./ RUN ./docker-setup.sh USER hoggett +WORKDIR /home/hoggett diff --git a/src/ci/docker-compose/cromwell-test/docker-setup.sh b/src/ci/docker-compose/cromwell-test/docker-setup.sh index d0181827e7e..7f18cdfb9bc 100755 --- a/src/ci/docker-compose/cromwell-test/docker-setup.sh +++ b/src/ci/docker-compose/cromwell-test/docker-setup.sh @@ -2,9 +2,12 @@ set -o errexit -o nounset -o pipefail -apt-get update +# Don't prompt for inputs +# https://manpages.ubuntu.com/manpages/focal/en/man7/debconf.7.html#unattended%20package%20installation +export DEBIAN_FRONTEND=noninteractive -# install mysql and other dependencies +# install first round of packages without dependencies or required for those with dependencies +apt-get update apt-get install -y \ apt-utils \ apt-transport-https \ @@ -12,6 +15,7 @@ apt-get install -y \ ca-certificates \ curl \ gnupg \ + gnupg-agent \ gnupg2 \ jq \ mysql-client \ @@ -19,26 +23,75 @@ apt-get install -y \ python3-dev \ software-properties-common \ sudo \ + wget \ + +# setup install for adoptopenjdk +# https://adoptopenjdk.net/installation.html#linux-pkg-deb +wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - +echo "deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb $( + grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 + ) main" | + tee /etc/apt/sources.list.d/adoptopenjdk.list -# install sbt -echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 +# setup install for gcloud +# https://cloud.google.com/sdk/docs/install#deb +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | + tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | + apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + +# setup install for docker +# https://docs.docker.com/engine/install/ubuntu/ +apt-get remove -y docker docker-engine docker.io containerd runc || true +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +# install packages that required setup apt-get update -apt-get install -y sbt +apt-get install -y \ + adoptopenjdk-11-hotspot \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + google-cloud-sdk \ + +# remove downloaded archive files +# https://manpages.ubuntu.com/manpages/focal/en/man8/apt-get.8.html#description +apt-get clean + +# install sbt launcher +# non-deb package installation instructions adapted from +# - https://github.com/sbt/sbt/releases/tag/v1.4.9 +# - https://github.com/broadinstitute/scala-baseimage/pull/4/files +curl -L --silent "https://github.com/sbt/sbt/releases/download/v1.4.9/sbt-1.4.9.tgz" | + tar zxf - -C /usr/share +update-alternatives --install /usr/bin/sbt sbt /usr/share/sbt/bin/sbt 1 + +# install docker compose +# https://docs.docker.com/compose/install/ +curl \ + --location --fail --silent --show-error \ + "https://github.com/docker/compose/releases/download/1.28.5/docker-compose-$(uname -s)-$(uname -m)" \ + -o /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose + +# set python as python3 +# https://manpages.ubuntu.com/manpages/focal/en/man1/update-alternatives.1.html#commands +update-alternatives --install /usr/bin/python python /usr/bin/python3 1 # upgrade python dependencies +# https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3 get-pip.py -pip install --upgrade --force-reinstall pyopenssl - -# install gcloud -echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ - | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ - | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - -apt-get update -apt-get install -y google-cloud-sdk +pip3 install --upgrade --force-reinstall pyopenssl +# create a non-root user with access to sudo (but not `sudo -u` / `sudo -g`) +# https://www.sudo.ws/man/1.9.4/sudoers.man.html#Runas_Spec +# https://www.sudo.ws/man/1.9.4/sudoers.man.html#NOPASSWD +# https://www.sudo.ws/man/1.9.4/sudoers.man.html#EXAMPLES useradd hoggett echo "hoggett ALL=NOPASSWD: ALL" >> /etc/sudoers mkdir -p /home/hoggett diff --git a/wom/src/main/scala/wom/views/GraphPrint.scala b/wom/src/main/scala/wom/views/GraphPrint.scala index e6e41b8fda1..6bf146d136a 100644 --- a/wom/src/main/scala/wom/views/GraphPrint.scala +++ b/wom/src/main/scala/wom/views/GraphPrint.scala @@ -3,7 +3,6 @@ package wom.views import java.util.concurrent.atomic.AtomicInteger import cats.syntax.all._ -import cats.instances.set._ import cats.instances.list._ import cats.Monoid import wom.callable.ExecutableCallable @@ -15,7 +14,11 @@ import wom.views.GraphPrint._ final class GraphPrint(executableCallable: ExecutableCallable) { - def dotString = WorkflowDigraph(executableCallable.name, listAllGraphNodes(executableCallable.graph, new AtomicInteger(0), Map.empty)).dotString + def dotString: String = + WorkflowDigraph( + workflowName = executableCallable.name, + digraph = listAllGraphNodes(executableCallable.graph, new AtomicInteger(0), Map.empty), + ).dotString // A "monoid" is just a fancy way of saying "thing you can add together". // The cats library makes it easy to turn case classes into Monoids - ie into "things you can add together". @@ -46,12 +49,10 @@ final class GraphPrint(executableCallable: ExecutableCallable) { val clusterNumber = clusterCounter.getAndIncrement() val id = s"cluster_$clusterNumber" - def handleScatterVariableNode(scatterVariableNode: ScatterVariableNode): NodesAndLinks = { - scatterNode.scatterVariableNodes foldMap { node => - val dotNode = DotScatterVariableNode.apply(node, clusterNumber) - val links = upstreamLinks(node.linkToOuterGraph.graphNode, dotNode, Map.empty) - NodesAndLinks(Set(dotNode), links) - } + def handleScatterVariableNode(node: ScatterVariableNode): NodesAndLinks = { + val dotNode = DotScatterVariableNode.apply(node, clusterNumber) + val links = upstreamLinks(node.linkToOuterGraph.graphNode, dotNode, Map.empty) + NodesAndLinks(Set(dotNode), links) } val scatterExpressionNodesAndLinks = scatterNode.scatterVariableNodes.foldMap(handleScatterVariableNode) @@ -110,10 +111,10 @@ object GraphPrint { | compound=true; | | # Links - | ${digraph.links.toList.flatMap(_.dotString.lines).mkString(System.lineSeparator + " ")} + | ${digraph.links.toList.flatMap(_.dotString.linesIterator).mkString(System.lineSeparator + " ")} | | # Nodes - | ${digraph.nodes.toList.flatMap(_.dotString.lines).mkString(System.lineSeparator + " ")} + | ${digraph.nodes.toList.flatMap(_.dotString.linesIterator).mkString(System.lineSeparator + " ")} |}""".stripMargin } @@ -156,7 +157,7 @@ object GraphPrint { s"""subgraph $id { | style="filled,solid"; | fillcolor=white; - | ${nodes.toList.flatMap(_.dotString.lines).mkString(System.lineSeparator() + " ")} + | ${nodes.toList.flatMap(_.dotString.linesIterator).mkString(System.lineSeparator() + " ")} |}""".stripMargin } @@ -165,7 +166,7 @@ object GraphPrint { s"""subgraph $id { | style="filled,dashed"; | fillcolor=white; - | ${nodes.toList.flatMap(_.dotString.lines).mkString(System.lineSeparator() + " ")} + | ${nodes.toList.flatMap(_.dotString.linesIterator).mkString(System.lineSeparator() + " ")} |}""".stripMargin } @@ -193,9 +194,9 @@ object GraphPrint { upstreamLinksforNode(originNode).map(DotLink(_, origin)) } - def hasCallAncestor(g: GraphNode) = g.upstreamAncestry.exists(_.isInstanceOf[CommandCallNode]) + def hasCallAncestor(g: GraphNode): Boolean = g.upstreamAncestry.exists(_.isInstanceOf[CommandCallNode]) - def escapeQuotes(s: String) = s.replace("\"", "\\\"") + def escapeQuotes(s: String): String = s.replace("\"", "\\\"") def worthDisplaying(node: GraphNode): Boolean = node match { case _: CommandCallNode => true diff --git a/womtool/src/test/scala/womtool/WomtoolJsonCommandSpec.scala b/womtool/src/test/scala/womtool/WomtoolJsonCommandSpec.scala index a7c49b46d42..be11acb7ec5 100644 --- a/womtool/src/test/scala/womtool/WomtoolJsonCommandSpec.scala +++ b/womtool/src/test/scala/womtool/WomtoolJsonCommandSpec.scala @@ -10,7 +10,8 @@ import womtool.WomtoolMain.SuccessfulTermination object WomtoolJsonCommandSpec { final case class TestDefinition(testName: String, commandFormat: Seq[String], expectationFilename: String) - private def jsonLines(json: String): Set[String] = json.lines.toSet[String].map(_.stripSuffix(",")).filterNot(_.forall(_.isWhitespace)) + private def jsonLines(json: String): Set[String] = + json.linesIterator.toSet[String].map(_.stripSuffix(",")).filterNot(_.forall(_.isWhitespace)) } // Test for womtool command line commands which output JSON diff --git a/womtool/src/test/scala/womtool/WomtoolValidateSpec.scala b/womtool/src/test/scala/womtool/WomtoolValidateSpec.scala index fe2d9900b4b..2bc2f45e101 100644 --- a/womtool/src/test/scala/womtool/WomtoolValidateSpec.scala +++ b/womtool/src/test/scala/womtool/WomtoolValidateSpec.scala @@ -16,8 +16,8 @@ import scala.collection.immutable class WomtoolValidateSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { private val presentWorkingDirectoryName = DefaultPathBuilder.get(".").toAbsolutePath.name - val validationTestCases = File("womtool/src/test/resources/validate") - val languageVersions = Option(validationTestCases.list).toList.flatten + val validationTestCases: File = File("womtool/src/test/resources/validate") + val languageVersions: List[File] = Option(validationTestCases.list).toList.flatten val knownUngraphableTests = List("task_only") @@ -72,16 +72,16 @@ class WomtoolValidateSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc case SuccessfulTermination(womtoolGraph) => // Check that every call in the WDL is represented in the 'womtool graph' output, and vice versa: - val callsInWdl = (Files.readAllLines(wdl.toPath).asScala.collect { + val callsInWdl = Files.readAllLines(wdl.toPath).asScala.collect { case WdlCallRegex(taskName, null, null, null, null) => taskName case WdlCallRegex(_, _, taskName, null, null) => taskName case WdlCallRegex(_, _, _, _, callAlias) => callAlias - }).toSet + }.toSet - val callsInWomtoolGraph = (womtoolGraph.lines.collect { + val callsInWomtoolGraph = womtoolGraph.linesIterator.collect { case WomtoolGraphCallRegex(call) => call - }).toSet + }.toSet if (!callsInWomtoolGraph.exists(_.startsWith("ScatterAt"))) { withClue(s"In WDL not in Graph: ${callsInWdl -- callsInWomtoolGraph}; In Graph not in WDL: ${callsInWomtoolGraph -- callsInWdl}") { @@ -123,10 +123,10 @@ class WomtoolValidateSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc behavior of "womtool validate with --list-dependencies flag" - val validationWithImportsTests = File("womtool/src/test/resources/validate-with-imports") - val validateWithImportsLanguageVersions = Option(validationWithImportsTests.list).toList.flatten - val userDirectory = sys.props("user.dir") - val workingDirectory = File(sys.env.getOrElse("CROMWELL_BUILD_ROOT_DIRECTORY", userDirectory)).pathAsString + val validationWithImportsTests: File = File("womtool/src/test/resources/validate-with-imports") + val validateWithImportsLanguageVersions: List[File] = Option(validationWithImportsTests.list).toList.flatten + val userDirectory: String = sys.props("user.dir") + val workingDirectory: String = File(sys.env.getOrElse("CROMWELL_BUILD_ROOT_DIRECTORY", userDirectory)).pathAsString it should "test at least one version" in { validateWithImportsLanguageVersions.isEmpty should be(false) @@ -146,6 +146,7 @@ class WomtoolValidateSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc it should s"successfully validate and print the workflow dependencies for $versionName workflow: '$caseName'" in { val rawOutput = expectedOutput(versionDirectory, caseName, "expected_imports.txt") + //noinspection RegExpRedundantEscape val importsExpectation = rawOutput.replaceAll("\\{REPLACE_WITH_ROOT_PATH\\}", workingDirectory) val res = WomtoolMain.runWomtool(Seq("validate", "-l", wdlFile.getAbsolutePath)) @@ -163,6 +164,7 @@ class WomtoolValidateSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc // The filterNot(_.contains(".DS")) stuff prevents Mac 'Desktop Services' hidden directories from accidentally being picked up: private def listFilesAndFilterDSFile(path: Path): immutable.Seq[String] = Option(path.toFile.list).toList.flatten.filterNot(_.contains(".DS")) + //noinspection SameParameterValue private def expectedOutput(versionDirectory: File, caseName: String, outputTextFileName: String): String = File(mustExist(versionDirectory.path.resolve(caseName).resolve(outputTextFileName).toFile).getAbsolutePath).contentAsString.trim }