Skip to content

Commit

Permalink
Scala 2.13 [CROM-6036] (broadinstitute#6724)
Browse files Browse the repository at this point in the history
* wip

* wip

* wip

* wip

* checkpoint

* wip

* wip

* wip

* wip

* wip

* compiles???

* cleanup

* grrr

* fix test

* fix test

* fix bug in choosing default language factory

* fix tests

* sheesh

* oops

* some test fixes, some commenting out

* fix ordering

* forcing with some [force ci]

* temporarily comment out more problematic varargs stuff

* fix test

* why do I have to do [force ci] to get most sub-builds to run?

* wip, varargs broken

* yaass

* checkpoint

* checkpoint [force ci]

* more [force ci]

* restore comment

* grrr

* grrr [force ci]

* cleanup

* debug attempt

* force debug

* grrr

* restructure

* fixy fix?

* cleanup

* oops

* Revert "debug attempt"

This reverts commit 59b4e01.

* Revert "force debug"

This reverts commit 6b1fba2.

* sweepy sweep

* try

* more

* wip

* docs

* Scala 2.13 codegen fixes

* oops

* PR feedback

* PR feedback

* PR feedback

* CI bump

* Poke CI

* Unpoke CI

Co-authored-by: Janet Gainer-Dewar <[email protected]>
  • Loading branch information
mcovarr and jgainerdewar authored Apr 13, 2022
1 parent 0bb86b9 commit a69d12e
Show file tree
Hide file tree
Showing 338 changed files with 1,082 additions and 992 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Cromwell Change Log

## 79 Release Notes

### Scala 2.13

Cromwell is now built with Scala version 2.13. This change should not be noticeable to users but may be of interest to developers of Cromwell backend implementations.

## 75 Release Notes

### New `AwaitingCloudQuota` backend status
Expand Down
2 changes: 1 addition & 1 deletion CromIAM/src/main/scala/cromiam/server/CromIamServer.scala
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ object CromIamServer extends HttpApp with CromIamApiService with SwaggerService

override val routes: Route = allRoutes ~ swaggerUiResourceRoute

override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus, Sam -> samClient.subsystemStatus))
override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus(), Sam -> samClient.subsystemStatus()))

// Override default shutdownsignal which was just "hit return/enter"
override def waitForShutdownSignal(actorSystem: ActorSystem)(implicit executionContext: ExecutionContext): Future[Done] = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import common.validation.Validation._
import cromiam.server.config.CromIamServerConfig._
import net.ceedubs.ficus.Ficus._

import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}

final case class CromIamServerConfig(cromIamConfig: CromIamConfig,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ trait StatusCheckedSubsystem {
* error messages, otherwise OK = false and include the response body
*/
def subsystemStatus()(implicit ec: ExecutionContext): Future[SubsystemStatus] = {
sttp.get(statusUri).send map { x =>
sttp.get(statusUri).send() map { x =>
x.body match {
case Right(_) => SubsystemStatus(true, None)
case Left(errors) => SubsystemStatus(false, Option(List(errors)))
Expand Down
3 changes: 2 additions & 1 deletion CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cromiam.webservice

import akka.event.LoggingAdapter
import akka.http.scaladsl.model.Uri.Query
import akka.http.scaladsl.model._
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server._
Expand Down Expand Up @@ -86,7 +87,7 @@ trait QuerySupport extends RequestSupport {
// DO NOT REMOVE THE NEXT LINE WITHOUT READING THE SCALADOC ON ensureNoLabelOrs
ensureNoLabelOrs(user, labelOrs)

val newQueryBuilder = query.newBuilder
val newQueryBuilder = Query.newBuilder
newQueryBuilder ++= query

val collectionLabels = userCollectionLabels(user, collections)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ object SubmissionSupport {
}

def extractInputAux: Directive1[Map[String, String]] = {
formFieldMap.map(_.filterKeys(_.startsWith(WorkflowInputsAuxPrefix)))
formFieldMap.map(_.view.filterKeys(_.startsWith(WorkflowInputsAuxPrefix)).toMap)
}

// FIXME: Much like CromwellClient see if there are ways of unifying this a bit w/ the mothership
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import org.yaml.snakeyaml.error.YAMLException
import org.yaml.snakeyaml.nodes.MappingNode
import org.yaml.snakeyaml.{Yaml => SnakeYaml}

import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._


class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with SwaggerService with ScalatestRouteTest with Matchers
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ trait BackendLifecycleActor extends Actor {
protected def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse],
onFailure: Throwable => BackendWorkflowLifecycleActorResponse,
andThen: => Unit = ()) = {
val respondTo: ActorRef = sender
val respondTo: ActorRef = sender()
operation onComplete {
case Success(r) =>
respondTo ! r
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ object BackendWorkflowFinalizationActor {
trait BackendWorkflowFinalizationActor extends BackendWorkflowLifecycleActor with ActorLogging {

def receive: Receive = LoggingReceive {
case Finalize => performActionThenRespond(afterAll map { _ => FinalizationSuccess }, onFailure = FinalizationFailed)
case Finalize => performActionThenRespond(afterAll() map { _ => FinalizationSuccess }, onFailure = FinalizationFailed)
}

/**
Expand Down
4 changes: 2 additions & 2 deletions backend/src/main/scala/cromwell/backend/OutputEvaluator.scala
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ object OutputEvaluator {
case Success(Invalid(errors)) => InvalidJobOutputs(errors)
case Failure(exception) => JobOutputsEvaluationException(exception)
}

/*
* Because Cromwell doesn't trust anyone, if custom evaluation is provided,
* still make sure that all the output ports have been filled with values
Expand All @@ -90,7 +90,7 @@ object OutputEvaluator {
case Nil =>
val errorMessagePrefix = "Error applying postMapper in short-circuit output evaluation"
TryUtil.sequenceMap(outputs map { case (k, v) => (k, postMapper(v))}, errorMessagePrefix) match {
case Failure(e) => InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).map(_.toString):_*))
case Failure(e) => InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).toIndexedSeq.map(_.toString):_*))
case Success(postMappedOutputs) => ValidJobOutputs(CallOutputs(postMappedOutputs))
}
case head :: tail => InvalidJobOutputs(NonEmptyList.of(toError(head), tail.map(toError): _*))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class DummyInitializationActor(pipelinesParams: StandardInitializationActorParam
val backendAttributeValidation: RuntimeAttributesValidation[String] = new RuntimeAttributesValidation[String] {
override def key: String = "backend"

override def coercion: Traversable[WomType] = Vector(WomStringType)
override def coercion: Iterable[WomType] = Vector(WomStringType)

override protected def validateValue: PartialFunction[WomValue, ErrorOr[String]] = {
case WomString("Dummy") => "Dummy".validNel
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ trait StandardAsyncExecutionActor
* @param directoryFiles The directories.
* @return The shell scripting.
*/
def directoryScripts(directoryFiles: Traversable[WomUnlistedDirectory]): String =
def directoryScripts(directoryFiles: Iterable[WomUnlistedDirectory]): String =
directoryFiles map directoryScript mkString "\n"

/**
Expand Down Expand Up @@ -284,7 +284,7 @@ trait StandardAsyncExecutionActor
* @param globFiles The globs.
* @return The shell scripting.
*/
def globScripts(globFiles: Traversable[WomGlobFile]): String =
def globScripts(globFiles: Iterable[WomGlobFile]): String =
globFiles map globScript mkString "\n"

/**
Expand Down Expand Up @@ -549,6 +549,7 @@ trait StandardAsyncExecutionActor
CommandSetupSideEffectFile(womValue, alternativeName)
case AsLocalizedAdHocValue(LocalizedAdHocValue(AdHocValue(womValue, alternativeName, _), _)) =>
CommandSetupSideEffectFile(womValue, alternativeName)
case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh")
}

lazy val evaluatedAdHocFiles: ErrorOr[List[AdHocValue]] = {
Expand Down Expand Up @@ -986,7 +987,7 @@ trait StandardAsyncExecutionActor
case InvalidJobOutputs(errors) =>
val exception = new MessageAggregation {
override def exceptionContext: String = "Failed to evaluate job outputs"
override def errorMessages: Traversable[String] = errors.toList
override def errorMessages: Iterable[String] = errors.toList
}
FailedNonRetryableExecutionHandle(exception, kvPairsToSave = None)
case JobOutputsEvaluationException(exception: Exception) if retryEvaluateOutputsAggregated(exception) =>
Expand Down Expand Up @@ -1068,7 +1069,7 @@ trait StandardAsyncExecutionActor
Map(key -> nextKvPair)
}

val kvsFromPreviousAttemptUpd = kvsFromPreviousAttempt.mapValues(kvPair => kvPair.copy(key = kvPair.key.copy(jobKey = nextKvJobKey)))
val kvsFromPreviousAttemptUpd = kvsFromPreviousAttempt.view.mapValues(kvPair => kvPair.copy(key = kvPair.key.copy(jobKey = nextKvJobKey)))

val failedRetryCountKvPair: Map[String, KvPair] =
if (incrementFailedRetryCount) getNextKvPair(FailedRetryCountKey, (previousFailedRetries + 1).toString)
Expand All @@ -1081,7 +1082,7 @@ trait StandardAsyncExecutionActor

val mergedKvs = kvsFromPreviousAttemptUpd ++ kvsForNextAttempt ++ failedRetryCountKvPair ++ memoryMultiplierKvPair

makeKvRequest(mergedKvs.values.map(KvPut).toSeq) map { respSeq =>
makeKvRequest(mergedKvs.toMap.values.map(KvPut).toSeq) map { respSeq =>
val failures = respSeq.filter(_.isInstanceOf[KvFailure])
if (failures.isEmpty) {
respSeq
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor:
// Hash Request
case hashCommand: IoHashCommandWithContext =>
val key = hashCommand.fileHashContext.file
lazy val requester = FileHashRequester(sender, hashCommand.fileHashContext, hashCommand.ioHashCommand)
lazy val requester = FileHashRequester(sender(), hashCommand.fileHashContext, hashCommand.ioHashCommand)
cache.get(key) match {
case FileHashValueNotRequested =>
// The hash is not in the cache and has not been requested. Make the hash request and register this requester
Expand All @@ -48,9 +48,9 @@ class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor:
// hash to become available.
cache.put(key, FileHashValueRequested(requesters = requester :: requesters))
case FileHashSuccess(value) =>
sender ! Tuple2(hashCommand.fileHashContext, IoSuccess(requester.ioCommand, value))
sender() ! Tuple2(hashCommand.fileHashContext, IoSuccess(requester.ioCommand, value))
case FileHashFailure(error) =>
sender ! Tuple2(hashCommand.fileHashContext, IoFailure(requester.ioCommand, new IOException(error)))
sender() ! Tuple2(hashCommand.fileHashContext, IoFailure(requester.ioCommand, new IOException(error)))
}
// Hash Success
case (hashContext: FileHashContext, success @ IoSuccess(_, value: String)) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import scala.util.{Failure, Try}

object RuntimeAttributesDefault {

def workflowOptionsDefault(options: WorkflowOptions, mapping: Map[String, Traversable[WomType]]):
def workflowOptionsDefault(options: WorkflowOptions, mapping: Map[String, Iterable[WomType]]):
Try[Map[String, WomValue]] = {
options.defaultRuntimeOptions flatMap { attrs =>
TryUtil.sequenceMap(attrs collect {
Expand All @@ -32,7 +32,7 @@ object RuntimeAttributesDefault {
*/
def withDefaults(attrs: EvaluatedRuntimeAttributes, defaultsList: List[EvaluatedRuntimeAttributes]): EvaluatedRuntimeAttributes = {
defaultsList.foldLeft(attrs)((acc, default) => {
acc ++ default.filterKeys(!acc.keySet.contains(_))
acc ++ default.view.filterKeys(!acc.keySet.contains(_))
})
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ object RuntimeAttributesValidation {
new RuntimeAttributesValidation[ValidatedType] {
override def key: String = validation.key

override def coercion: Traversable[WomType] = validation.coercion
override def coercion: Iterable[WomType] = validation.coercion

override protected def validateValue: PartialFunction[WomValue, ErrorOr[ValidatedType]] =
validation.validateValuePackagePrivate
Expand All @@ -108,7 +108,7 @@ object RuntimeAttributesValidation {
new RuntimeAttributesValidation[ValidatedType] {
override def key: String = validation.key

override def coercion: Traversable[WomType] = validation.coercion
override def coercion: Iterable[WomType] = validation.coercion

override protected def validateValue: PartialFunction[WomValue, ErrorOr[ValidatedType]] =
validation.validateValuePackagePrivate
Expand All @@ -132,7 +132,7 @@ object RuntimeAttributesValidation {
new OptionalRuntimeAttributesValidation[ValidatedType] {
override def key: String = validation.key

override def coercion: Traversable[WomType] = validation.coercion
override def coercion: Iterable[WomType] = validation.coercion

override protected def validateOption: PartialFunction[WomValue, ErrorOr[ValidatedType]] =
validation.validateValuePackagePrivate
Expand Down Expand Up @@ -162,7 +162,7 @@ object RuntimeAttributesValidation {
val attributeOptions: Map[String, Option[Any]] = validatedRuntimeAttributes.attributes.safeMapValues(unpackOption)

val attributes: Map[String, String] = attributeOptions collect {
case (name, Some(values: Traversable[_])) => (name, values.mkString(","))
case (name, Some(values: Iterable[_])) => (name, values.mkString(","))
case (name, Some(value)) => (name, value.toString)
}

Expand Down Expand Up @@ -271,7 +271,7 @@ trait RuntimeAttributesValidation[ValidatedType] {
*
* @return traversable of wdl types
*/
def coercion: Traversable[WomType]
def coercion: Iterable[WomType]

/**
* Validates the wdl value.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ trait ValidatedRuntimeAttributesBuilder {
/**
* Returns a map of coercions suitable for RuntimeAttributesDefault.workflowOptionsDefault.
*/
final lazy val coercionMap: Map[String, Traversable[WomType]] = {
final lazy val coercionMap: Map[String, Iterable[WomType]] = {
validations.map(validation => validation.key -> validation.coercion).toMap
}

Expand All @@ -64,7 +64,7 @@ trait ValidatedRuntimeAttributesBuilder {
case Invalid(nel) => throw new RuntimeException with MessageAggregation with NoStackTrace {
override def exceptionContext: String = "Runtime attribute validation failed"

override def errorMessages: Traversable[String] = nel.toList
override def errorMessages: Iterable[String] = nel.toList
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ package cromwell.backend.validation.exception
import common.exception.MessageAggregation

case class ValidationAggregatedException(override val exceptionContext: String,
override val errorMessages: Traversable[String]) extends MessageAggregation
override val errorMessages: Iterable[String]) extends MessageAggregation
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite
val valid =
ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue))
valid.isValid should be(result)
valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value))
valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value))
}

forAll(booleanRows) { value =>
Expand All @@ -90,7 +90,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite
val valid =
ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue))
valid.isValid should be(result)
valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value))
valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value))
}

forAll(booleanRows) { value =>
Expand All @@ -109,7 +109,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite
val valid =
ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue))
valid.isValid should be(result)
valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value)))
}

forAll(integerRows) { value =>
Expand All @@ -120,7 +120,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite
val valid =
ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue))
valid.isValid should be(result)
valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value)))
}

forAll(integerRows) { value =>
Expand All @@ -139,7 +139,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite
val valid =
ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue))
valid.isValid should be(result)
valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value)))
}

forAll(integerRows) { value =>
Expand All @@ -150,7 +150,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite
val valid =
ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue))
valid.isValid should be(result)
valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value)))
}

forAll(integerRows) { value =>
Expand All @@ -176,7 +176,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite
val valid =
ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue))
valid.isValid should be(result)
valid.toEither.left.get.toList should contain theSameElementsAs List(
valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List(
"Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]"
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with
val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"))
var warnings = List.empty[Any]
val mockLogger = mock[Logger]
mockLogger.warn(anyString).answers(warnings :+= _)
mockLogger.warn(anyString).answers((warnings :+= _): Any => Unit)
assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes,
includeDockerSupport = false, logger = mockLogger)
warnings should contain theSameElementsAs List("Unrecognized runtime attribute keys: docker")
Expand All @@ -93,7 +93,7 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with
val runtimeAttributes = Map("docker" -> WomInteger(1))
var warnings = List.empty[Any]
val mockLogger = mock[Logger]
mockLogger.warn(anyString).answers(warnings :+= _)
mockLogger.warn(anyString).answers((warnings :+= _): Any => Unit)
assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes,
includeDockerSupport = false, logger = mockLogger)
warnings should contain theSameElementsAs List("Unrecognized runtime attribute keys: docker")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo
|continueOnReturnCode = [0,1,2]
|""".stripMargin))

ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WomArray(WomArrayType(WomIntegerType), Array(WomInteger(0), WomInteger(1), WomInteger(2)))
ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1), WomInteger(2)))
}

"return failure when tries to validate an invalid maxRetries entry" in {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import net.ceedubs.ficus.Ficus._
import org.apache.commons.lang3.exception.ExceptionUtils
import org.threeten.bp.Duration

import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._
import scala.concurrent.ExecutionContext

class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorReporter {
Expand Down
Loading

0 comments on commit a69d12e

Please sign in to comment.