diff --git a/cli/src/main/scala/org/scalablytyped/converter/cli/Main.scala b/cli/src/main/scala/org/scalablytyped/converter/cli/Main.scala index a49eb2a4dd..f4b11ddc8e 100644 --- a/cli/src/main/scala/org/scalablytyped/converter/cli/Main.scala +++ b/cli/src/main/scala/org/scalablytyped/converter/cli/Main.scala @@ -2,6 +2,7 @@ package org.scalablytyped.converter.cli import com.olvind.logging.{stdout, storing, LogLevel, Logger} import fansi.{Attr, Color, Str} +import org.scalablytyped.converter.internal.ImportTypingsUtil import org.scalablytyped.converter.internal.importer._ import org.scalablytyped.converter.internal.importer.build.{BloopCompiler, PublishedSbtProject, SbtProject} import org.scalablytyped.converter.internal.importer.documentation.Npmjs @@ -15,7 +16,7 @@ import org.scalablytyped.converter.internal.{constants, files, sets, BuildInfo, import org.scalablytyped.converter.{Flavour, Selection} import scopt.{OParser, OParserBuilder, Read} -import scala.collection.immutable.SortedSet +import scala.collection.immutable.{SortedMap, SortedSet} import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.Duration @@ -66,8 +67,8 @@ object Main { includeProject = false, ) - val parseCachePath = Some(files.existing(constants.defaultCacheFolder / 'parse).toNIO) - val t0 = System.currentTimeMillis + val parseCacheDirOpt = Some(files.existing(constants.defaultCacheFolder / 'parse).toNIO) + val t0 = System.currentTimeMillis val logger: Logger[(Array[Logger.Stored], Unit)] = storing().zipWith(stdout.filter(LogLevel.warn)) @@ -207,9 +208,6 @@ object Main { val packageJson = Json.force[PackageJson](packageJsonPath) - val projectSource: Option[LibTsSource.FromFolder] = - if (includeProject) Some(LibTsSource.FromFolder(InFolder(inDir), TsIdentLibrary(inDir.last))) else None - val wantedLibs: SortedSet[TsIdentLibrary] = libsFromCmdLine match { case sets.EmptySet() => @@ -221,21 +219,54 @@ object Main { case otherwise => otherwise } - val bootstrapped = Bootstrap.fromNodeModules(InFolder(nodeModulesPath), conversion, wantedLibs) + /* ImportTypings only uses the library names, not the versions */ + val wantedLibsEmptyVersion: SortedMap[TsIdentLibrary, String] = + SortedMap.empty[TsIdentLibrary, String] ++ (wantedLibs.map(k => (k, ""))) - val sources: Vector[LibTsSource] = { - bootstrapped.initialLibs match { - case Left(unresolved) => sys.error(unresolved.msg) - case Right(initial) => projectSource.foldLeft(initial)(_ :+ _) - } - } + val publishLocalFolder = constants.defaultLocalPublishFolder + val fromFolder = InFolder(nodeModulesPath) + val targetFolder = c.paths.out + + val input = ImportTypingsUtil.Input( + converterVersion = BuildInfo.version, + conversion = conversion, + wantedLibs = wantedLibsEmptyVersion, + ) + + val logger: Logger[(Array[Logger.Stored], Unit)] = + storing().zipWith(stdout.filter(LogLevel.warn)) + + val parseCacheDirOpt = Some( + files.existing(constants.defaultCacheFolder / "parse").toNIO, + ) + + val compiler = Await.result( + BloopCompiler(logger.filter(LogLevel.debug).void, conversion.versions, failureCacheFolderOpt = None), + Duration.Inf, + ) + + val ensureSourceFilesWritten = true + + val (results, successes, failures) = ImportTypingsUtil.get( + input, + logger.void, + parseCacheDirOpt, + publishLocalFolder, + fromFolder, + targetFolder, + compiler, + includeProject, + ensureSourceFilesWritten, + ) println( table(fansi.Color.LightBlue)( "directory" -> inDir.toString, "includeDev" -> includeDev.toString, "includeProject" -> includeProject.toString, - "wantedLibs" -> sources.map(s => s.libName.value).mkString(", "), + // TODO how to resolve this? where to put this table printing? + // Bootstrapped is inside ImportTypingsUtil.get and includeDev, for example, is not. +// "wantedLibs" -> sources.map(s => s.libName.value).mkString(", "), "useScalaJsDomTypes" -> conversion.useScalaJsDomTypes.toString, "flavour" -> conversion.flavour.toString, "outputPackage" -> conversion.outputPackage.unescaped, @@ -250,67 +281,9 @@ object Main { ), ) - val compiler = Await.result( - BloopCompiler(logger.filter(LogLevel.debug).void, conversion.versions, failureCacheFolderOpt = None), - Duration.Inf, - ) - - val Pipeline: RecPhase[LibTsSource, PublishedSbtProject] = - RecPhase[LibTsSource] - .next( - new Phase1ReadTypescript( - calculateLibraryVersion = PackageJsonOnly, - resolve = bootstrapped.libraryResolver, - ignored = conversion.ignoredLibs, - ignoredModulePrefixes = conversion.ignoredModulePrefixes, - pedantic = false, - parser = PersistingParser(parseCachePath, bootstrapped.inputFolders, logger.void), - expandTypeMappings = conversion.expandTypeMappings, - ), - "typescript", - ) - .next( - new Phase2ToScalaJs( - pedantic = false, - scalaVersion = conversion.versions.scala, - enableScalaJsDefined = conversion.enableScalaJsDefined, - outputPkg = conversion.outputPackage, - flavour = conversion.flavourImpl, - useDeprecatedModuleNames = conversion.useDeprecatedModuleNames, - ), - "scala.js", - ) - .next( - new PhaseFlavour(conversion.flavourImpl, maybePrivateWithin = conversion.privateWithin), - conversion.flavourImpl.toString, - ) - .next( - new Phase3Compile( - versions = conversion.versions, - compiler = compiler, - targetFolder = c.paths.out, - organization = conversion.organization, - publishLocalFolder = constants.defaultLocalPublishFolder, - metadataFetcher = Npmjs.No, - softWrites = true, - flavour = conversion.flavourImpl, - generateScalaJsBundlerFile = false, - ensureSourceFilesWritten = true, - ), - "build", - ) - - val results: Map[LibTsSource, PhaseRes[LibTsSource, PublishedSbtProject]] = - sources - .map(source => source -> PhaseRunner(Pipeline, (_: LibTsSource) => logger.void, NoListener)(source)) - .toMap - val td = System.currentTimeMillis - t0 logger.warn(td) - val failures: Map[LibTsSource, Either[Throwable, String]] = - results.collect { case (_, PhaseRes.Failure(errors)) => errors }.reduceOption(_ ++ _).getOrElse(Map.empty) - if (failures.nonEmpty) { println( Color.Red(s"Failure: You might try --ignoredLibs ${failures.keys.map(_.libName.value).mkString(", ")}"), @@ -326,13 +299,6 @@ object Main { 1 } else { - val allSuccesses: Map[LibTsSource, PublishedSbtProject] = { - def go(source: LibTsSource, p: PublishedSbtProject): Map[LibTsSource, PublishedSbtProject] = - Map(source -> p) ++ p.project.deps.flatMap { case (k, v) => go(k, v) } - - results.collect { case (s, PhaseRes.Ok(res)) => go(s, res) }.reduceOption(_ ++ _).getOrElse(Map.empty) - } - val short: Seq[SbtProject] = results .collect { case (_, PhaseRes.Ok(res)) => res.project } @@ -341,7 +307,7 @@ object Main { .sortBy(_.name) println() - println(s"Successfully converted ${allSuccesses.keys.map(x => Color.Green(x.libName.value)).mkString(", ")}") + println(s"Successfully converted ${successes.keys.map(x => Color.Green(x.libName.value)).mkString(", ")}") println("To use in sbt: ") println(Color.DarkGray(s"""libraryDependencies ++= Seq( | ${short.map(p => p.reference.asSbt).mkString("", ",\n ", "")} diff --git a/importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ImportTypingsUtil.scala b/importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ImportTypingsUtil.scala new file mode 100644 index 0000000000..c954894c32 --- /dev/null +++ b/importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ImportTypingsUtil.scala @@ -0,0 +1,137 @@ +package org.scalablytyped.converter +package internal + +import java.nio.file.Path + +import com.olvind.logging.Logger +import io.circe013.{Decoder, Encoder} +import org.scalablytyped.converter.internal.importer._ +import org.scalablytyped.converter.internal.importer.build.{Compiler, IvyLayout, PublishedSbtProject} +import org.scalablytyped.converter.internal.importer.documentation.Npmjs +import org.scalablytyped.converter.internal.maps._ +import org.scalablytyped.converter.internal.phases.{PhaseListener, PhaseRes, PhaseRunner, RecPhase} +import org.scalablytyped.converter.internal.scalajs.Dep +import org.scalablytyped.converter.internal.ts._ +import os.RelPath + +import scala.collection.immutable.SortedMap + +object ImportTypingsUtil { + case class Input( + converterVersion: String, + conversion: ConversionOptions, + wantedLibs: SortedMap[TsIdentLibrary, String], + ) { + lazy val packageJsonHash: String = + Digest.of(IArray.fromTraversable(wantedLibs.map { case (name, v) => s"${name.value} $v" })).hexString + } + + object Input { + implicit val encodes: Encoder[Input] = io.circe013.generic.semiauto.deriveEncoder + implicit val decodes: Decoder[Input] = io.circe013.generic.semiauto.deriveDecoder + } + + type Results = Map[LibTsSource, PhaseRes[LibTsSource, PublishedSbtProject]] + type Successes = Map[LibTsSource, PublishedSbtProject] + type Failures = Map[LibTsSource, Either[Throwable, String]] + + def get( + input: Input, + logger: Logger[Unit], + parseCacheDirOpt: Option[Path], + publishLocalFolder: os.Path, + fromFolder: InFolder, + targetFolder: os.Path, + compiler: Compiler, + includeProject: Boolean = false, + ensureSourceFilesWritten: Boolean = false, + ): (Results, Successes, Failures) = { + if (input.conversion.expandTypeMappings =/= EnabledTypeMappingExpansion.DefaultSelection) { + logger.warn("Changing stInternalExpandTypeMappings not encouraged. It might blow up") + } + + if (input.conversion.enableLongApplyMethod) { + logger.warn("enableLongApplyMethod is deprecated and untested. You should migrate to the builder pattern.") + } + + val bootstrapped = Bootstrap.fromNodeModules(fromFolder, input.conversion, input.wantedLibs.keySet) + + val inDir = fromFolder.path / os.up + + val projectSource: Option[LibTsSource.FromFolder] = + if (includeProject) Some(LibTsSource.FromFolder(InFolder(inDir), TsIdentLibrary(inDir.last))) else None + + val sources: Vector[LibTsSource] = { + bootstrapped.initialLibs match { + case Left(unresolved) => sys.error(unresolved.msg) + case Right(initial) => projectSource.foldLeft(initial)(_ :+ _) + } + } + + logger.info(s"Importing ${sources.map(_.libName.value).mkString(", ")}") + + val cachedParser = PersistingParser(parseCacheDirOpt, bootstrapped.inputFolders, logger) + + val Phases: RecPhase[LibTsSource, PublishedSbtProject] = RecPhase[LibTsSource] + .next( + new Phase1ReadTypescript( + resolve = bootstrapped.libraryResolver, + calculateLibraryVersion = CalculateLibraryVersion.PackageJsonOnly, + ignored = input.conversion.ignoredLibs, + ignoredModulePrefixes = input.conversion.ignoredModulePrefixes, + pedantic = false, + parser = cachedParser, + expandTypeMappings = input.conversion.expandTypeMappings, + ), + "typescript", + ) + .next( + new Phase2ToScalaJs( + pedantic = false, + scalaVersion = input.conversion.versions.scala, + enableScalaJsDefined = input.conversion.enableScalaJsDefined, + outputPkg = input.conversion.outputPackage, + flavour = input.conversion.flavourImpl, + useDeprecatedModuleNames = input.conversion.useDeprecatedModuleNames, + ), + "scala.js", + ) + .next( + new PhaseFlavour(input.conversion.flavourImpl, maybePrivateWithin = input.conversion.privateWithin), + input.conversion.flavourImpl.toString, + ) + .next( + new Phase3Compile( + versions = input.conversion.versions, + compiler = compiler, + targetFolder = targetFolder, + organization = input.conversion.organization, + publishLocalFolder = publishLocalFolder, + metadataFetcher = Npmjs.No, + softWrites = true, + flavour = input.conversion.flavourImpl, + generateScalaJsBundlerFile = false, + ensureSourceFilesWritten = false, + ), + "build", + ) + + val results: Results = + sources + .map(s => (s: LibTsSource) -> PhaseRunner(Phases, (_: LibTsSource) => logger, PhaseListener.NoListener)(s)) + .toMap + .toSorted + + val successes: Successes = { + def go(source: LibTsSource, p: PublishedSbtProject): Successes = + Map(source -> p) ++ p.project.deps.flatMap { case (k, v) => go(k, v) } + + results.collect { case (s, PhaseRes.Ok(res)) => go(s, res) }.reduceOption(_ ++ _).getOrElse(Map.empty) + } + + val failures: Failures = + results.collect { case (_, PhaseRes.Failure(errors)) => errors }.reduceOption(_ ++ _).getOrElse(Map.empty) + + (results, successes, failures) + } +} diff --git a/sbt-converter/src/main/scala/org/scalablytyped/converter/internal/ImportTypings.scala b/sbt-converter/src/main/scala/org/scalablytyped/converter/internal/ImportTypings.scala index e6959eefc8..766cdd96ea 100644 --- a/sbt-converter/src/main/scala/org/scalablytyped/converter/internal/ImportTypings.scala +++ b/sbt-converter/src/main/scala/org/scalablytyped/converter/internal/ImportTypings.scala @@ -6,6 +6,7 @@ import java.nio.file.Path import com.olvind.logging.Logger import io.circe013.{Decoder, Encoder} import org.scalablytyped.converter.internal.importer._ +import org.scalablytyped.converter.internal.ImportTypingsUtil.{Failures, Results, Successes} import org.scalablytyped.converter.internal.importer.build.{Compiler, IvyLayout, PublishedSbtProject} import org.scalablytyped.converter.internal.importer.documentation.Npmjs import org.scalablytyped.converter.internal.maps._ @@ -18,22 +19,9 @@ import sbt.librarymanagement.ModuleID import scala.collection.immutable.SortedMap object ImportTypings { + type Input = ImportTypingsUtil.Input type InOut = (Input, Output) - case class Input( - converterVersion: String, - conversion: ConversionOptions, - wantedLibs: SortedMap[TsIdentLibrary, String], - ) { - lazy val packageJsonHash: String = - Digest.of(IArray.fromTraversable(wantedLibs.map { case (name, v) => s"${name.value} $v" })).hexString - } - - object Input { - implicit val encodes: Encoder[Input] = io.circe013.generic.semiauto.deriveEncoder - implicit val decodes: Decoder[Input] = io.circe013.generic.semiauto.deriveDecoder - } - case class Output(externalDeps: Set[Dep.Concrete], allProjects: IArray[Dep.Concrete]) { val allRelPaths: IArray[RelPath] = allProjects.flatMap(p => IvyLayout.unit(p).all.map { case (k, _) => k }) @@ -55,93 +43,17 @@ object ImportTypings { fromFolder: InFolder, targetFolder: os.Path, compiler: Compiler, - ): Either[Map[LibTsSource, Either[Throwable, String]], Output] = { - - if (input.conversion.expandTypeMappings =/= EnabledTypeMappingExpansion.DefaultSelection) { - logger.warn("Changing stInternalExpandTypeMappings not encouraged. It might blow up") - } - - if (input.conversion.enableLongApplyMethod) { - logger.warn("enableLongApplyMethod is deprecated and untested. You should migrate to the builder pattern.") - } - - val bootstrapped = Bootstrap.fromNodeModules(fromFolder, input.conversion, input.wantedLibs.keySet) - - val initial = bootstrapped.initialLibs match { - case Left(unresolved) => sys.error(unresolved.msg) - case Right(initial) => initial - } - - logger.info(s"Importing ${initial.map(_.libName.value).mkString(", ")}") - - val cachedParser = PersistingParser(parseCacheDirOpt, bootstrapped.inputFolders, logger) - - val Phases: RecPhase[LibTsSource, PublishedSbtProject] = RecPhase[LibTsSource] - .next( - new Phase1ReadTypescript( - resolve = bootstrapped.libraryResolver, - calculateLibraryVersion = CalculateLibraryVersion.PackageJsonOnly, - ignored = input.conversion.ignoredLibs, - ignoredModulePrefixes = input.conversion.ignoredModulePrefixes, - pedantic = false, - parser = cachedParser, - expandTypeMappings = input.conversion.expandTypeMappings, - ), - "typescript", - ) - .next( - new Phase2ToScalaJs( - pedantic = false, - scalaVersion = input.conversion.versions.scala, - enableScalaJsDefined = input.conversion.enableScalaJsDefined, - outputPkg = input.conversion.outputPackage, - flavour = input.conversion.flavourImpl, - useDeprecatedModuleNames = input.conversion.useDeprecatedModuleNames, - ), - "scala.js", - ) - .next( - new PhaseFlavour(input.conversion.flavourImpl, maybePrivateWithin = input.conversion.privateWithin), - input.conversion.flavourImpl.toString, - ) - .next( - new Phase3Compile( - versions = input.conversion.versions, - compiler = compiler, - targetFolder = targetFolder, - organization = input.conversion.organization, - publishLocalFolder = publishLocalFolder, - metadataFetcher = Npmjs.No, - softWrites = true, - flavour = input.conversion.flavourImpl, - generateScalaJsBundlerFile = false, - ensureSourceFilesWritten = false, - ), - "build", - ) - - val results: SortedMap[LibTsSource, PhaseRes[LibTsSource, PublishedSbtProject]] = - initial - .map(s => (s: LibTsSource) -> PhaseRunner(Phases, (_: LibTsSource) => logger, PhaseListener.NoListener)(s)) - .toMap - .toSorted - - val successes: Map[LibTsSource, Dep.Concrete] = { - def go(source: LibTsSource, lib: PublishedSbtProject): Map[LibTsSource, Dep.Concrete] = - Map(source -> lib.project.reference) ++ lib.project.deps.flatMap { case (k, v) => go(k, v) } - - results.collect { case (s, PhaseRes.Ok(res)) => go(s, res) }.reduceOption(_ ++ _).getOrElse(Map.empty) - } + ): Either[Failures, Output] = { - val failures: Map[LibTsSource, Either[Throwable, String]] = - results.collect { case (_, PhaseRes.Failure(errors)) => errors }.reduceOption(_ ++ _).getOrElse(Map.empty) + val (_, successes, failures) = + ImportTypingsUtil.get(input, logger, parseCacheDirOpt, publishLocalFolder, fromFolder, targetFolder, compiler) if (failures.nonEmpty) Left(failures) else Right( Output( input.conversion.flavourImpl.dependencies.map(_.concrete(input.conversion.versions)), - IArray.fromTraversable(successes.values), + IArray.fromTraversable(successes.values.map(_.project.reference)), ), ) } diff --git a/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterExternalNpmPlugin.scala b/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterExternalNpmPlugin.scala index f84151614e..29dcfbd787 100644 --- a/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterExternalNpmPlugin.scala +++ b/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterExternalNpmPlugin.scala @@ -35,7 +35,7 @@ object ScalablyTypedConverterExternalNpmPlugin extends AutoPlugin { val conversion = stConversionOptions.value - val input = ImportTypings.Input( + val input = new ImportTypings.Input( converterVersion = BuildInfo.version, conversion = conversion, wantedLibs = wantedLibs, diff --git a/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterPlugin.scala b/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterPlugin.scala index 4a55461f6c..e32650be4d 100644 --- a/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterPlugin.scala +++ b/sbt-converter/src/main/scala/org/scalablytyped/converter/plugin/ScalablyTypedConverterPlugin.scala @@ -29,7 +29,7 @@ object ScalablyTypedConverterPlugin extends AutoPlugin { val fromFolder = InFolder(os.Path((Compile / npmUpdate / Keys.crossTarget).value / "node_modules")) val targetFolder = os.Path(Keys.streams.value.cacheDirectory) / "sources" - val input = ImportTypings.Input( + val input = new ImportTypings.Input( converterVersion = BuildInfo.version, conversion = conversion, wantedLibs = WantedLibs.setting.value,