Add server to host sources of published Java libraries.

Previously, Sourcegraph was only able to LSIF index Java repositories.
There was no good way to LSIF index a Java library on Maven Central
because they don't have an associated git repository.

This commit adds a new `PackageHub` server that is implemented as a
proxy on top of Maven repositories that serves git repositories that
Sourcegraph can understand. This new server can be used with lsif-java
to enable navigation between repositories and their transitive
Java library dependencies.

.gitignore

@@ -55,3 +55,4 @@ test-report.json
 dump.lsif
 
 ./generated
+/sources

Dockerfile

@@ -0,0 +1,11 @@
+FROM openjdk:8-jdk-alpine
+COPY bin/coursier coursier
+RUN apk add --no-cache git curl \
+    && git config --global user.email "[email protected]" \
+    && git config --global user.name "Your Name" \
+    && git config --global http.postBuffer 1048576000 \
+    && curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src \
+    && chmod +x /src \
+    && /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:0.5.0-12-69905fcb-SNAPSHOT -o /packagehub
+ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
+ENTRYPOINT /packagehub --host 0.0.0.0 --port $PORT --src /src --coursier /coursier --postgres.username=$DB_USER --postgres.password=$DB_PASS --postgres.url=$DB_URL --auto-index-delay=PT1M

Dockerfile.template

@@ -0,0 +1,11 @@
+FROM openjdk:8-jdk-alpine
+COPY bin/coursier coursier
+RUN apk add --no-cache git curl \
+    && git config --global user.email "[email protected]" \
+    && git config --global user.name "Your Name" \
+    && git config --global http.postBuffer 1048576000 \
+    && curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src \
+    && chmod +x /src \
+    && /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:VERSION -o /packagehub
+ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
+ENTRYPOINT /packagehub --host 0.0.0.0 --port $PORT --src /src --coursier /coursier --postgres.username=$DB_USER --postgres.password=$DB_PASS --postgres.url=$DB_URL --auto-index-delay=PT1M

build.sbt

@@ -15,6 +15,7 @@ lazy val V =
     def scala213 = "2.13.4"
     def scala212 = "2.12.12"
     def scalameta = "4.4.8"
+    def testcontainers = "0.39.3"
     def requests = "0.6.5"
   }
 
@@ -195,6 +196,45 @@ lazy val cli = project
   )
   .enablePlugins(NativeImagePlugin, BuildInfoPlugin)
   .dependsOn(lsif)
+commands +=
+  Command.command("dockerfile") { s =>
+    "commitall" :: "reload" :: "packagehub/dockerfileUpdate" :: "publish" :: s
+  }
+
+def commitAll(): Unit = {
+  import scala.sys.process._
+  "git add .".!!
+  "git commit --allow-empty -m WIP".!!
+
+}
+commands +=
+  Command.command("commitall") { s =>
+    commitAll()
+    s
+  }
+
+lazy val packagehub = project
+  .in(file("packagehub"))
+  .settings(
+    moduleName := "packagehub",
+    mainClass.in(Compile) := Some("com.sourcegraph.packagehub.PackageHub"),
+    TaskKey[Unit]("dockerfileUpdate") := {
+      val template = IO.read(file("Dockerfile.template"))
+      IO.write(file("Dockerfile"), template.replace("VERSION", version.value))
+      commitAll()
+    },
+    libraryDependencies ++=
+      List(
+        "com.google.cloud.sql" % "postgres-socket-factory" % "1.2.1",
+        "com.zaxxer" % "HikariCP" % "4.0.3",
+        "org.flywaydb" % "flyway-core" % "7.7.1",
+        "org.postgresql" % "postgresql" % "42.2.14",
+        "org.scalameta" %% "scalameta" % V.scalameta,
+        "com.lihaoyi" %% "cask" % "0.7.8"
+      )
+  )
+  .enablePlugins(AssemblyPlugin)
+  .dependsOn(cli)
 
 commands +=
   Command.command("nativeImageProfiled") { s =>
@@ -273,7 +313,7 @@ lazy val unit = project
       ),
     buildInfoPackage := "tests"
   )
-  .dependsOn(plugin, cli)
+  .dependsOn(plugin, cli, packagehub)
   .enablePlugins(BuildInfoPlugin)
 
 lazy val buildTools = project
@@ -344,6 +384,8 @@ lazy val testSettings = List(
   libraryDependencies ++=
     List(
       "org.scalameta" %% "munit" % "0.7.23",
+      "com.dimafeng" %% "testcontainers-scala-munit" % V.testcontainers,
+      "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testcontainers,
       "org.scalameta" %% "moped-testkit" % V.moped,
       "org.scalameta" %% "scalameta" % V.scalameta,
       "io.get-coursier" %% "coursier" % V.coursier,

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala

@@ -167,6 +167,7 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
   /** Recursively collects all Java files in the working directory */
   private def collectAllJavaFiles(dir: Path): List[Path] = {
     val javaPattern = FileSystems.getDefault.getPathMatcher("glob:**.java")
+    val moduleInfo = Paths.get("module-info.java")
     val buf = ListBuffer.empty[Path]
     Files.walkFileTree(
       dir,
@@ -184,7 +185,7 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
             file: Path,
             attrs: BasicFileAttributes
         ): FileVisitResult = {
-          if (javaPattern.matches(file)) {
+          if (javaPattern.matches(file) && !file.endsWith(moduleInfo)) {
             buf += file
           }
           FileVisitResult.CONTINUE

packagehub/src/main/resources/db/migration/V1__Create_tables.sql

@@ -0,0 +1,7 @@
+CREATE TABLE packages (
+ id VARCHAR(1000) NOT NULL PRIMARY KEY
+);
+
+CREATE TABLE indexed_packages (
+ id VARCHAR(1000) NOT NULL PRIMARY KEY
+);

packagehub/src/main/scala/com/sourcegraph/packagehub/EmptyJsonCodec.scala

@@ -0,0 +1,22 @@
+package com.sourcegraph.packagehub
+
+import moped.json.DecodingContext
+import moped.json.ErrorResult
+import moped.json.JsonCodec
+import moped.json.JsonElement
+import moped.json.JsonString
+import moped.json.Result
+import moped.macros.ClassShape
+import moped.reporters.Diagnostic
+
+/**
+ * Codec that always fails the decoding step.
+ *
+ * Useful for types that cannot be configured from the command-line.
+ */
+class EmptyJsonCodec[T] extends JsonCodec[T] {
+  def decode(context: DecodingContext): Result[T] =
+    ErrorResult(Diagnostic.error(s"not supported: $context"))
+  def encode(value: T): JsonElement = JsonString(value.toString())
+  def shape: ClassShape = ClassShape.empty
+}

packagehub/src/main/scala/com/sourcegraph/packagehub/Package.scala

@@ -0,0 +1,120 @@
+package com.sourcegraph.packagehub
+
+import java.nio.file.Path
+import java.nio.file.Paths
+
+import scala.util.control.NonFatal
+
+import com.sourcegraph.lsif_java.Dependencies
+import coursier.core.Dependency
+import coursier.core.Module
+import coursier.core.ModuleName
+import coursier.core.Organization
+import ujson.Obj
+
+/**
+ * Package represents a published library such as a Java artifact, or the JDK.
+ *
+ * @param id
+ *   unique representation for this package that does not include the forward
+ *   slash character. Can be used as the primary key in a relational database.
+ *   Should ideally be human-readable and be easy to parse.
+ * @param path
+ *   relative URL of this package.
+ */
+sealed abstract class Package(
+    val id: String,
+    val path: String,
+    val version: String
+) {
+  def toJsonRepo: Obj = Obj("Name" -> path, "URI" -> s"/repos/$path")
+  def relativePath: Path = Paths.get(path)
+}
+object Package {
+  def jdk(version: String): JdkPackage = {
+    JdkPackage(version)
+  }
+  def maven(org: String, name: String, version: String): MavenPackage = {
+    MavenPackage(
+      Dependency(
+        Module(Organization(org), ModuleName(name), Map.empty),
+        version
+      )
+    )
+  }
+  def parse(value: String): Package = {
+    value match {
+      case s"jdk:$version" =>
+        JdkPackage(version)
+      case s"maven:$library" =>
+        val Right(dep) = Dependencies.parseDependencyEither(library)
+        MavenPackage(dep)
+    }
+  }
+  def fromPath(path: List[String]): Option[(Package, List[String])] =
+    path match {
+      case "maven" :: org :: name :: version :: requestPath =>
+        Some(Package.maven(org, name, version) -> requestPath)
+      case "jdk" :: version :: requestPath =>
+        Some(Package.jdk(version) -> requestPath)
+      case _ =>
+        None
+    }
+  def fromString(value: String, coursier: String): Either[String, Package] = {
+    value match {
+      case s"jdk:$version" =>
+        val exit = os
+          .proc(coursier, "java-home", "--jvm", version)
+          .call(check = false)
+        if (exit.exitCode == 0)
+          Right(JdkPackage(version))
+        else
+          Left(exit.out.trim())
+      case s"maven:$library" =>
+        Dependencies
+          .parseDependencyEither(library)
+          .flatMap { dep =>
+            try {
+              // Report an error if the dependency can't be resolved.
+              Dependencies.resolveProvidedDeps(dep)
+              Right(MavenPackage(dep))
+            } catch {
+              case NonFatal(e) =>
+                Left(e.getMessage())
+            }
+          }
+      case other =>
+        Left(
+          s"unsupported package '$other'. To fix this problem, use a valid syntax " +
+            s"such as 'maven:ORGANIZATION:ARTIFACT_NAME_VERSION' for Java libraries."
+        )
+    }
+  }
+}
+
+/**
+ * A Java library that is published "Maven style".
+ *
+ * The most widely used Maven package host is "Maven Central"
+ * https://search.maven.org/. Most companies self-host an Artifactory instance
+ * to publish internal libraries and to proxy Maven Central.
+ */
+case class MavenPackage(dep: Dependency)
+    extends Package(
+      s"maven:${dep.module.repr}:${dep.version}",
+      s"maven/${dep.module.organization.value}/${dep.module.name.value}/${dep.version}",
+      dep.version
+    ) {
+  def repr = id.stripPrefix("maven:")
+}
+
+/**
+ * The Java standard library.
+ *
+ * The sources of the Java standard library are typically available under
+ * JAVA_HOME.
+ */
+case class JdkPackage(override val version: String)
+    extends Package(s"jdk:${version}", s"jdk/${version}", version) {
+  def repr = id.stripPrefix("jdk:")
+}