Merge pull request #1435 from skingle/main

979 - Extract versionScheme, scaladocUrl, developers from pom files
scalacenter · Aug 19, 2024 · 7a8bec2 · 7a8bec2
2 parents e5012df + 3f83d1c
commit 7a8bec2
Show file tree

Hide file tree

Showing 14 changed files with 170 additions and 125 deletions.
diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala
@@ -7,7 +7,6 @@ import java.time.format.DateTimeFormatter
 import fastparse.P
 import fastparse.Start
 import fastparse._
-import scaladex.core.model.PatchVersion
 import scaladex.core.util.Parsers._
 
 /**
@@ -30,9 +29,13 @@ case class Artifact(
     isNonStandardLib: Boolean,
     platform: Platform,
     language: Language,
-    fullScalaVersion: Option[SemanticVersion]
+    fullScalaVersion: Option[SemanticVersion],
+    scaladocUrl: Option[Url],
+    versionScheme: Option[String],
+    developers: Seq[Contributor]
 ) {
   val binaryVersion: BinaryVersion = BinaryVersion(platform, language)
+  val mavenReference: Artifact.MavenReference = Artifact.MavenReference(groupId.value, artifactId, version.encode)
 
   def isValid: Boolean = binaryVersion.isValid
 
@@ -45,11 +48,15 @@ case class Artifact(
     s"$groupId$sep$artifactName"
   }
 
-  private def artifactHttpPath: String = s"/${projectRef.organization}/${projectRef.repository}/$artifactName"
+  def releaseDateFormat: String = Artifact.dateFormatter.format(releaseDate)
 
-  val mavenReference: Artifact.MavenReference = Artifact.MavenReference(groupId.value, artifactId, version.encode)
+  def httpUrl: String = {
+    val binaryVersionQuery = s"?binaryVersion=${binaryVersion.encode}"
+    s"$artifactHttpPath/$version$binaryVersionQuery"
+  }
 
-  def releaseDateFormat: String = Artifact.dateFormatter.format(releaseDate)
+  def badgeUrl(env: Env, platform: Option[Platform] = None): String =
+    s"${fullHttpUrl(env)}/latest-by-scala-version.svg?platform=${platform.map(_.label).getOrElse(binaryVersion.platform.label)}"
 
   def fullHttpUrl(env: Env): String =
     env match {
@@ -60,13 +67,7 @@ case class Artifact(
         s"http://localhost:8080$artifactHttpPath" // todo: fix locally
     }
 
-  def httpUrl: String = {
-    val binaryVersionQuery = s"?binaryVersion=${binaryVersion.encode}"
-    s"$artifactHttpPath/$version$binaryVersionQuery"
-  }
-
-  def badgeUrl(env: Env, platform: Option[Platform] = None): String =
-    s"${fullHttpUrl(env)}/latest-by-scala-version.svg?platform=${platform.map(_.label).getOrElse(binaryVersion.platform.label)}"
+  private def artifactHttpPath: String = s"/${projectRef.organization}/${projectRef.repository}/$artifactName"
 
   def latestBadgeUrl(env: Env): String =
     s"${fullHttpUrl(env)}/latest.svg"

diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Contributor.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Contributor.scala
@@ -0,0 +1,27 @@
+package scaladex.core.model
+
+/**
+ * Description of a person who has contributed to the project, but
+ * who does not have
+ * commit privileges. Usually, these contributions come in
+ * the form of patches submitted.
+ */
+case class Contributor(
+    name: Option[String],
+    email: Option[String],
+    url: Option[String],
+    organization: Option[String],
+    organizationUrl: Option[String],
+    roles: List[String],
+    /*
+                        The timezone the contributor is in. Typically, this is a number in the range
+                        <a href="http://en.wikipedia.org/wiki/UTC%E2%88%9212:00">-12</a> to <a href="http://en.wikipedia.org/wiki/UTC%2B14:00">+14</a>
+                        or a valid time zone id like "America/Montreal" (UTC-05:00) or "Europe/Paris" (UTC+01:00).
+     */
+    timezone: Option[String],
+    // Properties about the contributor, such as an instant messenger handle.
+    properties: Map[String, String],
+    // Developer
+    // The unique ID of the developer in the SCM.
+    id: Option[String]
+)
diff --git a/modules/core/shared/src/test/scala/scaladex/core/model/ArtifactSelectionTests.scala b/modules/core/shared/src/test/scala/scaladex/core/model/ArtifactSelectionTests.scala
@@ -28,7 +28,10 @@ class ArtifactSelectionTests extends AsyncFunSpec with Matchers {
       isNonStandardLib = false,
       artifactId.binaryVersion.platform,
       artifactId.binaryVersion.language,
-      None
+      None,
+      None,
+      None,
+      Nil
     )
   }
 

diff --git a/modules/core/shared/src/test/scala/scaladex/core/model/ArtifactTests.scala b/modules/core/shared/src/test/scala/scaladex/core/model/ArtifactTests.scala
@@ -204,7 +204,10 @@ class ArtifactTests extends AnyFunSpec with Matchers {
       binaryVersion: BinaryVersion,
       artifactName: Option[Artifact.Name] = None,
       resolver: Option[Resolver] = None,
-      projectRef: Option[Project.Reference] = None
+      projectRef: Option[Project.Reference] = None,
+      developers: Seq[Contributor] = Nil,
+      scaladocUrl: Option[Url] = None,
+      versionScheme: Option[String] = None
   ) = {
     // An artifact always have an artifactId that can be parsed, but in the case we don't really care about if it can
     // be parsed or not, we just want to test methods in artifacts like sbtInstall
@@ -224,7 +227,10 @@ class ArtifactTests extends AnyFunSpec with Matchers {
       resolver = resolver,
       licenses = Set(),
       isNonStandardLib = false,
-      fullScalaVersion = None
+      fullScalaVersion = None,
+      developers = developers,
+      scaladocUrl = scaladocUrl,
+      versionScheme = versionScheme
     )
   }
 }
diff --git a/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala b/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala
@@ -9,6 +9,7 @@ import scaladex.core.model.ArtifactDependency
 import scaladex.core.model.ArtifactDependency.Scope
 import scaladex.core.model.BinaryVersion
 import scaladex.core.model.Category
+import scaladex.core.model.Contributor
 import scaladex.core.model.GithubCommitActivity
 import scaladex.core.model.GithubContributor
 import scaladex.core.model.GithubInfo
@@ -48,6 +49,12 @@ object Values {
   val `_sjs0.6_2.13` = BinaryVersion(ScalaJs.`0.6`, Scala.`2.13`)
   val `_native0.4_2.13` = BinaryVersion(ScalaNative.`0.4`, Scala.`2.13`)
 
+  private def contributor(login: String): GithubContributor =
+    GithubContributor(login, "", Url(""), 1)
+
+  private def developer(name: String, url: String, id: String) =
+    Contributor(Some(name), None, Some(url), None, None, List(), None, Map(), Some(id))
+
   object Scalafix {
     val reference: Project.Reference = Project.Reference.from("scalacenter", "scalafix")
     val creationDate: Instant = Instant.ofEpochMilli(1475505237265L)
@@ -65,7 +72,10 @@ object Values {
       resolver = None,
       licenses = Set(),
       isNonStandardLib = false,
-      fullScalaVersion = None
+      fullScalaVersion = None,
+      scaladocUrl = None,
+      versionScheme = None,
+      developers = Nil
     )
     val githubInfo: GithubInfo =
       GithubInfo.empty
@@ -122,7 +132,10 @@ object Values {
       resolver = None,
       licenses = Set(),
       isNonStandardLib = false,
-      fullScalaVersion = None
+      fullScalaVersion = None,
+      developers = Nil,
+      scaladocUrl = None,
+      versionScheme = None
     )
     val dependency: ArtifactDependency =
       ArtifactDependency(
@@ -159,38 +172,15 @@ object Values {
     )
     val groupId: GroupId = GroupId("org.typelevel")
     val license: License = License("MIT License", "MIT", Some("https://spdx.org/licenses/MIT.html"))
-
-    private def getArtifact(
-        name: String,
-        binaryVersion: BinaryVersion,
-        version: SemanticVersion,
-        description: Option[String] = None,
-        fullScalaVersion: Option[SemanticVersion] = None
-    ): Artifact = {
-      val artifactId = ArtifactId(Name(name), binaryVersion)
-      Artifact(
-        groupId = groupId,
-        artifactId = artifactId.value,
-        version = version,
-        artifactName = artifactId.name,
-        platform = binaryVersion.platform,
-        language = binaryVersion.language,
-        projectRef = reference,
-        description = description,
-        releaseDate = Instant.ofEpochMilli(1620911032000L),
-        resolver = None,
-        licenses = Set(license),
-        isNonStandardLib = false,
-        fullScalaVersion = fullScalaVersion
-      )
-    }
-
     val `core_3:2.6.1`: Artifact = getArtifact(
       "cats-core",
       `_3`,
       `2.6.1`,
       description = Some("Cats core"),
-      fullScalaVersion = SemanticVersion.parse("3.0.0")
+      fullScalaVersion = SemanticVersion.parse("3.0.0"),
+      scaladocUrl = Some(Url("http://typelevel.org/cats/api/")),
+      versionScheme = Some("semver-spec"),
+      developers = developers("org.typelevel:cats-core_3:jar:2.6.1")
     )
     val `core_2.13:2.6.1`: Artifact = getArtifact("cats-core", `_2.13`, `2.6.1`, description = Some("Cats core"))
     val `core_3:4`: Artifact = getArtifact("cats-core", `_3`, `4`, description = Some("Cats core"))
@@ -199,19 +189,27 @@ object Values {
       `_3`,
       `2.7.0`,
       description = Some("Cats core"),
-      fullScalaVersion = SemanticVersion.parse("3.0.2")
+      fullScalaVersion = SemanticVersion.parse("3.0.2"),
+      scaladocUrl = Some(Url("http://typelevel.org/cats/api/")),
+      versionScheme = Some("semver-spec"),
+      developers = developers("org.typelevel:cats-core_3:jar:2.7.0")
+    )
+    val `core_sjs1_3:2.6.1`: Artifact = getArtifact(
+      "cats-core",
+      `_sjs1_3`,
+      `2.6.1`,
+      description = Some("Cats core"),
+      scaladocUrl = Some(Url("http://typelevel.org/cats/api/")),
+      versionScheme = Some("semver-spec"),
+      developers = developers("org.typelevel:cats-core_sjs1_3:jar:2.6.1")
     )
-
-    val `core_sjs1_3:2.6.1`: Artifact = getArtifact("cats-core", `_sjs1_3`, `2.6.1`, description = Some("Cats core"))
     val `core_sjs06_2.13:2.6.1`: Artifact =
       getArtifact("cats-core", `_sjs0.6_2.13`, `2.6.1`, description = Some("Cats core"))
     val `core_native04_2.13:2.6.1`: Artifact =
       getArtifact("cats-core", `_native0.4_2.13`, `2.6.1`, description = Some("Cats core"))
-
     val `kernel_2.13`: Artifact = getArtifact("cats-kernel", `_2.13`, `2.6.1`)
     val `kernel_3:2.6.1`: Artifact = getArtifact("cats-kernel", `_3`, `2.6.1`)
     val `laws_3:2.6.1`: Artifact = getArtifact("cats-laws", `_3`, `2.6.1`)
-
     val allArtifacts: Seq[Artifact] =
       Seq(
         `core_3:2.6.1`,
@@ -222,7 +220,6 @@ object Values {
         `kernel_3:2.6.1`,
         `laws_3:2.6.1`
       )
-
     val dependencies: Seq[ArtifactDependency] = Seq(
       ArtifactDependency(
         source = `core_3:2.6.1`.mavenReference,
@@ -253,6 +250,55 @@ object Values {
         1,
         Seq.empty
       )
+
+    def developers(id: String): Seq[Contributor] = Seq(
+      developer("Cody Allen", "https://github.com/ceedubs/", id),
+      developer("Ross Baker", "https://github.com/rossabaker/", id),
+      developer("P. Oscar Boykin", "https://github.com/johnynek/", id),
+      developer("Travis Brown", "https://github.com/travisbrown/", id),
+      developer("Adelbert Chang", "https://github.com/adelbertc/", id),
+      developer("Peter Neyens", "https://github.com/peterneyens/", id),
+      developer("Rob Norris", "https://github.com/tpolecat/", id),
+      developer("Erik Osheim", "https://github.com/non/", id),
+      developer("LukaJCB", "https://github.com/LukaJCB/", id),
+      developer("Michael Pilquist", "https://github.com/mpilquist/", id),
+      developer("Miles Sabin", "https://github.com/milessabin/", id),
+      developer("Daniel Spiewak", "https://github.com/djspiewak/", id),
+      developer("Frank Thomas", "https://github.com/fthomas/", id),
+      developer("Julien Truffaut", "https://github.com/julien-truffaut/", id),
+      developer("Kailuo Wang", "https://github.com/kailuowang/", id)
+    )
+
+    private def getArtifact(
+        name: String,
+        binaryVersion: BinaryVersion,
+        version: SemanticVersion,
+        description: Option[String] = None,
+        fullScalaVersion: Option[SemanticVersion] = None,
+        developers: Seq[Contributor] = Nil,
+        scaladocUrl: Option[Url] = None,
+        versionScheme: Option[String] = None
+    ): Artifact = {
+      val artifactId = ArtifactId(Name(name), binaryVersion)
+      Artifact(
+        groupId = groupId,
+        artifactId = artifactId.value,
+        version = version,
+        artifactName = artifactId.name,
+        platform = binaryVersion.platform,
+        language = binaryVersion.language,
+        projectRef = reference,
+        description = description,
+        releaseDate = Instant.ofEpochMilli(1620911032000L),
+        resolver = None,
+        licenses = Set(license),
+        isNonStandardLib = false,
+        fullScalaVersion = fullScalaVersion,
+        developers = developers,
+        scaladocUrl = scaladocUrl,
+        versionScheme = versionScheme
+      )
+    }
   }
 
   object CatsEffect {
@@ -289,7 +335,4 @@ object Values {
     val reference: Project.Reference = Project.Reference.from("scala/scala3")
   }
 
-  private def contributor(login: String): GithubContributor =
-    GithubContributor(login, "", Url(""), 1)
-
 }
diff --git a/modules/data/src/main/scala/scaladex/data/maven/ArtifactModel.scala b/modules/data/src/main/scala/scaladex/data/maven/ArtifactModel.scala
@@ -2,6 +2,8 @@ package scaladex.data
 package maven
 
 import scaladex.core.model.Artifact
+import scaladex.core.model.Contributor
+import scaladex.core.model.Url
 
 /** Abstract model of a released artifact. Initially modeled after the POM model. Tweaked to fit with ivy.xml descriptors */
 // POM Model
@@ -30,7 +32,9 @@ case class ArtifactModel(
     repositories: List[Repository] = Nil,
     organization: Option[Organization] = None,
     sbtPluginTarget: Option[SbtPluginTarget] =
-      None // Information on the target scala and sbt versions, in case this artifact is an sbt plugin
+      None, // Information on the target scala and sbt versions, in case this artifact is an sbt plugin
+    scaladocUrl: Option[Url] = None,
+    versionScheme: Option[String] = None
 ) {
   private val packagingOfInterest = Set("aar", "jar", "bundle", "pom")
   val isPackagingOfInterest: Boolean = packagingOfInterest.contains(packaging)
@@ -67,30 +71,6 @@ case class License(
     comments: Option[String] = None
 )
 
-/**
- * Description of a person who has contributed to the project, but who does not
- *    have commit privileges. Usually, these contributions come in the form of patches submitted.
- */
-case class Contributor(
-    name: Option[String],
-    email: Option[String],
-    url: Option[String],
-    organization: Option[String],
-    organizationUrl: Option[String],
-    roles: List[String],
-    /*
-    The timezone the contributor is in. Typically, this is a number in the range
-    <a href="http://en.wikipedia.org/wiki/UTC%E2%88%9212:00">-12</a> to <a href="http://en.wikipedia.org/wiki/UTC%2B14:00">+14</a>
-    or a valid time zone id like "America/Montreal" (UTC-05:00) or "Europe/Paris" (UTC+01:00).
-     */
-    timezone: Option[String],
-    // Properties about the contributor, such as an instant messenger handle.
-    properties: Map[String, String],
-    // Developer
-    // The unique ID of the developer in the SCM.
-    id: Option[String]
-)
-
 case class Dependency(
     groupId: String, // org.apache.maven
     artifactId: String, // maven-artifact

diff --git a/modules/data/src/main/scala/scaladex/data/maven/PomConvert.scala b/modules/data/src/main/scala/scaladex/data/maven/PomConvert.scala
@@ -1,6 +1,9 @@
 package scaladex.data
 package maven
 
+import scaladex.core.model.Contributor
+import scaladex.core.model.Url
+
 private[maven] object PomConvert {
   def apply(model: org.apache.maven.model.Model): ArtifactModel = {
     import model._
@@ -125,7 +128,9 @@ private[maven] object PomConvert {
           scalaVersion <- properties.get("scalaVersion")
           sbtVersion <- properties.get("sbtVersion")
         } yield SbtPluginTarget(scalaVersion, sbtVersion)
-      }
+      },
+      Option(getProperties).flatMap(_.asScala.toMap.get("info.apiURL")).map(Url),
+      Option(getProperties).flatMap(_.asScala.toMap.get("info.versionScheme"))
     )
   }
 }
diff --git a/modules/infra/src/main/resources/migrations/V22__new_artifact_fields.sql b/modules/infra/src/main/resources/migrations/V22__new_artifact_fields.sql
@@ -0,0 +1,4 @@
+ALTER TABLE artifacts
+ADD COLUMN scaladoc_url VARCHAR,
+ADD COLUMN version_scheme VARCHAR,
+ADD COLUMN developers VARCHAR NOT NULL DEFAULT '[]';