Merge pull request #1438 from adpi2/add-is-latest-version

Pre-compute the latest version of each artifact

modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala

@@ -299,4 +299,9 @@ object Artifact {
     def repoUrl: String =
       s"https://repo1.maven.org/maven2/${groupId.replace('.', '/')}/$artifactId/$version/"
   }
+
+  object MavenReference {
+    def apply(groupId: GroupId, artifactId: String, version: SemanticVersion): MavenReference =
+      MavenReference(groupId.value, artifactId, version.encode)
+  }
 }

modules/core/shared/src/main/scala/scaladex/core/model/Project.scala

@@ -93,6 +93,9 @@ object Project {
       settings = settings.getOrElse(Settings.empty)
     )
 
+  def default(ref: Project.Reference, githubStatus: GithubStatus): Project =
+    Project(ref.organization, ref.repository, None, githubStatus, None, Settings.empty)
+
   case class Settings(
       preferStableVersion: Boolean,
       defaultArtifact: Option[Artifact.Name],

modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala

@@ -9,7 +9,7 @@ class ProjectService(database: WebDatabase)(implicit context: ExecutionContext)
   def getProjectHeader(project: Project): Future[Option[ProjectHeader]] = {
     val ref = project.reference
     for {
-      latestArtifacts <- database.getLatestArtifacts(ref, project.settings.preferStableVersion)
+      latestArtifacts <- database.getLatestArtifacts(ref)
       versionCount <- database.countVersions(ref)
     } yield ProjectHeader(
       project.reference,

modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala

@@ -13,6 +13,7 @@ import scaladex.core.model.SemanticVersion
 
 trait SchedulerDatabase extends WebDatabase {
   // project and github
+  def getAllProjects(): Future[Seq[Project]]
   def insertProject(project: Project): Future[Unit]
   def updateProjectCreationDate(ref: Project.Reference, creationDate: Instant): Future[Unit]
   def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]]
@@ -28,6 +29,8 @@ trait SchedulerDatabase extends WebDatabase {
   def updateArtifacts(artifacts: Seq[Artifact], newRef: Project.Reference): Future[Int]
   def updateArtifactReleaseDate(reference: MavenReference, releaseDate: Instant): Future[Int]
   def getAllGroupIds(): Future[Seq[Artifact.GroupId]]
+  def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]]
   def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]]
   def getDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]]
+  def updateLatestVersion(ref: MavenReference): Future[Unit]
 }

modules/core/shared/src/main/scala/scaladex/core/service/WebDatabase.scala

@@ -1,6 +1,5 @@
 package scaladex.core.service
 
-import java.time.Instant
 import java.util.UUID
 
 import scala.concurrent.Future
@@ -9,14 +8,13 @@ import scaladex.core.model._
 
 trait WebDatabase {
   // artifacts
-  // insertArtifact return a boolean. It's true if a new project is inserted, false otherwise
-  def insertArtifact(artifact: Artifact, dependencies: Seq[ArtifactDependency], time: Instant): Future[Boolean]
-  def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]]
+  def insertArtifact(artifact: Artifact): Future[Boolean]
+  def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]]
   def getArtifacts(projectRef: Project.Reference): Future[Seq[Artifact]]
   def getArtifacts(ref: Project.Reference, artifactName: Artifact.Name, preReleases: Boolean): Future[Seq[Artifact]]
   def getArtifacts(ref: Project.Reference, artifactName: Artifact.Name, version: SemanticVersion): Future[Seq[Artifact]]
   def getArtifactsByName(projectRef: Project.Reference, artifactName: Artifact.Name): Future[Seq[Artifact]]
-  def getLatestArtifacts(ref: Project.Reference, preferStableVersions: Boolean): Future[Seq[Artifact]]
+  def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]]
   def getArtifactByMavenReference(mavenRef: Artifact.MavenReference): Future[Option[Artifact]]
   def getAllArtifacts(language: Option[Language], platform: Option[Platform]): Future[Seq[Artifact]]
   def countArtifacts(): Future[Long]
@@ -26,8 +24,8 @@ trait WebDatabase {
   def getReverseDependencies(artifact: Artifact): Future[Seq[ArtifactDependency.Reverse]]
 
   // projects
+  def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean]
   def updateProjectSettings(ref: Project.Reference, settings: Project.Settings): Future[Unit]
-  def getAllProjects(): Future[Seq[Project]]
   def getAllProjectsStatuses(): Future[Map[Project.Reference, GithubStatus]]
   def getProject(projectRef: Project.Reference): Future[Option[Project]]
   def getFormerReferences(projectRef: Project.Reference): Future[Seq[Project.Reference]]

modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

@@ -30,24 +30,26 @@ class InMemoryDatabase extends SchedulerDatabase {
     allDependencies.clear()
   }
 
-  override def insertArtifact(
-      artifact: Artifact,
-      dependencies: Seq[ArtifactDependency],
-      now: Instant
-  ): Future[Boolean] = {
-    val ref = artifact.projectRef
+  override def insertArtifact(artifact: Artifact): Future[Boolean] = {
+    val isNewArtifact = !allArtifacts.values.flatten.exists(a => a.mavenReference == artifact.mavenReference)
+    allArtifacts.addOne(artifact.projectRef -> (allArtifacts.getOrElse(artifact.projectRef, Seq.empty) :+ artifact))
+    Future.successful(isNewArtifact)
+  }
+
+  override def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] = {
     val isNewProject = !allProjects.contains(ref)
-    if (isNewProject) allProjects.addOne(ref -> Project.default(ref, now = now))
-    allArtifacts.addOne(ref -> (allArtifacts.getOrElse(ref, Seq.empty) :+ artifact))
-    dependencies.appendedAll(dependencies)
+    if (isNewProject) allProjects.addOne(ref -> Project.default(ref, status))
     Future.successful(isNewProject)
   }
 
   override def insertProject(project: Project): Future[Unit] = ???
 
   override def insertArtifacts(allArtifacts: Seq[Artifact]): Future[Unit] = ???
 
-  override def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit] = ???
+  override def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit] = {
+    allDependencies ++= dependencies
+    Future.successful(())
+  }
 
   override def deleteProjectDependencies(ref: Project.Reference): Future[Int] = ???
 
@@ -59,10 +61,10 @@ class InMemoryDatabase extends SchedulerDatabase {
   override def getProject(projectRef: Project.Reference): Future[Option[Project]] =
     Future.successful(allProjects.get(projectRef))
 
-  override def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]] =
+  override def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]] =
     Future.successful {
       allArtifacts.values.flatten.filter { artifact: Artifact =>
-        artifact.groupId == groupId && artifact.artifactId == artifactId.value
+        artifact.groupId == groupId && artifact.artifactId == artifactId
       }.toSeq
     }
 
@@ -197,12 +199,18 @@ class InMemoryDatabase extends SchedulerDatabase {
     Future.successful(())
   }
 
-  override def getLatestArtifacts(ref: Project.Reference, preferStableVersion: Boolean): Future[Seq[Artifact]] = {
+  override def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]] = {
     val res = allArtifacts(ref)
       .groupBy(a => (a.groupId, a.artifactId))
       .values
       .map(artifacts => artifacts.maxBy(_.releaseDate))
       .toSeq
     Future.successful(res)
   }
+
+  override def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] =
+    Future.successful(allArtifacts(ref).map(a => (a.groupId, a.artifactId)).toSeq)
+
+  override def updateLatestVersion(ref: Artifact.MavenReference): Future[Unit] =
+    Future.successful(())
 }

modules/core/shared/src/test/scala/scaladex/core/test/Values.scala

@@ -32,6 +32,7 @@ import scaladex.core.model.search.ProjectDocument
 object Values {
   val now: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS)
   val ok: GithubStatus = GithubStatus.Ok(now)
+  val unknown: GithubStatus = GithubStatus.Unknown(now)
 
   val `2.6.1` = PatchVersion(2, 6, 1)
   val `4`: SemanticVersion = MajorVersion(4)

modules/data/src/main/resources/subindex.txt

@@ -128,7 +128,7 @@ foundweekends/giter8
 cb372/scalacache
 blemale/scaffeine
 scala/scala
-lampepfl/dotty
+scala/scala3
 sbt/zinc
 scala-js/scala-js
 scala-native/scala-native

modules/infra/src/main/resources/migrations/V22__new_artifact_fields.sql

@@ -1,4 +1,4 @@
 ALTER TABLE artifacts
-ADD COLUMN scaladoc_url VARCHAR,
-ADD COLUMN version_scheme VARCHAR,
-ADD COLUMN developers VARCHAR NOT NULL DEFAULT '[]';
+ADD COLUMN IF NOT EXISTS scaladoc_url VARCHAR,
+ADD COLUMN IF NOT EXISTS version_scheme VARCHAR,
+ADD COLUMN IF NOT EXISTS developers VARCHAR NOT NULL DEFAULT '[]';

modules/infra/src/main/resources/migrations/V23__add_is_latest_version.sql

@@ -0,0 +1,2 @@
+ALTER TABLE artifacts
+ADD COLUMN IF NOT EXISTS is_latest_version BOOLEAN NOT NULL default 'false';

modules/infra/src/main/scala/scaladex/infra/GithubClientImpl.scala

@@ -143,14 +143,15 @@ class GithubClientImpl(token: Secret)(implicit val system: ActorSystem)
       .flatMap {
         case (headers, entity) =>
           val lastPage = headers.find(_.is("link")).map(_.value()).flatMap(extractLastPage)
+          val contributors = Unmarshal(entity).to[List[GithubModel.Contributor]]
           lastPage match {
             case Some(lastPage) if lastPage > 1 =>
               for {
-                page1 <- Unmarshal(entity).to[List[GithubModel.Contributor]]
+                page1 <- contributors
                 nextPages <- (2 to lastPage).mapSync(getContributionPage).map(_.flatten)
               } yield page1 ++ nextPages
 
-            case _ => Unmarshal(entity).to[List[GithubModel.Contributor]]
+            case _ => contributors
           }
       }
       .fallbackTo(Future.successful(List.empty))
@@ -180,14 +181,15 @@ class GithubClientImpl(token: Secret)(implicit val system: ActorSystem)
       .flatMap {
         case (headers, entity) =>
           val lastPage = headers.find(_.is("link")).map(_.value()).flatMap(extractLastPage)
+          val issues = Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]]
           lastPage match {
             case Some(lastPage) if lastPage > 1 =>
               for {
-                page1 <- Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]]
+                page1 <- issues
                 nextPages <- (2 to lastPage).mapSync(getOpenIssuePage).map(_.flatten)
               } yield page1.flatten ++ nextPages
 
-            case _ => Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]].map(_.flatten)
+            case _ => issues.map(_.flatten)
           }
       }
       .fallbackTo(Future.successful(Seq.empty))

modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

@@ -33,24 +33,11 @@ import scaladex.infra.sql.UserSessionsTable
 
 class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging {
   private val flyway = DoobieUtils.flyway(datasource)
-  def migrate: IO[Unit] = IO {
-    flyway.repair()
-    flyway.migrate()
-  }
+  def migrate: IO[Unit] = IO(flyway.migrate())
   def dropTables: IO[Unit] = IO(flyway.clean())
 
-  override def insertArtifact(
-      artifact: Artifact,
-      dependencies: Seq[ArtifactDependency],
-      time: Instant
-  ): Future[Boolean] = {
-    val unknownStatus = GithubStatus.Unknown(time)
-    for {
-      isNewProject <- insertProjectRef(artifact.projectRef, unknownStatus)
-      _ <- run(ArtifactTable.insertIfNotExist(artifact))
-      _ <- insertDependencies(dependencies)
-    } yield isNewProject
-  }
+  override def insertArtifact(artifact: Artifact): Future[Boolean] =
+    run(ArtifactTable.insertIfNotExist(artifact)).map(_ >= 1)
 
   override def insertArtifacts(artifacts: Seq[Artifact]): Future[Unit] =
     run(ArtifactTable.insertIfNotExist(artifacts)).map(_ => ())
@@ -63,7 +50,7 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   override def updateArtifactReleaseDate(reference: Artifact.MavenReference, releaseDate: Instant): Future[Int] =
     run(ArtifactTable.updateReleaseDate.run((releaseDate, reference)))
 
-  override def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]] =
+  override def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]] =
     run(ArtifactTable.selectArtifactByGroupIdAndArtifactId.to[Seq](groupId, artifactId))
 
   override def getArtifacts(projectRef: Project.Reference): Future[Seq[Artifact]] =
@@ -79,21 +66,8 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   override def getArtifactsByName(ref: Project.Reference, artifactName: Artifact.Name): Future[Seq[Artifact]] =
     run(ArtifactTable.selectArtifactByProjectAndName.to[Seq]((ref, artifactName)))
 
-  override def getLatestArtifacts(ref: Project.Reference, preferStableVersions: Boolean): Future[Seq[Artifact]] = {
-    val latestArtifactsF = run(ArtifactTable.selectLatestArtifacts(stableOnly = false).to[Seq](ref))
-    if (preferStableVersions) {
-      for {
-        latestStableArtifacts <- run(ArtifactTable.selectLatestArtifacts(stableOnly = true).to[Seq](ref))
-        latestArtifacts <- latestArtifactsF
-      } yield
-      // override non-stable version with the latest stable version
-      (latestStableArtifacts ++ latestArtifacts)
-        .groupBy(a => (a.groupId, a.artifactId))
-        .valuesIterator
-        .map(_.head)
-        .toSeq
-    } else latestArtifactsF
-  }
+  override def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]] =
+    run(ArtifactTable.selectLatestArtifacts.to[Seq](ref))
 
   override def getArtifactByMavenReference(mavenRef: Artifact.MavenReference): Future[Option[Artifact]] =
     run(ArtifactTable.selectByMavenReference.option(mavenRef))
@@ -120,7 +94,7 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
     run(ArtifactDependencyTable.insertIfNotExist.updateMany(dependencies)).map(_ => ())
 
   // return true if inserted, false if it already existed
-  private def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] =
+  override def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] =
     run(ProjectTable.insertIfNotExists.run((ref, status))).map(x => x >= 1)
 
   override def getAllProjectsStatuses(): Future[Map[Project.Reference, GithubStatus]] =
@@ -206,6 +180,9 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   override def getAllGroupIds(): Future[Seq[Artifact.GroupId]] =
     run(ArtifactTable.selectGroupIds.to[Seq])
 
+  override def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] =
+    run(ArtifactTable.selectArtifactIds.to[Seq](ref))
+
   override def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]] =
     run(ArtifactTable.selectMavenReference.to[Seq])
 
@@ -231,6 +208,14 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   ): Future[Seq[Artifact]] =
     run(ArtifactTable.selectArtifactByParams(preReleases).to[Seq](ref, artifactName))
 
+  override def updateLatestVersion(ref: Artifact.MavenReference): Future[Unit] = {
+    val transaction = for {
+      _ <- ArtifactTable.setLatestVersion.run(ref)
+      _ <- ArtifactTable.unsetOthersLatestVersion.run(ref)
+    } yield ()
+    run(transaction)
+  }
+
   override def countVersions(ref: Project.Reference): Future[Long] =
     run(ArtifactTable.countVersionsByProject.unique(ref))
 

modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala

@@ -38,6 +38,7 @@ object ArtifactTable {
     )
   // these field are usually excluded when we read artifacts from the artifacts table.
   val versionFields: Seq[String] = Seq("is_semantic", "is_prerelease")
+  val isLatestVersion: String = "is_latest_version"
 
   def insertIfNotExist(artifact: Artifact): ConnectionIO[Int] =
     insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease))
@@ -65,7 +66,7 @@ object ArtifactTable {
     selectRequest(table, fields, where = where)
   }
 
-  val selectArtifactByGroupIdAndArtifactId: Query[(Artifact.GroupId, Artifact.ArtifactId), Artifact] =
+  val selectArtifactByGroupIdAndArtifactId: Query[(Artifact.GroupId, String), Artifact] =
     selectRequest(table, fields, Seq("group_id", "artifact_id"))
 
   val selectArtifactByProject: Query[Project.Reference, Artifact] =
@@ -103,6 +104,9 @@ object ArtifactTable {
   val selectGroupIds: Query0[Artifact.GroupId] =
     selectRequest(table, Seq("DISTINCT group_id"))
 
+  val selectArtifactIds: Query[Project.Reference, (Artifact.GroupId, String)] =
+    selectRequest(table, Seq("DISTINCT group_id", "artifact_id"), keys = projectReferenceFields)
+
   val selectMavenReference: Query0[Artifact.MavenReference] =
     selectRequest(table, Seq("DISTINCT group_id", "artifact_id", "\"version\""))
 
@@ -117,26 +121,16 @@ object ArtifactTable {
       groupBy = projectReferenceFields
     )
 
-  def selectLatestArtifacts(stableOnly: Boolean): Query[Project.Reference, Artifact] =
-    selectRequest1(latestDateTable(stableOnly), fields.map(c => s"a.$c"))
-
-  // the latest release date of all artifact IDs
-  private def latestDateTable(stableOnly: Boolean): String =
-    s"($table a " +
-      s"INNER JOIN (${selectLatestDate(stableOnly).sql}) d " +
-      s"ON a.group_id=d.group_id " +
-      s"AND a.artifact_id=d.artifact_id " +
-      s"AND a.release_date=d.release_date)"
-
-  private def selectLatestDate(
-      stableOnly: Boolean
-  ): Query[Project.Reference, (Artifact.GroupId, String, Instant)] = {
-    val isReleaseFilters = if (stableOnly) Seq("is_semantic='true'", "is_prerelease='false'") else Seq.empty
-    selectRequest1(
+  def selectLatestArtifacts: Query[Project.Reference, Artifact] =
+    selectRequest1(table, fields, where = Seq("organization=?", "repository=?", "is_latest_version=true"))
+
+  def setLatestVersion: Update[Artifact.MavenReference] =
+    updateRequest0(table, set = Seq("is_latest_version=true"), where = Seq("group_id=?", "artifact_id=?", "version=?"))
+
+  def unsetOthersLatestVersion: Update[Artifact.MavenReference] =
+    updateRequest0(
       table,
-      Seq("group_id", "artifact_id", "MAX(release_date) as release_date"),
-      where = Seq("release_date IS NOT NULL") ++ isReleaseFilters ++ Seq("organization=?", "repository=?"),
-      groupBy = Seq("group_id", "artifact_id")
+      set = Seq("is_latest_version=false"),
+      where = Seq("group_id=?", "artifact_id=?", "version<>?")
     )
-  }
 }

modules/infra/src/main/scala/scaladex/infra/sql/DoobieUtils.scala

@@ -77,6 +77,12 @@ object DoobieUtils {
     Update(s"UPDATE $table SET $fieldsStr WHERE $keysStr")
   }
 
+  def updateRequest0[T: Write](table: String, set: Seq[String], where: Seq[String]): Update[T] = {
+    val setStr = set.mkString(", ")
+    val whereStr = where.mkString(" AND ")
+    Update(s"UPDATE $table SET $setStr WHERE $whereStr")
+  }
+
   def selectRequest[A: Read](table: String, fields: Seq[String]): Query0[A] = {
     val fieldsStr = fields.mkString(", ")
     Query0(s"SELECT $fieldsStr FROM $table")