Add API doc

build.sbt

@@ -177,7 +177,8 @@ lazy val server = project
       "org.webjars.npm" % "chartjs-adapter-date-fns" % "3.0.0",
       "org.webjars" % "font-awesome" % "6.5.2",
       "org.webjars" % "jquery" % "3.7.1",
-      "org.webjars.bower" % "select2" % "4.0.13"
+      "org.webjars.bower" % "select2" % "4.0.13",
+      "org.webjars" % "swagger-ui" % "5.17.14"
     ),
     Compile / unmanagedResourceDirectories += (Assets / WebKeys.public).value,
     Compile / resourceGenerators += (Assets / WebKeys.assets).map(Seq(_)),

modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala

@@ -52,14 +52,7 @@ case class Artifact(
     s"${fullHttpUrl(env)}/latest-by-scala-version.svg?platform=${platform.map(_.value).getOrElse(this.platform.value)}"
 
   // TODO move this out
-  def fullHttpUrl(env: Env): String =
-    env match {
-      case Env.Prod => s"https://index.scala-lang.org$artifactHttpPath"
-      case Env.Dev =>
-        s"https://index-dev.scala-lang.org$artifactHttpPath" // todo: fix locally
-      case Env.Local =>
-        s"http://localhost:8080$artifactHttpPath" // todo: fix locally
-    }
+  def fullHttpUrl(env: Env): String = env.rootUrl + artifactHttpPath
 
   private def artifactHttpPath: String = s"/${projectRef.organization}/${projectRef.repository}/$name"
 

modules/core/shared/src/main/scala/scaladex/core/model/Env.scala

@@ -5,16 +5,20 @@ sealed trait Env {
   def isDev: Boolean = false
   def isLocal: Boolean = false
   def isDevOrProd: Boolean = isDev || isProd
+  def rootUrl: String
 }
 object Env {
   case object Local extends Env {
     override def isLocal: Boolean = true
+    override def rootUrl: String = "http://localhost:8080"
   }
   case object Dev extends Env {
     override def isDev: Boolean = true
+    override def rootUrl: String = "https://index-dev.scala-lang.org"
   }
   case object Prod extends Env {
     override def isProd: Boolean = true
+    override def rootUrl: String = "https://index.scala-lang.org"
   }
 
   def from(s: String): Env =

modules/core/shared/src/main/scala/scaladex/core/model/Project.scala

@@ -72,11 +72,14 @@ object Project {
     def from(org: String, repo: String): Reference =
       Reference(Organization(org), Repository(repo))
 
-    def unsafe(string: String): Reference =
-      string.split('/') match {
-        case Array(org, repo) => from(org, repo)
+    def parse(value: String): Option[Reference] =
+      value.split('/') match {
+        case Array(org, repo) => Some(from(org, repo))
+        case _                => None
       }
 
+    def unsafe(value: String): Reference = parse(value).get
+
     implicit val ordering: Ordering[Reference] =
       Ordering.by(ref => (ref.organization.value, ref.repository.value))
   }

modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala

@@ -4,18 +4,12 @@ import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 
 import scaladex.core.model._
-import scaladex.core.model.search.PageParams
 import scaladex.core.model.search.SearchParams
-import scaladex.core.util.ScalaExtensions._
 
 class ProjectService(database: WebDatabase, searchEngine: SearchEngine)(implicit context: ExecutionContext) {
   def getProjects(languages: Seq[Language], platforms: Seq[Platform]): Future[Seq[Project.Reference]] = {
     val searchParams = SearchParams(languages = languages, platforms = platforms)
-    for {
-      firstPage <- searchEngine.find(searchParams, PageParams(0, 10000))
-      p = firstPage.pagination
-      otherPages <- 1.until(p.pageCount).map(PageParams(_, 10000)).mapSync(p => searchEngine.find(searchParams, p))
-    } yield (firstPage +: otherPages).flatMap(_.items).map(_.document.reference)
+    searchEngine.findRefs(searchParams)
   }
 
   def getProject(ref: Project.Reference): Future[Option[Project]] = database.getProject(ref)

modules/core/shared/src/main/scala/scaladex/core/service/SearchEngine.scala

@@ -28,14 +28,6 @@ trait SearchEngine {
   def getMostDependedUpon(limit: Int): Future[Seq[ProjectDocument]]
   def getLatest(limit: Int): Future[Seq[ProjectDocument]]
 
-  // Old Search API
-  def find(
-      query: String,
-      binaryVersion: Option[BinaryVersion],
-      cli: Boolean,
-      page: PageParams
-  ): Future[Page[ProjectDocument]]
-
   // Search Page
   def find(params: SearchParams, page: PageParams): Future[Page[ProjectHit]]
   def autocomplete(params: SearchParams, limit: Int): Future[Seq[ProjectDocument]]
@@ -47,4 +39,15 @@ trait SearchEngine {
   def find(category: Category, params: AwesomeParams, page: PageParams): Future[Page[ProjectDocument]]
   def countByLanguages(category: Category, params: AwesomeParams): Future[Seq[(Language, Int)]]
   def countByPlatforms(category: Category, params: AwesomeParams): Future[Seq[(Platform, Int)]]
+
+  // Old Search API
+  def find(
+      query: String,
+      binaryVersion: Option[BinaryVersion],
+      cli: Boolean,
+      page: PageParams
+  ): Future[Page[ProjectDocument]]
+
+  // API
+  def findRefs(params: SearchParams): Future[Seq[Project.Reference]]
 }

modules/core/shared/src/test/scala/scaladex/core/test/InMemorySearchEngine.scala

@@ -43,15 +43,24 @@ class InMemorySearchEngine extends SearchEngine {
       page: PageParams
   ): Future[Page[ProjectDocument]] = ???
 
-  override def find(params: SearchParams, page: PageParams): Future[Page[ProjectHit]] =
-    Future.successful {
-      val hits = allDocuments.values
-        .filter(doc => (params.languages.toSet -- doc.languages).isEmpty)
-        .filter(doc => (params.platforms.toSet -- doc.platforms).isEmpty)
-        .toSeq
-        .map(ProjectHit(_, Seq.empty))
-      Page(Pagination(1, 1, hits.size), hits)
-    }
+  override def find(params: SearchParams, page: PageParams): Future[Page[ProjectHit]] = {
+    val hits = allDocuments.values
+      .filter(doc => (params.languages.toSet -- doc.languages).isEmpty)
+      .filter(doc => (params.platforms.toSet -- doc.platforms).isEmpty)
+      .toSeq
+      .map(ProjectHit(_, Seq.empty))
+    val res = Page(Pagination(1, 1, hits.size), hits)
+    Future.successful(res)
+  }
+
+  override def findRefs(params: SearchParams): Future[Seq[Project.Reference]] = {
+    val res = allDocuments.values
+      .filter(doc => (params.languages.toSet -- doc.languages).isEmpty)
+      .filter(doc => (params.platforms.toSet -- doc.platforms).isEmpty)
+      .toSeq
+      .map(_.reference)
+    Future.successful(res)
+  }
 
   override def autocomplete(params: SearchParams, limit: Int): Future[Seq[ProjectDocument]] = ???
 

modules/infra/src/main/scala/scaladex/infra/ElasticsearchEngine.scala

@@ -5,10 +5,12 @@ import java.io.Closeable
 import scala.annotation.nowarn
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
+import scala.concurrent.duration._
 
 import com.sksamuel.elastic4s.ElasticClient
 import com.sksamuel.elastic4s.ElasticDsl._
 import com.sksamuel.elastic4s.ElasticProperties
+import com.sksamuel.elastic4s.Hit
 import com.sksamuel.elastic4s.Response
 import com.sksamuel.elastic4s.analysis.Analysis
 import com.sksamuel.elastic4s.http.JavaClient
@@ -173,6 +175,30 @@ class ElasticsearchEngine(esClient: ElasticClient, index: String)(implicit ec: E
     findPage(request, page).map(_.flatMap(toProjectHit))
   }
 
+  override def findRefs(params: SearchParams): Future[Seq[Project.Reference]] = {
+    val request = searchRequest(filteredSearchQuery(params), params.sorting)
+      .sourceInclude("organization", "repository")
+      .limit(10000)
+    scroll(request, 30.seconds).map(_.flatMap(hit => Project.Reference.parse(hit.id)))
+  }
+
+  private def scroll(request: SearchRequest, timeout: FiniteDuration): Future[Seq[Hit]] = {
+    val r0 = request.keepAlive(timeout)
+    val keepAlive = r0.keepAlive.get
+    def recur(resp: Response[SearchResponse]): Future[Seq[Hit]] = {
+      val hits = resp.result.hits.hits.toSeq
+      resp.result.scrollId match {
+        case None => Future.successful(hits)
+        case Some(id) =>
+          for {
+            r <- esClient.execute(searchScroll(id, keepAlive))
+            nextHits <- recur(r)
+          } yield hits ++ nextHits
+      }
+    }
+    esClient.execute(request).flatMap(recur)
+  }
+
   private def findPage(request: SearchRequest, page: PageParams): Future[Page[SearchHit]] = {
     val clamp = if (page.page <= 0) 1 else page.page
     val pagedRequest = request.from(page.size * (clamp - 1)).size(page.size)
@@ -212,7 +238,10 @@ class ElasticsearchEngine(esClient: ElasticClient, index: String)(implicit ec: E
       case _               => scoreSort().order(SortOrder.Desc)
     }
 
-    search(index).query(scoringQuery).sortBy(sortQuery)
+    search(index)
+      .sourceExclude("githubInfo.readme")
+      .query(scoringQuery)
+      .sortBy(sortQuery)
   }
 
   private def extractDocuments(response: Response[SearchResponse]): Seq[ProjectDocument] =

modules/server/src/main/assets/css/partials/_header.scss

@@ -54,19 +54,15 @@
 
         .logo {
             margin-bottom: 15px;
+            margin-left: 10px;
 
             @media screen and (min-width: $screen-md-min) {
                 margin-bottom: 0;
             }
         }
 
-        .awesome {
-          font-family: Caveat;
-          font-size: 21px;
-          padding: 3px 10px;
-        }
-
         .btn-default {
+          padding: 7px 8px;
           background-color: #224951;
           color: white;
           &:hover, &:focus {
@@ -75,8 +71,14 @@
           }
         }
 
+        .awesome {
+          padding: 3px 8px;
+          font-family: Caveat;
+          font-size: 21px;
+        }
+
         .btn {
-          margin-left: 16px;
+          margin-right: 8px;
           min-height: 39px;
           i {
               margin-right: 8px;

modules/server/src/main/resources/lib/swagger-initializer.js

@@ -0,0 +1,18 @@
+window.onload = function() {
+  window.ui = SwaggerUIBundle({
+    urls: [
+      { name: "Scaladex API v1", url: "/api/v1/open-api.json" },
+      { name: "Scaladex API v0", url: "/api/open-api.json" }
+    ],
+    dom_id: '#swagger-ui',
+    deepLinking: true,
+    presets: [
+      SwaggerUIBundle.presets.apis,
+      SwaggerUIStandalonePreset
+    ],
+    plugins: [
+      SwaggerUIBundle.plugins.DownloadUrl
+    ],
+    layout: "StandaloneLayout"
+  });
+};

modules/server/src/main/scala/scaladex/server/route/Assets.scala

@@ -1,5 +1,6 @@
 package scaladex.server.route
 
+import org.apache.pekko.http.scaladsl.model.StatusCodes
 import org.apache.pekko.http.scaladsl.server.Directives._
 import org.apache.pekko.http.scaladsl.server.Route
 
@@ -8,10 +9,12 @@ object Assets {
     pathPrefix("assets") {
       get(
         concat(
-          path("lib" / Remaining)(path => getFromResource("lib/" + path)),
-          path("img" / Remaining)(path => getFromResource("img/" + path)),
-          path("css" / Remaining)(path => getFromResource("css/" + path)),
-          path("js" / Remaining)(path => getFromResource("js/" + path)),
+          pathPrefix("lib" / "swagger-ui")(redirect("/api/doc", StatusCodes.PermanentRedirect)),
+          // be explicit on what we can get to avoid security leak
+          pathPrefix("lib")(getFromResourceDirectory("lib")),
+          pathPrefix("img")(getFromResourceDirectory("img")),
+          pathPrefix("css")(getFromResourceDirectory("css")),
+          pathPrefix("js")(getFromResourceDirectory("js")),
           path("webclient-opt.js")(
             getFromResource("webclient-opt.js")
           ),

modules/server/src/main/scala/scaladex/server/route/api/DocumentationRoutes.scala

@@ -1,25 +1,36 @@
 package scaladex.server.route.api
 
-import endpoints4s.openapi.model.OpenApi
-import endpoints4s.pekkohttp.server
+import endpoints4s.Encoder
 import org.apache.pekko.http.cors.scaladsl.CorsDirectives.cors
-import org.apache.pekko.http.scaladsl.server.Directives.concat
+import org.apache.pekko.http.scaladsl.marshalling.Marshaller
+import org.apache.pekko.http.scaladsl.marshalling.ToEntityMarshaller
+import org.apache.pekko.http.scaladsl.model.MediaTypes
+import org.apache.pekko.http.scaladsl.model.StatusCodes
+import org.apache.pekko.http.scaladsl.server.Directives._
 import org.apache.pekko.http.scaladsl.server.Route
 
 /**
  * Akka-Http routes serving the documentation of the public HTTP API of Scaladex
  */
-object DocumentationRoute extends server.Endpoints with server.JsonEntitiesFromEncodersAndDecoders {
+object DocumentationRoute {
+  implicit def marshallerFromEncoder[T](implicit encoder: Encoder[T, String]): ToEntityMarshaller[T] =
+    Marshaller
+      .stringMarshaller(MediaTypes.`application/json`)
+      .compose(c => encoder.encode(c))
+
   val route: Route = cors() {
-    concat(
-      endpoint(
-        get(path / "api" / "open-api.json"),
-        ok(jsonResponse[OpenApi])
-      ).implementedBy(_ => ApiDocumentation.apiV0),
-      endpoint(
-        get(path / "api" / "v1" / "open-api.json"),
-        ok(jsonResponse[OpenApi])
-      ).implementedBy(_ => ApiDocumentation.apiV1)
-    )
+    get {
+      concat(
+        pathPrefix("api" / "doc")(
+          pathEnd(redirect("/api/doc/", StatusCodes.PermanentRedirect)) ~
+            pathSingleSlash(getFromResource("lib/swagger-ui/index.html")) ~
+            // override default swagger-initializer
+            path("swagger-initializer.js")(getFromResource("lib/swagger-initializer.js")) ~
+            getFromResourceDirectory("lib/swagger-ui")
+        ),
+        path("api" / "open-api.json")(complete(StatusCodes.OK, ApiDocumentation.apiV0)),
+        path("api" / "v1" / "open-api.json")(complete(StatusCodes.OK, ApiDocumentation.apiV1))
+      )
+    }
   }
 }