Set an extended timeout to endpoints that might stream a lot of data

e.g. GraphQL and EAD endpoints
EHRI · Aug 15, 2023 · 8974e92 · 8974e92
1 parent 6f6c2eb
commit 8974e92
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 6 deletions.
diff --git a/conf/application.conf b/conf/application.conf
@@ -215,6 +215,9 @@ ehri {
 
         # How long to cache backend data for...
         cacheExpiration = 5 minutes
+
+        # Extended timeout for streaming data from the backend...
+        streamingTimeout = 10 minutes
     }
 
     # THIS ENSURES SECURED ROUTES ARE SECURED. MAKE SURE IT'S EITHER

diff --git a/modules/api/app/controllers/api/graphql/GraphQL.scala b/modules/api/app/controllers/api/graphql/GraphQL.scala
@@ -10,6 +10,7 @@ import play.api.mvc.{Action, AnyContent, ControllerComponents, RawBuffer}
 import services.data.Constants
 
 import javax.inject.{Inject, Singleton}
+import scala.concurrent.duration.Duration
 
 
 @Singleton
@@ -45,6 +46,7 @@ case class GraphQL @Inject()(
     val serviceConfig = ServiceConfig("ehridata", config)
     ws.url(s"${serviceConfig.baseUrl}/graphql")
       .withMethod(HttpVerbs.POST)
+      .withRequestTimeout(config.get[Duration]("ehri.backend.streamingTimeout"))
       .addHttpHeaders(serviceConfig.authHeaders: _*)
       .addHttpHeaders(streamHeader.map(Constants.STREAM_HEADER_NAME -> _).toSeq: _*)
       .addHttpHeaders(request.userOpt.map(u => Constants.AUTH_HEADER_NAME -> u.id).toSeq: _*)

diff --git a/modules/backend/src/main/scala/services/data/DataServiceBuilder.scala b/modules/backend/src/main/scala/services/data/DataServiceBuilder.scala
@@ -7,6 +7,7 @@ import play.api.libs.ws.WSResponse
 import play.api.mvc.Headers
 import utils._
 
+import scala.concurrent.duration.Duration
 import scala.concurrent.{ExecutionContext, Future}
 
 /**
@@ -66,9 +67,10 @@ trait DataService {
     * @param urlPart the URL backend path
     * @param headers the required headers
     * @param params  additional parameters
+    * @param timeout an optional timeout for the request
     * @return a web response
     */
-  def stream(urlPart: String, headers: Headers = Headers(), params: Map[String, Seq[String]] = Map.empty): Future[WSResponse]
+  def stream(urlPart: String, headers: Headers = Headers(), params: Map[String, Seq[String]] = Map.empty, timeout: Option[Duration] = None): Future[WSResponse]
 
   /**
     * Create a new user profile.

diff --git a/modules/backend/src/main/scala/services/data/WsDataService.scala b/modules/backend/src/main/scala/services/data/WsDataService.scala
@@ -48,9 +48,11 @@ case class WsDataService(eventHandler: EventHandler, config: Configuration, cach
     userCall(enc(baseUrl, urlPart) + (if (params.nonEmpty) "?" + utils.http.joinQueryString(params) else ""))
       .withHeaders(headers.headers: _*).get()
 
-  override def stream(urlPart: String, headers: Headers = Headers(), params: Map[String,Seq[String]] = Map.empty): Future[WSResponse] =
-    userCall(enc(baseUrl, urlPart) + (if(params.nonEmpty) "?" + utils.http.joinQueryString(params) else ""))
-      .withHeaders(headers.headers: _*).withMethod(HttpVerbs.GET).stream()
+  override def stream(urlPart: String, headers: Headers = Headers(), params: Map[String,Seq[String]] = Map.empty, timeout: Option[Duration] = None): Future[WSResponse] = {
+    val request = userCall(enc(baseUrl, urlPart) + (if (params.nonEmpty) "?" + utils.http.joinQueryString(params) else ""))
+    val timeoutRequest = timeout.fold(request)(t => request.withTimeout(t))
+    timeoutRequest.withHeaders(headers.headers: _*).withMethod(HttpVerbs.GET).stream()
+  }
 
   override def createNewUserProfile[T <: WithId : Readable](data: Map[String, String] = Map.empty, groups: Seq[String] = Seq.empty): Future[T] = {
     userCall(enc(baseUrl, "admin", "create-default-user-profile"))

diff --git a/modules/portal/app/controllers/portal/base/PortalController.scala b/modules/portal/app/controllers/portal/base/PortalController.scala
@@ -17,12 +17,13 @@ import services.data.{DataServiceBuilder, DataUser}
 import services.search.{SearchEngine, SearchItemResolver}
 import utils._
 import views.html.MarkdownRenderer
-import views.html.errors.{itemNotFound, maintenance, pageNotFound, gone}
+import views.html.errors.{gone, itemNotFound, maintenance, pageNotFound}
 
 import java.nio.charset.StandardCharsets
 import java.time.ZonedDateTime
 import scala.concurrent.Future
 import scala.concurrent.Future.{successful => immediate}
+import scala.concurrent.duration.{Duration, DurationInt}
 
 
 trait PortalController
@@ -221,8 +222,10 @@ trait PortalController
     implicit apiUser: DataUser, request: RequestHeader): Future[Result] = {
     val fmt: String = format.filter(supportedFormats.contains).getOrElse(supportedFormats.head)
     val params = request.queryString.filterKeys(_ == "lang")
+    // since rendering EAD can take a long time, override the default timeout
+    val timeout: Option[Duration] = config.getOptional[Duration]("ehri.backend.streamingTimeout")
     userDataApi.stream(s"classes/$entityType/$id/$fmt", params = params,
-      headers = Headers(HeaderNames.ACCEPT -> "text/xml,application/zip")).map { sr =>
+      headers = Headers(HeaderNames.ACCEPT -> "text/xml,application/zip"), timeout = timeout).map { sr =>
       val ct = sr.headers.get(HeaderNames.CONTENT_TYPE)
         .flatMap(_.headOption).getOrElse(ContentTypes.XML)