diff --git a/conf/application.conf b/conf/application.conf index 916f68e68..43c591b35 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -215,6 +215,9 @@ ehri { # How long to cache backend data for... cacheExpiration = 5 minutes + + # Extended timeout for streaming data from the backend... + streamingTimeout = 10 minutes } # THIS ENSURES SECURED ROUTES ARE SECURED. MAKE SURE IT'S EITHER diff --git a/modules/api/app/controllers/api/graphql/GraphQL.scala b/modules/api/app/controllers/api/graphql/GraphQL.scala index 12f269280..456554451 100644 --- a/modules/api/app/controllers/api/graphql/GraphQL.scala +++ b/modules/api/app/controllers/api/graphql/GraphQL.scala @@ -10,6 +10,7 @@ import play.api.mvc.{Action, AnyContent, ControllerComponents, RawBuffer} import services.data.Constants import javax.inject.{Inject, Singleton} +import scala.concurrent.duration.Duration @Singleton @@ -45,6 +46,7 @@ case class GraphQL @Inject()( val serviceConfig = ServiceConfig("ehridata", config) ws.url(s"${serviceConfig.baseUrl}/graphql") .withMethod(HttpVerbs.POST) + .withRequestTimeout(config.get[Duration]("ehri.backend.streamingTimeout")) .addHttpHeaders(serviceConfig.authHeaders: _*) .addHttpHeaders(streamHeader.map(Constants.STREAM_HEADER_NAME -> _).toSeq: _*) .addHttpHeaders(request.userOpt.map(u => Constants.AUTH_HEADER_NAME -> u.id).toSeq: _*) diff --git a/modules/backend/src/main/scala/services/data/DataServiceBuilder.scala b/modules/backend/src/main/scala/services/data/DataServiceBuilder.scala index 9c693aa45..73546069a 100644 --- a/modules/backend/src/main/scala/services/data/DataServiceBuilder.scala +++ b/modules/backend/src/main/scala/services/data/DataServiceBuilder.scala @@ -7,6 +7,7 @@ import play.api.libs.ws.WSResponse import play.api.mvc.Headers import utils._ +import scala.concurrent.duration.Duration import scala.concurrent.{ExecutionContext, Future} /** @@ -66,9 +67,10 @@ trait DataService { * @param urlPart the URL backend path * @param headers the required headers * @param params additional parameters + * @param timeout an optional timeout for the request * @return a web response */ - def stream(urlPart: String, headers: Headers = Headers(), params: Map[String, Seq[String]] = Map.empty): Future[WSResponse] + def stream(urlPart: String, headers: Headers = Headers(), params: Map[String, Seq[String]] = Map.empty, timeout: Option[Duration] = None): Future[WSResponse] /** * Create a new user profile. diff --git a/modules/backend/src/main/scala/services/data/WsDataService.scala b/modules/backend/src/main/scala/services/data/WsDataService.scala index 8376a904a..cecc7eb6c 100644 --- a/modules/backend/src/main/scala/services/data/WsDataService.scala +++ b/modules/backend/src/main/scala/services/data/WsDataService.scala @@ -48,9 +48,11 @@ case class WsDataService(eventHandler: EventHandler, config: Configuration, cach userCall(enc(baseUrl, urlPart) + (if (params.nonEmpty) "?" + utils.http.joinQueryString(params) else "")) .withHeaders(headers.headers: _*).get() - override def stream(urlPart: String, headers: Headers = Headers(), params: Map[String,Seq[String]] = Map.empty): Future[WSResponse] = - userCall(enc(baseUrl, urlPart) + (if(params.nonEmpty) "?" + utils.http.joinQueryString(params) else "")) - .withHeaders(headers.headers: _*).withMethod(HttpVerbs.GET).stream() + override def stream(urlPart: String, headers: Headers = Headers(), params: Map[String,Seq[String]] = Map.empty, timeout: Option[Duration] = None): Future[WSResponse] = { + val request = userCall(enc(baseUrl, urlPart) + (if (params.nonEmpty) "?" + utils.http.joinQueryString(params) else "")) + val timeoutRequest = timeout.fold(request)(t => request.withTimeout(t)) + timeoutRequest.withHeaders(headers.headers: _*).withMethod(HttpVerbs.GET).stream() + } override def createNewUserProfile[T <: WithId : Readable](data: Map[String, String] = Map.empty, groups: Seq[String] = Seq.empty): Future[T] = { userCall(enc(baseUrl, "admin", "create-default-user-profile")) diff --git a/modules/portal/app/controllers/portal/base/PortalController.scala b/modules/portal/app/controllers/portal/base/PortalController.scala index 396caa49c..2577ade8f 100644 --- a/modules/portal/app/controllers/portal/base/PortalController.scala +++ b/modules/portal/app/controllers/portal/base/PortalController.scala @@ -17,12 +17,13 @@ import services.data.{DataServiceBuilder, DataUser} import services.search.{SearchEngine, SearchItemResolver} import utils._ import views.html.MarkdownRenderer -import views.html.errors.{itemNotFound, maintenance, pageNotFound, gone} +import views.html.errors.{gone, itemNotFound, maintenance, pageNotFound} import java.nio.charset.StandardCharsets import java.time.ZonedDateTime import scala.concurrent.Future import scala.concurrent.Future.{successful => immediate} +import scala.concurrent.duration.{Duration, DurationInt} trait PortalController @@ -221,8 +222,10 @@ trait PortalController implicit apiUser: DataUser, request: RequestHeader): Future[Result] = { val fmt: String = format.filter(supportedFormats.contains).getOrElse(supportedFormats.head) val params = request.queryString.filterKeys(_ == "lang") + // since rendering EAD can take a long time, override the default timeout + val timeout: Option[Duration] = config.getOptional[Duration]("ehri.backend.streamingTimeout") userDataApi.stream(s"classes/$entityType/$id/$fmt", params = params, - headers = Headers(HeaderNames.ACCEPT -> "text/xml,application/zip")).map { sr => + headers = Headers(HeaderNames.ACCEPT -> "text/xml,application/zip"), timeout = timeout).map { sr => val ct = sr.headers.get(HeaderNames.CONTENT_TYPE) .flatMap(_.headOption).getOrElse(ContentTypes.XML)