Use trails from s3

common/app/agents/DeeplyReadAgent.scala

@@ -3,25 +3,98 @@ package agents
 import com.gu.contentapi.client.model.v1.{Content, ElementType}
 import com.gu.contentapi.client.utils.CapiModelEnrichment.RenderingFormat
 import common._
+import conf.Configuration
+import conf.switches.Switches.UseTrailsFromS3
+import model.dotcomrendering.Trail
+import services.{FaciaContentConvert, OphanApi, S3Async}
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.control.NonFatal
 import contentapi.ContentApiClient
 import layout.DiscussionSettings
 import model.ContentFormat
-import services.{FaciaContentConvert, OphanApi}
 
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.control.NonFatal
-import model.dotcomrendering.Trail
+object DeeplyReadS3Agent extends S3Async {
+  override lazy val bucket = Configuration.cache.bucket;
+  lazy val stage: String = Configuration.environment.stage.toUpperCase
+}
 
 class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) extends GuLogging {
 
   private val deeplyReadItems = Box[Map[Edition, Seq[Trail]]](Map.empty)
 
+  def getTrailsFromS3(edition: Edition)(implicit ec: ExecutionContext): Future[Unit] = {
+    Future
+      .sequence(Edition.allEditions.map { edition =>
+        DeeplyReadS3Agent
+          .getObjectAsJson[Seq[Trail]](
+            s"${DeeplyReadS3Agent.stage}/deeply-read/${edition.id.toLowerCase()}.json",
+          )
+          .map(trailsList => {
+            edition -> trailsList.take(10)
+          })
+      })
+      .map(trailsList => {
+        val map = trailsList.toMap
+        for {
+          (edition, list) <- map
+        } yield log.debug(s"Deeply Read in ${edition.displayName}, ${list.size} items: ${list.map(_.url).toString()}")
+
+        val mapWithTenItems = map.filter { case (_, list) => list.size == 10 }
+        log.debug(
+          s"Updating the following ${mapWithTenItems.size} editions: ${mapWithTenItems.keys.map(_.id).toList.sorted.toString()}",
+        )
+
+        deeplyReadItems.alter(deeplyReadItems.get() ++ mapWithTenItems)
+      })
+  }
+
   def removeStartingSlash(path: String): String = {
     if (path.startsWith("/")) path.stripPrefix("/") else path
   }
 
-  def refresh()(implicit ec: ExecutionContext): Future[Unit] = {
-    log.debug(s"Deeply Read Agent refresh()")
+  def correctPillar(pillar: String): String = if (pillar == "arts") "culture" else pillar
+
+  def deeplyReadUrlToTrail(content: Content): Option[Trail] = {
+
+    val contentFormat: ContentFormat = ContentFormat(content.design, content.theme, content.display)
+
+    for {
+      webPublicationDate <- content.webPublicationDate
+      fields <- content.fields
+      linkText <- fields.trailText
+      pillar <- content.pillarName
+      headline <- fields.headline
+      shortUrl <- fields.shortUrl
+    } yield Trail(
+      url = content.webUrl,
+      linkText = linkText,
+      showByline = false,
+      byline = fields.byline,
+      masterImage = None,
+      image = fields.thumbnail,
+      carouselImages = Map.empty,
+      ageWarning = None,
+      isLiveBlog = fields.liveBloggingNow.getOrElse(false),
+      pillar = correctPillar(pillar.toLowerCase),
+      designType = content.`type`.toString,
+      format = contentFormat,
+      webPublicationDate = webPublicationDate.toString(),
+      headline = headline,
+      mediaType = None,
+      shortUrl = shortUrl,
+      kickerText = None,
+      starRating = None,
+      avatarUrl = None,
+      branding = None,
+      discussion = DiscussionSettings.fromTrail(FaciaContentConvert.contentToFaciaContent(content)),
+      trailText = content.fields.flatMap(_.trailText),
+      galleryCount =
+        content.elements.map(_.count(el => el.`type` == ElementType.Image && el.relation == "gallery")).filter(_ > 0),
+    )
+  }
+
+  def getTrailsFromCAPI(edition: Edition)(implicit ec: ExecutionContext): Future[Unit] = {
     /*
       We query Ophan for the deeply read URLs and use them to queryCapi
       then use this information to create a sequence of trails that we cache
@@ -84,45 +157,15 @@ class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) ex
       })
   }
 
-  def correctPillar(pillar: String): String = if (pillar == "arts") "culture" else pillar
-
-  def deeplyReadUrlToTrail(content: Content): Option[Trail] = {
-
-    val contentFormat: ContentFormat = ContentFormat(content.design, content.theme, content.display)
-
-    for {
-      webPublicationDate <- content.webPublicationDate
-      fields <- content.fields
-      linkText <- fields.trailText
-      pillar <- content.pillarName
-      headline <- fields.headline
-      shortUrl <- fields.shortUrl
-    } yield Trail(
-      url = content.webUrl,
-      linkText = linkText,
-      showByline = false,
-      byline = fields.byline,
-      masterImage = None,
-      image = fields.thumbnail,
-      carouselImages = Map.empty,
-      ageWarning = None,
-      isLiveBlog = fields.liveBloggingNow.getOrElse(false),
-      pillar = correctPillar(pillar.toLowerCase),
-      designType = content.`type`.toString,
-      format = contentFormat,
-      webPublicationDate = webPublicationDate.toString(),
-      headline = headline,
-      mediaType = None,
-      shortUrl = shortUrl,
-      kickerText = None,
-      starRating = None,
-      avatarUrl = None,
-      branding = None,
-      discussion = DiscussionSettings.fromTrail(FaciaContentConvert.contentToFaciaContent(content)),
-      trailText = content.fields.flatMap(_.trailText),
-      galleryCount =
-        content.elements.map(_.count(el => el.`type` == ElementType.Image && el.relation == "gallery")).filter(_ > 0),
-    )
+  def refresh()(implicit ec: ExecutionContext): Future[Unit] = {
+    if (UseTrailsFromS3.isSwitchedOn) {
+      // TODO revert to log.debug once we switch to S3 permanently
+      log.info(s"Deeply Read Agent refresh() - Using S3")
+      getTrailsFromS3(Edition.defaultEdition)
+    } else {
+      log.info(s"Deeply Read Agent refresh() - Using CAPI")
+      getTrailsFromCAPI(Edition.defaultEdition)
+    }
   }
 
   def getTrails(edition: Edition)(implicit ec: ExecutionContext): Seq[Trail] = {
@@ -132,5 +175,4 @@ class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) ex
     }
     updatedTrails
   }
-
 }

common/app/common/configuration.scala

@@ -705,6 +705,10 @@ class GuardianConfiguration extends GuLogging {
     lazy val host = configuration.getStringProperty("newsletterApi.host")
     lazy val origin = configuration.getStringProperty("newsletterApi.origin")
   }
+
+  object cache {
+    lazy val bucket = configuration.getMandatoryStringProperty("cache.bucket")
+  }
 }
 
 object ManifestData {

common/app/conf/switches/FeatureSwitches.scala

@@ -671,4 +671,15 @@ trait FeatureSwitches {
     exposeClientSide = true,
     highImpact = false,
   )
+
+  val UseTrailsFromS3 = Switch(
+    group = SwitchGroup.Feature,
+    name = "use-trails-from-s3",
+    description = "Use trails from S3",
+    owners = Seq(Owner.withEmail("dotcom.platform@theguardian.com")),
+    sellByDate = never,
+    safeState = Off,
+    exposeClientSide = false,
+    highImpact = false,
+  )
 }

common/app/model/dotcomrendering/Trail.scala

@@ -1,14 +1,21 @@
 package model.dotcomrendering
 
 import com.github.nscala_time.time.Imports.DateTimeZone
-import com.gu.commercial.branding.{Branding, BrandingType, Dimensions, Logo => CommercialLogo}
+import com.gu.commercial.branding.{
+  Branding,
+  BrandingType,
+  Dimensions,
+  Foundation,
+  PaidContent,
+  Sponsored,
+  Logo => CommercialLogo,
+}
 import common.{Edition, LinkTo}
 import implicits.FaciaContentFrontendHelpers.FaciaContentFrontendHelper
 import layout.DiscussionSettings
-import model.dotcomrendering.DotcomRenderingUtils.withoutNull
 import model.{Article, ContentFormat, ImageMedia, Pillar}
 import model.pressed.PressedContent
-import play.api.libs.json.{Json, OWrites, Writes}
+import play.api.libs.json.{JsNull, JsObject, JsResult, JsValue, Json, OFormat, Reads, Writes}
 import play.api.mvc.RequestHeader
 import views.support.{ImageProfile, ImgSrc, Item300, Item460, RemoveOuterParaHtml}
 
@@ -40,50 +47,116 @@ case class Trail(
 
 object Trail {
 
-  implicit val brandingTypeWrites: Writes[BrandingType] = new Writes[BrandingType] {
-    def writes(bt: BrandingType) = {
-      Json.obj(
-        "name" -> bt.name,
-      )
+  implicit val carouselImagesReads: Reads[Map[String, Option[String]]] =
+    Reads[Map[String, Option[String]]] { json =>
+      json.validate[JsObject].map {
+        _.fields.map { case (k, v) => k -> v.asOpt[String] }.toMap
+      }
     }
+
+  implicit val brandingTypeFormat: OFormat[BrandingType] = new OFormat[BrandingType] {
+    def reads(json: JsValue): JsResult[BrandingType] =
+      (json \ "name").validate[String].map {
+        case PaidContent.name => PaidContent
+        case Foundation.name  => Foundation
+        case _                => Sponsored
+      }
+    def writes(bt: BrandingType): JsObject = Json.obj("name" -> bt.name)
   }
 
-  implicit val dimensionsWrites: OWrites[Dimensions] = Json.writes[Dimensions]
-
-  implicit val logoWrites: OWrites[CommercialLogo] = Json.writes[CommercialLogo]
-
-  implicit val brandingWrites: OWrites[Branding] = Json.writes[Branding]
-
-  implicit val discussionWrites: OWrites[DiscussionSettings] = Json.writes[DiscussionSettings]
-
-  implicit val OnwardItemWrites: Writes[Trail] = Writes { trail =>
-    val jsObject = Json.obj(
-      "url" -> trail.url,
-      "linkText" -> trail.linkText,
-      "showByline" -> trail.showByline,
-      "byline" -> trail.byline,
-      "masterImage" -> trail.masterImage,
-      "image" -> trail.image,
-      "carouselImages" -> trail.carouselImages,
-      "ageWarning" -> trail.ageWarning,
-      "isLiveBlog" -> trail.isLiveBlog,
-      "pillar" -> trail.pillar,
-      "designType" -> trail.designType,
-      "format" -> trail.format,
-      "webPublicationDate" -> trail.webPublicationDate,
-      "headline" -> trail.headline,
-      "mediaType" -> trail.mediaType,
-      "shortUrl" -> trail.shortUrl,
-      "kickerText" -> trail.kickerText,
-      "starRating" -> trail.starRating,
-      "avatarUrl" -> trail.avatarUrl,
-      "branding" -> trail.branding,
-      "discussion" -> trail.discussion,
-      "trailText" -> trail.trailText,
-      "galleryCount" -> trail.galleryCount,
-    )
+  implicit val dimensionsFormat: OFormat[Dimensions] = Json.format[Dimensions]
+
+  implicit val logoFormat: OFormat[CommercialLogo] = Json.format[CommercialLogo]
+
+  implicit val brandingFormat: OFormat[Branding] = Json.format[Branding]
 
-    withoutNull(jsObject)
+  implicit val discussionSettingsFormat: OFormat[DiscussionSettings] = Json.format[DiscussionSettings]
+
+  implicit val contentFormatFormat: OFormat[ContentFormat] = new OFormat[ContentFormat] {
+    def reads(json: JsValue): JsResult[ContentFormat] = ContentFormat.contentFormatReads.reads(json)
+    def writes(cf: ContentFormat): JsObject = ContentFormat.contentFormatWrites.writes(cf).as[JsObject]
+  }
+
+  implicit val trailFormat: OFormat[Trail] = new OFormat[Trail] {
+    override def reads(json: JsValue): JsResult[Trail] =
+      for {
+        url <- (json \ "url").validate[String]
+        linkText <- (json \ "linkText").validate[String]
+        showByline <- (json \ "showByline").validate[Boolean]
+        byline <- (json \ "byline").validateOpt[String]
+        masterImage <- (json \ "masterImage").validateOpt[String]
+        image <- (json \ "image").validateOpt[String]
+        carouselImages <- (json \ "carouselImages").validate[Map[String, Option[String]]]
+        ageWarning <- (json \ "ageWarning").validateOpt[String]
+        isLiveBlog <- (json \ "isLiveBlog").validate[Boolean]
+        pillar <- (json \ "pillar").validate[String]
+        designType <- (json \ "designType").validate[String]
+        format <- (json \ "format").validate[ContentFormat]
+        webPublicationDate <- (json \ "webPublicationDate").validate[String]
+        headline <- (json \ "headline").validate[String]
+        mediaType <- (json \ "mediaType").validateOpt[String]
+        shortUrl <- (json \ "shortUrl").validate[String]
+        kickerText <- (json \ "kickerText").validateOpt[String]
+        starRating <- (json \ "starRating").validateOpt[Int]
+        avatarUrl <- (json \ "avatarUrl").validateOpt[String]
+        branding <- (json \ "branding").validateOpt[Branding]
+        discussion <- (json \ "discussion").validate[DiscussionSettings]
+        trailText <- (json \ "trailText").validateOpt[String]
+        galleryCount <- (json \ "galleryCount").validateOpt[Int]
+      } yield Trail(
+        url,
+        linkText,
+        showByline,
+        byline,
+        masterImage,
+        image,
+        carouselImages,
+        ageWarning,
+        isLiveBlog,
+        pillar,
+        designType,
+        format,
+        webPublicationDate,
+        headline,
+        mediaType,
+        shortUrl,
+        kickerText,
+        starRating,
+        avatarUrl,
+        branding,
+        discussion,
+        trailText,
+        galleryCount,
+      )
+
+    override def writes(trail: Trail): JsObject = {
+      val obj = Json.obj(
+        "url" -> trail.url,
+        "linkText" -> trail.linkText,
+        "showByline" -> trail.showByline,
+        "byline" -> trail.byline,
+        "masterImage" -> trail.masterImage,
+        "image" -> trail.image,
+        "carouselImages" -> trail.carouselImages,
+        "ageWarning" -> trail.ageWarning,
+        "isLiveBlog" -> trail.isLiveBlog,
+        "pillar" -> trail.pillar,
+        "designType" -> trail.designType,
+        "format" -> trail.format,
+        "webPublicationDate" -> trail.webPublicationDate,
+        "headline" -> trail.headline,
+        "mediaType" -> trail.mediaType,
+        "shortUrl" -> trail.shortUrl,
+        "kickerText" -> trail.kickerText,
+        "starRating" -> trail.starRating,
+        "avatarUrl" -> trail.avatarUrl,
+        "branding" -> trail.branding,
+        "discussion" -> trail.discussion,
+        "trailText" -> trail.trailText,
+        "galleryCount" -> trail.galleryCount,
+      )
+      JsObject(obj.fields.filterNot(_._2 == JsNull))
+    }
   }
 
   // We ideally want this to be replaced by something else in the near future. Probably