Skip to content

Commit 9e0db52

Browse files
arelramarjisound
arelra
and
marjisound
committed
Add a feature switch to toggle between the old and new method of getting deeply read data into the cache.
Co-authored-by: Marjan K <15894063+marjisound@users.noreply.github.com>
1 parent de54ed7 commit 9e0db52

6 files changed

Lines changed: 147 additions & 9 deletions

File tree

‎common/app/agents/DeeplyReadAgent.scala‎

Lines changed: 129 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,29 @@
11
package agents
22

3+
import com.gu.contentapi.client.model.v1.{Content, ElementType}
4+
import com.gu.contentapi.client.utils.CapiModelEnrichment.RenderingFormat
35
import common._
46
import conf.Configuration
7+
import conf.switches.Switches.UseTrailsFromS3
58
import model.dotcomrendering.Trail
6-
import services.S3Async
9+
import services.{FaciaContentConvert, OphanApi, S3Async}
710

811
import scala.concurrent.{ExecutionContext, Future}
12+
import scala.util.control.NonFatal
13+
import contentapi.ContentApiClient
14+
import layout.DiscussionSettings
15+
import model.ContentFormat
916

1017
object DeeplyReadS3Agent extends S3Async {
1118
override lazy val bucket = Configuration.cache.bucket;
1219
lazy val stage: String = Configuration.environment.stage.toUpperCase
1320
}
1421

15-
class DeeplyReadAgent extends GuLogging {
22+
class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) extends GuLogging {
1623

1724
private val deeplyReadItems = Box[Map[Edition, Seq[Trail]]](Map.empty)
1825

19-
def refresh()(implicit ec: ExecutionContext): Future[Unit] = {
20-
log.debug(s"Deeply Read Agent refresh()")
26+
def getTrailsFromS3(edition: Edition)(implicit ec: ExecutionContext): Future[Unit] = {
2127
Future
2228
.sequence(Edition.allEditions.map { edition =>
2329
DeeplyReadS3Agent
@@ -43,12 +49,130 @@ class DeeplyReadAgent extends GuLogging {
4349
})
4450
}
4551

52+
def removeStartingSlash(path: String): String = {
53+
if (path.startsWith("/")) path.stripPrefix("/") else path
54+
}
55+
56+
def correctPillar(pillar: String): String = if (pillar == "arts") "culture" else pillar
57+
58+
def deeplyReadUrlToTrail(content: Content): Option[Trail] = {
59+
60+
val contentFormat: ContentFormat = ContentFormat(content.design, content.theme, content.display)
61+
62+
for {
63+
webPublicationDate <- content.webPublicationDate
64+
fields <- content.fields
65+
linkText <- fields.trailText
66+
pillar <- content.pillarName
67+
headline <- fields.headline
68+
shortUrl <- fields.shortUrl
69+
} yield Trail(
70+
url = content.webUrl,
71+
linkText = linkText,
72+
showByline = false,
73+
byline = fields.byline,
74+
masterImage = None,
75+
image = fields.thumbnail,
76+
carouselImages = Map.empty,
77+
ageWarning = None,
78+
isLiveBlog = fields.liveBloggingNow.getOrElse(false),
79+
pillar = correctPillar(pillar.toLowerCase),
80+
designType = content.`type`.toString,
81+
format = contentFormat,
82+
webPublicationDate = webPublicationDate.toString(),
83+
headline = headline,
84+
mediaType = None,
85+
shortUrl = shortUrl,
86+
kickerText = None,
87+
starRating = None,
88+
avatarUrl = None,
89+
branding = None,
90+
discussion = DiscussionSettings.fromTrail(FaciaContentConvert.contentToFaciaContent(content)),
91+
trailText = content.fields.flatMap(_.trailText),
92+
galleryCount =
93+
content.elements.map(_.count(el => el.`type` == ElementType.Image && el.relation == "gallery")).filter(_ > 0),
94+
)
95+
}
96+
97+
def getTrailsFromCAPI(edition: Edition)(implicit ec: ExecutionContext): Future[Unit] = {
98+
/*
99+
We query Ophan for the deeply read URLs and use them to queryCapi
100+
then use this information to create a sequence of trails that we cache
101+
using a Box structure.
102+
*/
103+
Future
104+
.sequence(Edition.allEditions.map { edition =>
105+
ophanApi
106+
.getDeeplyRead(edition)
107+
.flatMap { ophanDeeplyReadItems =>
108+
log.debug(s"Fetched ${ophanDeeplyReadItems.size} Deeply Read items for ${edition.displayName}")
109+
val constructedTrail: Seq[Future[Option[Trail]]] = ophanDeeplyReadItems.map { ophanItem =>
110+
log.debug(s"CAPI lookup for Ophan deeply read item: ${ophanItem.toString}")
111+
val path = removeStartingSlash(ophanItem.path)
112+
log.debug(s"CAPI Lookup for path: $path")
113+
val capiRequest = contentApiClient
114+
.item(path)
115+
.showTags("all")
116+
.showFields("all")
117+
.showReferences("none")
118+
.showAtoms("none")
119+
120+
contentApiClient
121+
.getResponse(capiRequest)
122+
.map { res =>
123+
res.content.flatMap { capiData =>
124+
log.debug(s"Retrieved CAPI data for Deeply Read item: ${path}")
125+
deeplyReadUrlToTrail(capiData)
126+
}
127+
}
128+
.recover { case NonFatal(e) =>
129+
log.error(s"Error retrieving CAPI data for Deeply Read item: ${path}. ${e.getMessage}")
130+
None
131+
}
132+
}
133+
Future
134+
.sequence(constructedTrail)
135+
.map { maybeTrails =>
136+
(edition, maybeTrails.flatten.take(10))
137+
}
138+
139+
}
140+
.recover { e =>
141+
log.error(s"Failed to fetch Deeply Read items for ${edition.displayName}. ${e.getMessage()}")
142+
(edition, Seq.empty)
143+
}
144+
})
145+
.map(trailsList => {
146+
val map = trailsList.toMap
147+
for {
148+
(edition, list) <- map
149+
} yield log.debug(s"Deeply Read in ${edition.displayName}, ${list.size} items: ${list.map(_.url).toString()}")
150+
151+
val mapWithTenItems = map.filter { case (_, list) => list.size == 10 }
152+
log.debug(
153+
s"Updating the following ${mapWithTenItems.size} editions: ${mapWithTenItems.keys.map(_.id).toList.sorted.toString()}",
154+
)
155+
156+
deeplyReadItems.alter(deeplyReadItems.get() ++ mapWithTenItems)
157+
})
158+
}
159+
160+
def refresh()(implicit ec: ExecutionContext): Future[Unit] = {
161+
if (UseTrailsFromS3.isSwitchedOn) {
162+
// TODO revert to log.debug once we switch to S3 permanently
163+
log.info(s"Deeply Read Agent refresh() - Using S3")
164+
getTrailsFromS3(Edition.defaultEdition)
165+
} else {
166+
log.info(s"Deeply Read Agent refresh() - Using CAPI")
167+
getTrailsFromCAPI(Edition.defaultEdition)
168+
}
169+
}
170+
46171
def getTrails(edition: Edition)(implicit ec: ExecutionContext): Seq[Trail] = {
47172
val updatedTrails = deeplyReadItems.get().getOrElse(edition, Seq.empty)
48173
if (updatedTrails.isEmpty) {
49174
refresh()
50175
}
51176
updatedTrails
52177
}
53-
54178
}

‎common/app/conf/switches/FeatureSwitches.scala‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -671,4 +671,15 @@ trait FeatureSwitches {
671671
exposeClientSide = true,
672672
highImpact = false,
673673
)
674+
675+
val UseTrailsFromS3 = Switch(
676+
group = SwitchGroup.Feature,
677+
name = "use-trails-from-s3",
678+
description = "Use trails from S3",
679+
owners = Seq(Owner.withEmail("dotcom.platform@theguardian.com")),
680+
sellByDate = never,
681+
safeState = Off,
682+
exposeClientSide = false,
683+
highImpact = false,
684+
)
674685
}

‎common/test/agents/DeeplyReadAgentTest.scala‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@ import test.{
2525
"DeeplyReadAgent" should "initialise with trails being an empty Seq" in {
2626
val ophanApi = new OphanApi(wsClient)
2727
val contentApiClient = testContentApiClient
28-
val agent = new DeeplyReadAgent
28+
val agent = new DeeplyReadAgent(
29+
contentApiClient = contentApiClient,
30+
ophanApi = ophanApi,
31+
)
2932
Edition.allEditions.map(edition => {
3033
agent.getTrails(edition) shouldBe Seq.empty
3134
})

‎facia/test/FaciaControllerTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ import scala.concurrent.{Await, Future}
4747
play.api.test.Helpers.stubControllerComponents(),
4848
wsClient,
4949
new MostViewedAgent(testContentApiClient, new OphanApi(wsClient)),
50-
new DeeplyReadAgent,
50+
new DeeplyReadAgent(testContentApiClient, new OphanApi(wsClient)),
5151
assets = assets,
5252
)
5353
val articleUrl = "/environment/2012/feb/22/capitalise-low-carbon-future"

‎facia/test/FaciaMetaDataTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ import scala.concurrent.duration._
4949
play.api.test.Helpers.stubControllerComponents(),
5050
wsClient,
5151
new MostViewedAgent(testContentApiClient, new OphanApi(wsClient)),
52-
new DeeplyReadAgent,
52+
new DeeplyReadAgent(testContentApiClient, new OphanApi(wsClient)),
5353
assets = assets,
5454
)
5555
val frontPath = "music"

‎onward/test/MostPopularControllerTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import agents.DeeplyReadAgent
2727
testContentApiClient,
2828
new GeoMostPopularAgent(testContentApiClient, ophanApi),
2929
new MostPopularAgent(testContentApiClient),
30-
new DeeplyReadAgent,
30+
new DeeplyReadAgent(testContentApiClient, new OphanApi(wsClient)),
3131
play.api.test.Helpers.stubControllerComponents(),
3232
)
3333

0 commit comments

Comments
 (0)