filter matches based on the competitions data in S3

Author: marjisound

football/src/main/scala/com/gu/mobile/notifications/football/Configuration.scala

@@ -6,7 +6,6 @@ import com.gu.{AppIdentity, AwsIdentity, DevIdentity}
 import com.typesafe.config.Config
 import software.amazon.awssdk.auth.credentials.{AwsCredentialsProviderChain => AwsCredentialsProviderChainV2, DefaultCredentialsProvider => DefaultCredentialsProviderV2, ProfileCredentialsProvider => ProfileCredentialsProviderV2}
 import com.gu.conf.{ConfigurationLoader, SSMConfigurationLocation}
-import software.amazon.awssdk.regions.Region.EU_WEST_1
 
 class Configuration extends Logging {
 
@@ -47,4 +46,5 @@ class Configuration extends Logging {
   val notificationsApiKey = conf.getString("notifications-client.api-key")
   val mapiHost = conf.getString("mapi.host")
   val capiApiKey = conf.getString("capi.key")
+  val paDataBucket = conf.getString("buckets.football-feed-data")
 }

football/src/main/scala/com/gu/mobile/notifications/football/Lambda.scala

@@ -5,9 +5,10 @@ import java.time.ZonedDateTime
 import java.util.concurrent.TimeUnit
 import com.amazonaws.regions.Regions
 import com.amazonaws.services.dynamodbv2.{AmazonDynamoDBAsync, AmazonDynamoDBAsyncClientBuilder}
+import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import com.gu.contentapi.client.GuardianContentClient
 import com.gu.mobile.liveactivities.event.bus.LiveActivityPusher
-import com.gu.mobile.notifications.football.lib.{ArticleSearcher, DynamoDistinctCheck, DynamoMatchLiveActivity, DynamoMatchNotification, EventConsumer, EventFilter, FootballData, LiveActivityEventConsumer, NotificationHttpProvider, NotificationSender, NotificationsApiClient, PaFootballClient, SyntheticMatchEventGenerator}
+import com.gu.mobile.notifications.football.lib.{ArticleSearcher, DynamoDistinctCheck, DynamoMatchLiveActivity, DynamoMatchNotification, EventConsumer, EventFilter, FootballData, LiveActivityEventConsumer, NotificationHttpProvider, NotificationSender, NotificationsApiClient, PACompetition, PaFootballClient, S3DataStore, SyntheticMatchEventGenerator}
 import com.gu.mobile.notifications.football.notificationbuilders.{MatchStatusLiveActivityPayloadBuilder, MatchStatusNotificationBuilder}
 import play.api.libs.json.Json
 
@@ -22,6 +23,7 @@ import com.gu.mobile.notifications.football.models.MatchDataWithArticle
 
 import scala.concurrent.Future
 import org.scanamo.generic.auto._
+import pa.Competition
 
 object Lambda extends Logging {
 
@@ -56,7 +58,14 @@ object Lambda extends Logging {
 
   val apiClient = new NotificationsApiClient(configuration)
 
-  lazy val footballData = new FootballData(paFootballClient, syntheticMatchEventGenerator)
+  lazy val s3Client: AmazonS3 = AmazonS3ClientBuilder.standard
+    .withRegion(Regions.EU_WEST_1)
+    .withCredentials(configuration.credentials)
+    .build()
+
+  lazy val competitionsDataStore = new S3DataStore[PACompetition](s3Client, configuration.paDataBucket)
+
+  lazy val footballData = new FootballData(paFootballClient, syntheticMatchEventGenerator, competitionsDataStore, configuration.stage)
 
   lazy val articleSearcher = new ArticleSearcher(capiClient)
 

football/src/main/scala/com/gu/mobile/notifications/football/lib/FootballData.scala

@@ -1,11 +1,11 @@
 package com.gu.mobile.notifications.football.lib
 
-import java.time.ZonedDateTime
-
-import com.gu.mobile.notifications.football.Logging
+import java.time.{LocalDate, ZonedDateTime}
+import com.gu.mobile.notifications.football.{Configuration, Logging}
 import com.gu.mobile.notifications.football.models.RawMatchData
 import org.joda.time.DateTime
 import pa.MatchDay
+import play.api.libs.json.{Format, Json}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -14,9 +14,24 @@ import scala.util.{Failure, Success, Try}
 
 case class EndedMatch(matchId: String, startTime: DateTime)
 
+case class PACompetition(
+                        id: String,
+                        tag: String,
+                        fullName: String,
+                        shortName: String,
+                        startDate: Option[LocalDate] = None,
+                        endDate: Option[LocalDate] = None,
+                      )
+
+object PACompetition {
+  implicit val competitionFormat: Format[PACompetition] = Json.format[PACompetition]
+}
+
 class FootballData(
-  paClient: PaFootballClient,
-  syntheticEvents: SyntheticMatchEventGenerator
+                    paClient: PaFootballClient,
+                    syntheticEvents: SyntheticMatchEventGenerator,
+                    competitionsDataStore: S3DataStore[PACompetition],
+                    stage: String
 ) extends Logging {
 
   implicit class RichMatchDay(matchDay: MatchDay) {
@@ -92,13 +107,29 @@ class FootballData(
       }.getOrElse(false) //Shouldn't ever happen
     }
 
+    def competitionIsSupported(supportedCompetitions: List[PACompetition])(matchDay: MatchDay): Boolean =
+      matchDay.competition.exists { matchComp =>
+        supportedCompetitions.exists(_.id == matchComp.id)
+      }
+
     logger.info(s"Retrieving matches on or around $dateTime from PA")
-    val matches = paClient.aroundToday(dateTime)
-    matches.map(
-      _.filter(inProgress)
-       .filter(paProvideAlerts)
-       .filterNot(isMidnight)
-    )
+    for {
+       matches <- paClient.aroundToday(dateTime)
+       competitions <- competitionsDataStore
+         .fetch(s"${stage}/competition/competitions.json")
+         .recover { case exception =>
+           // We don't want to fail the whole process if we can't retrieve the list of competitions,
+           // we'll just assume all competitions are supported and log the error
+           logger.error(s"Failed to retrieve list of competitions: ${exception.getMessage}", exception)
+           List.empty[PACompetition]
+         }
+    } yield {
+      matches
+        .filter(m => competitions.isEmpty || competitionIsSupported(competitions)(m))
+        .filter(inProgress)
+        .filter(paProvideAlerts)
+        .filterNot(isMidnight)
+    }
   }
 
   private def processMatch(matchDay: MatchDay): Future[Option[RawMatchData]] = {

football/src/main/scala/com/gu/mobile/notifications/football/lib/S3DataStore.scala

@@ -0,0 +1,43 @@
+package com.gu.mobile.notifications.football.lib
+
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.services.s3.model.S3Object
+import com.amazonaws.util.IOUtils
+import com.gu.mobile.notifications.football.Logging
+import play.api.libs.json.{Format, JsError, JsSuccess, Json}
+
+import scala.concurrent.Future
+import scala.util.{Failure, Success, Try}
+
+class S3DataStore[T](s3Client: AmazonS3, bucketName: String) extends Logging {
+  def fetch(path: String)(implicit format: Format[T]) : Future[List[T]] = {
+    Try(parseS3Object(path)) match {
+      case Success(list) => Future.successful(list)
+      case Failure(ex) =>
+        logger.error(s"Error retrieving items from s3. Bucket: $bucketName, Path: $path")
+        Future.failed(ex)
+    }
+  }
+
+  private def parseS3Object(path: String)(implicit format: Format[T]) : List[T] = {
+    Json.fromJson[List[T]](Json.parse(asString(s3Client.getObject(bucketName, path)))) match {
+      case JsSuccess(list, __) =>
+        logger.info(s"Got ${list.length} items from s3, path $path")
+        list
+      case JsError(errors) =>
+        val errorPaths = errors.map { error => error._1.toString() }.mkString(",")
+        logger.error(s"Error parsing S3 items on path $path. Error path(s): $errorPaths")
+        throw new Exception(s"could not extract list $path. Errors paths(s): $errors")
+    }
+  }
+
+  private def asString(s3Object: S3Object): String = {
+    val s3ObjectContent = s3Object.getObjectContent
+    try {
+      IOUtils.toString(s3ObjectContent)
+    }
+    finally {
+      s3ObjectContent.close()
+    }
+  }
+}