Merge pull request #2830 from britneywwc/discourse-response-cache

feat: add discourse response cache

Author: britneywwc

.env

@@ -25,10 +25,6 @@ RECAPTCHA_SCORE_THRESHOLD=0.5
 # Secret for sitemap endpoint
 SITEMAP_SECRET=somesecret
 
-# Charmhub API
-CHARMHUB_DISCOURSE_API_KEY=charmhub_discourse_api_key
-CHARMHUB_DISCOURSE_API_USER=charmhub_discourse_api_user
-
 # DiscourseAPI
 DISCOURSE_API_KEY=discourse_api_key
 DISCOURSE_API_USERNAME=discourse_api_username

AGENTS.md

@@ -98,7 +98,7 @@ djlint templates/path/to/file.html --lint --profile=jinja   # djlint for html/ji
 
 | Service       | Purpose                          | Key Env Vars |
 | ------------- | -------------------------------- | ------------ |
-| Discourse API | Blog, takeovers, docs, tutorials | `DISCOURSE_API_KEY`, `DISCOURSE_API_USERNAME`, `CHARMHUB_DISCOURSE_API_KEY`, `CHARMHUB_DISCOURSE_API_USERNAME`, `MAAS_DISCOURSE_API_KEY`, `MAAS_DISCOURSE_API_USERNAME` |
+| Discourse API | Blog, takeovers, docs, tutorials | `DISCOURSE_API_KEY`, `DISCOURSE_API_USERNAME`, `MAAS_DISCOURSE_API_KEY`, `MAAS_DISCOURSE_API_USERNAME` |
 | Careers       | Careers related                  | `HARVEST_API_KEY`, `APPLICATION_CRYPTO_SECRET_KEY` |
 | Greenhouse    | Candidate applications           | `GREENHOUSE_API_KEY` |
 | Google Calendar | Google Calendar service-account auth             | `SERVICE_ACCOUNT_EMAIL`, `SERVICE_ACCOUNT_PRIVATE_KEY` |

charm/charmcraft.yaml

@@ -30,14 +30,6 @@ config:
       type: string
       description: "Canonical CLA API URL"
       default: "https://cla.staging.canonical.com"
-    charmhub-discourse-api-key:
-      type: string
-      description: "Charmhub Discourse API key"
-      default: "charmhub_discourse_api_key"
-    charmhub-discourse-api-user:
-      type: string
-      description: "Charmhub Discourse API user"
-      default: "charmhub_discourse_api_user"
     directory-api-token:
       description: "Directory API token"
       default: "disrectory_api_token"

konf/site.yaml

@@ -62,16 +62,6 @@ env:
       key: ubuntu-api-username
       name: discourse-api
 
-  - name: CHARMHUB_DISCOURSE_API_KEY
-    secretKeyRef:
-      key: charmhub-api-key
-      name: discourse-api
-
-  - name: CHARMHUB_DISCOURSE_API_USERNAME
-    secretKeyRef:
-      key: charmhub-api-username
-      name: discourse-api
-
   - name: MAAS_DISCOURSE_API_KEY
     secretKeyRef:
       key: maas-api-key

requirements.txt

@@ -3,7 +3,7 @@ canonicalwebteam.blog==6.8.4
 canonicalwebteam.http==1.0.4
 canonicalwebteam.image-template==1.9.0
 canonicalwebteam.templatefinder==1.0.0
-canonicalwebteam.discourse==7.1.1
+canonicalwebteam.discourse==7.8.0
 canonicalwebteam.search==2.1.2
 canonicalwebteam.form-generator==2.2.0
 canonicalwebteam.directory-parser==1.2.11

webapp/app.py

@@ -35,6 +35,7 @@
     EngagePages,
     TutorialParser,
     Tutorials,
+    ResponseCache,
 )
 from canonicalwebteam.flask_base.app import FlaskBase
 from canonicalwebteam.flask_base.env import get_flask_env
@@ -121,10 +122,6 @@
 # Serves any page as Markdown via ?format=md query parameter
 MarkdownResponse(app)
 
-# Load env variables after the app is initialized
-CHARMHUB_DISCOURSE_API_KEY = os.getenv("CHARMHUB_DISCOURSE_API_KEY")
-CHARMHUB_DISCOURSE_API_USERNAME = os.getenv("CHARMHUB_DISCOURSE_API_USERNAME")
-
 RECAPTCHA_CONFIG = load_recaptcha_config()
 RECAPTCHA_SITE_KEY = RECAPTCHA_CONFIG.get("site_key")
 if not RECAPTCHA_SITE_KEY:
@@ -146,14 +143,6 @@
 
 # Loader supplied to jinja_loader overwrites default jinja_loader
 app.jinja_loader = loader
-
-charmhub_discourse_api = DiscourseAPI(
-    base_url="https://discourse.charmhub.io/",
-    session=get_requests_session(),
-    api_key=CHARMHUB_DISCOURSE_API_KEY,
-    api_username=CHARMHUB_DISCOURSE_API_USERNAME,
-    get_topics_query_id=2,
-)
 search_session = get_requests_session()
 discourse_session = get_requests_session()
 
@@ -1144,6 +1133,7 @@ def allow_src(tag, name, value):
         api=DiscourseAPI(
             base_url="https://discourse.dqlite.io/",
             session=discourse_session,
+            cache=ResponseCache(ttl=600),
         ),
         index_topic_id=34,
         url_prefix="/dqlite/docs",
@@ -1177,6 +1167,7 @@ def allow_src(tag, name, value):
             base_url="https://discourse.maas.io/",
             session=discourse_session,
             get_topics_query_id=2,
+            cache=ResponseCache(ttl=600),
         ),
         index_topic_id=6662,
         url_prefix=maas_url_prefix,
@@ -1261,6 +1252,7 @@ def maas_docs_api():
             api_key=MAAS_DISCOURSE_API_KEY,
             api_username=MAAS_DISCOURSE_API_USERNAME,
             get_topics_query_id=2,
+            cache=ResponseCache(ttl=600),
         ),
         index_topic_id=1289,
         url_prefix="/maas/tutorials",
@@ -1524,6 +1516,7 @@ def cred_exam_content(**_):
     get_topics_query_id=14,
     api_key=DISCOURSE_API_KEY,
     api_username=DISCOURSE_API_USERNAME,
+    cache=ResponseCache(ttl=600),
 )
 engage_pages = EngagePages(
     api=engage_pages_discourse_api,
@@ -1553,6 +1546,7 @@ def cred_exam_content(**_):
     session=search_session,
     api_key=DISCOURSE_API_KEY,
     api_username=DISCOURSE_API_USERNAME,
+    cache=ResponseCache(ttl=600),
 )
 
 
@@ -1588,6 +1582,7 @@ def cred_exam_content(**_):
         api=DiscourseAPI(
             base_url="https://discuss.kubernetes.io/",
             session=get_requests_session(),
+            cache=ResponseCache(ttl=600),
         ),
         index_topic_id=11243,
         url_prefix=microk8s_url_prefix,

webapp/handlers.py

@@ -8,6 +8,7 @@
 
 import flask
 from canonicalwebteam.flask_base.env import get_flask_env
+from canonicalwebteam.discourse import RateLimitedError
 
 logger = logging.getLogger(__name__)
 
@@ -476,3 +477,43 @@ def get_csp_as_str(csp={}, nonce=None):
         if get_flask_env("FLASK_ENV", "production") != "production":
             response.headers["X-Robots-Tag"] = "none"
         return response
+
+    @app.errorhandler(503)
+    def service_unavailable(error):
+        """
+        Rendered when an upstream API (e.g. Discourse) is rate-limiting
+        us and there is no cached response to fall back on. Reuses the
+        styled 500 template (the directory_parser sitemap excludes it,
+        and it is the app's standard "couldn't load this page" error)
+        rather than leaking the internal reason to users.
+
+        JSON endpoints get a JSON body so their fetch() consumers don't
+        choke on HTML, and Retry-After tells well-behaved clients and
+        crawlers when to come back.
+        """
+        accepts = flask.request.accept_mimetypes
+        wants_json = flask.request.path.endswith(".json") or (
+            accepts.accept_json and not accepts.accept_html
+        )
+        if wants_json:
+            response = flask.make_response(
+                flask.jsonify(error="Service temporarily unavailable"),
+                503,
+            )
+        else:
+            response = flask.make_response(
+                flask.render_template("500.html"), 503
+            )
+
+        retry_after = getattr(error, "retry_after", None)
+        response.headers["Retry-After"] = str(retry_after or 60)
+        return response
+
+    @app.errorhandler(RateLimitedError)
+    def discourse_rate_limited(error):
+        """
+        The discourse package raises RateLimitedError when Discourse
+        returns 429 and no cached response is available; serve the same
+        503 as any other upstream outage.
+        """
+        return service_unavailable(error)