Merge pull request #250 from mapswipe/fix/data-fetch-script

Improve documentation and data fetch script

Author: AdityaKhatri

README.md

@@ -4,6 +4,12 @@ NextJs application for [Mapswipe community website](https://mapswipe.org).
 
 ## Development
 
+Get all the submodules
+
+```bash
+git submodule update --init --recursive
+```
+
 Before you start, create `.env.local` file:
 
 ```bash
@@ -13,24 +19,31 @@ touch .env.local
 Set these environment variables:
 
 ```env
-APP_ENVIRONMENT=DEV
+APP_ENVIRONMENT=PROD  # Use DEV if you are running your own server instance
 MAPSWIPE_API_ENDPOINT=https://backend.mapswipe.org/
-MAPSWIPE_REFERRER_ENDPOINT=https://website.mapswipe.org/
 NEXT_PUBLIC_POSTHOG_KEY=<posthog-key>
 NEXT_PUBLIC_POSTHOG_HOST_API=<posthog-host-api>
 ```
 
 ### Running
 
 ```bash
+# Install dependencies
 yarn install
-# This fetches latest data from MapSwipe database for projects
+
+# Generate typescript types from graphql schema
+yarn generate:type
+
+# Fetch latest data from MapSwipe database for projects
 yarn fetch-data:local
 
-> NOTE: Currently the platform runs smoothly in node 16, so developers might have to switch to node 16 for development.
+# Run
 yarn dev
 ```
 
+> [!NOTE]
+> Currently the platform runs smoothly in node 16, so developers might have to switch to node 16 for development.
+
 Whenever new texts for translation are added, translation files need to be generated.
 
 ```bash

scripts/fetchData.ts

@@ -195,14 +195,19 @@ async function getCsrfTokenValue() {
     try {
         const healthcheckData = await fetch(healthcheckUrl, { credentials: 'include' });
 
-        const setCookies = 'getSetCookie' in healthcheckData.headers && typeof healthcheckData.headers.getSetCookie === 'function'
-            ? healthcheckData.headers.getSetCookie()
-            : undefined;
+        const cookiesToSet = (
+            healthcheckData.headers as (Headers & { getSetCookie: () => string[] })
+        ).getSetCookie();
 
-        // FIXME: do a proper parsing
-        const token = setCookies?.[0].split('; ')[0].split('=')[1];
+        const parsedCookiesToSet = cookiesToSet
+            .flatMap((item: string) => item.split('; '))
+            .map((item: string) => {
+                const [key, value] = item.split('=');
+                return { key, value } as { key: string, value: string };
+            });
 
-        return token;
+        const csrfToken = parsedCookiesToSet.find((item) => item.key === COOKIE_NAME);
+        return csrfToken?.value;
     } catch (err) {
         // eslint-disable-next-line no-console
         console.error('failed to do the healthcheck', healthcheckUrl);
@@ -223,9 +228,14 @@ async function fetchAndWriteData() {
             console.error('Could not fetch crsf token');
             return;
         }
+        const referer = process.env.MAPSWIPE_REFERER_ENDPOINT ?? baseUrl;
+
+        console.log('CSRF Token exists:', !!csrfTokenValue);
+        console.log('Referer exists:', !!referer);
+
         graphQLClient.setHeader('X-CSRFToken', csrfTokenValue);
         graphQLClient.setHeader('Cookie', `${COOKIE_NAME}=${csrfTokenValue}`);
-        graphQLClient.setHeader('Referer', process.env.MAPSWIPE_REFERER_ENDPOINT ?? '');
+        graphQLClient.setHeader('Referer', referer);
         data = (await graphQLClient.request(query)) as AllDataQuery;
     }