Skip to content

Commit 88a6be3

Browse files
FlorentRvlFlorentRvl
committed
fix: gsheet request too large -> use a function created in data
1 parent 9b4fac7 commit 88a6be3

9 files changed

Lines changed: 95 additions & 91 deletions

‎__pipeline__.ipynb‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,11 +243,10 @@
243243
" prompt=\"Find timezone in the format 'Region/City'. If there is no exact match, please return a subjective answer based on the data you received\",\n",
244244
" message=f\"Region: {region}, Country: {country}\",\n",
245245
" )\n",
246-
" print(\"- Timezone:\", timezone)\n",
247-
" print()\n",
248-
" naas.set_remote_timezone(timezone)\n",
249246
" pdump(output_dir, timezone, \"timezone\")\n",
250247
" naas.dependency.add(os.path.join(output_dir, \"timezone.pickle\"))\n",
248+
" print(\"- Timezone:\", timezone)\n",
249+
" naas.set_remote_timezone(timezone)\n",
251250
" print()\n",
252251
" \n",
253252
" # Create notebook steps\n",
@@ -358,7 +357,7 @@
358357
"outputs": [],
359358
"source": [
360359
"# Schedule pipeline\n",
361-
"cron = \"0 8 * * *\"\n",
360+
"cron = \"0 12 * * *\"\n",
362361
"print(\"⏰ Scheduler:\", cron)\n",
363362
"naas.scheduler.add(cron=cron)"
364363
]

‎models/growth-engine/core/domain/Google_Sheets_Update_contacts_view.ipynb‎

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@
176176
"outputs": [],
177177
"source": [
178178
"df_init = gsheet.connect(spreadsheet_url).get(sheet_name=sheet_contacts)\n",
179+
"df_init = pload(output_dir, file_contacts)\n",
179180
"if not isinstance(df_init, pd.DataFrame):\n",
180181
" df_init = pd.DataFrame()\n",
181182
" messaging_options = {}\n",
@@ -430,7 +431,9 @@
430431
" messaging = create_chat_completion(api_key, prompt_messaging, str(tmp_df.to_dict()))\n",
431432
" except Exception as e:\n",
432433
" print(e)\n",
433-
" print(\"Messaging options:\", messaging)\n",
434+
" if str(messaging) == \"None\":\n",
435+
" messaging = \"TBD\"\n",
436+
" print(\"Messaging options:\", messaging)\n",
434437
" print()\n",
435438
" messaging_options[profile_url] = messaging\n",
436439
" pdump(output_dir, messaging_options, \"messaging_options\")\n",
@@ -491,31 +494,14 @@
491494
{
492495
"cell_type": "code",
493496
"execution_count": null,
494-
"id": "32e91b35-c1f8-4fe6-ae78-a2d4b79c8be7",
497+
"id": "ea691820-773d-4148-943b-34aad838a759",
495498
"metadata": {
496-
"papermill": {},
497499
"tags": []
498500
},
499501
"outputs": [],
500502
"source": [
501-
"df_check = pd.concat([df_init.astype(str), df_contacts.astype(str)]).drop_duplicates(keep=False)\n",
502-
"if len(df_check) > 0:\n",
503-
" if len(df_contacts) < 3000:\n",
504-
" gsheet.connect(spreadsheet_url).send(data=df_contacts, sheet_name=sheet_contacts, append=False)\n",
505-
" else:\n",
506-
" gsheet.connect(spreadsheet_url).send(data=df_contacts[:3000], sheet_name=sheet_contacts, append=False)\n",
507-
" gsheet.connect(spreadsheet_url).send(data=df_contacts[3000:], sheet_name=sheet_contacts, append=True)\n",
508-
"else:\n",
509-
" print(\"Noting to update in Google Sheets!\")"
503+
"send_data_to_gsheet(df_contacts, df_init, spreadsheet_url, sheet_contacts)"
510504
]
511-
},
512-
{
513-
"cell_type": "code",
514-
"execution_count": null,
515-
"id": "ea691820-773d-4148-943b-34aad838a759",
516-
"metadata": {},
517-
"outputs": [],
518-
"source": []
519505
}
520506
],
521507
"metadata": {

‎models/growth-engine/core/domain/Google_Sheets_Update_growth_db.ipynb‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@
134134
"# Inputs\n",
135135
"entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"entity_dir\")\n",
136136
"input_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
137-
"file_interactions = \"linkedin_interactions\"\n",
137+
"file_interactions = \"interactions\"\n",
138138
"\n",
139139
"# Outputs\n",
140140
"spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"abi_spreadsheet\")\n",
@@ -439,17 +439,13 @@
439439
},
440440
"outputs": [],
441441
"source": [
442-
"df_check = pd.concat([df_init.astype(str), df_growth.astype(str)]).drop_duplicates(keep=False)\n",
443-
"if len(df_check) > 0:\n",
444-
" gsheet.connect(spreadsheet_url).send(sheet_name=sheet_growth, data=df_growth, append=False)\n",
445-
"else:\n",
446-
" print(\"Noting to update in Google Sheets!\")"
442+
"send_data_to_gsheet(df_growth, df_init, spreadsheet_url, sheet_growth)"
447443
]
448444
},
449445
{
450446
"cell_type": "code",
451447
"execution_count": null,
452-
"id": "13cc4241-9a56-4495-9369-bf3de2bcfb42",
448+
"id": "c81ada64-6da0-4c75-a433-751e28b62b47",
453449
"metadata": {},
454450
"outputs": [],
455451
"source": []

‎models/growth-engine/core/domain/Google_Sheets_Update_interactions_db.ipynb‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@
138138
"\n",
139139
"# Outputs\n",
140140
"output_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
141-
"output_file = \"linkedin_interactions\"\n",
141+
"output_file = \"interactions\"\n",
142142
"spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"abi_spreadsheet\")\n",
143143
"sheet_interaction = \"INTERACTIONS\""
144144
]
@@ -216,11 +216,7 @@
216216
" tmp_posts_url = tmp_df_reactions[\"POST_URL\"].unique().tolist()\n",
217217
" for x in tmp_posts_url:\n",
218218
" if x not in posts_url:\n",
219-
" # Histo\n",
220-
" if date_dir < date(2024, 5 , 1):\n",
221-
" tmp_df_reactions[\"DATE_REACTION\"] = pd.to_datetime(tmp_df_reactions['PUBLISHED_DATE'], format='%Y-%m-%d %H:%M:%S%z').dt.tz_convert(TIMEZONE).dt.strftime(\"%Y-%m-%d %H:%M:%S%z\")\n",
222-
" else:\n",
223-
" tmp_df_reactions[\"DATE_REACTION\"] = tmp_df_reactions['PUBLISHED_DATE']\n",
219+
" tmp_df_reactions[\"DATE_REACTION\"] = tmp_df_reactions['PUBLISHED_DATE']\n",
224220
" posts_url.append(x)\n",
225221
" else:\n",
226222
" tmp_df_reactions[\"DATE_REACTION\"] = pd.to_datetime(tmp_df_reactions['DATE_EXTRACT'], format='%Y-%m-%d %H:%M:%S').dt.tz_localize(pytz.timezone(\"Europe/Paris\")).dt.tz_convert(TIMEZONE).dt.strftime(\"%Y-%m-%d %H:%M:%S%z\")\n",
@@ -329,10 +325,10 @@
329325
" \n",
330326
" # Concat df\n",
331327
" df = pd.concat([df1, df2]).reset_index(drop=True)\n",
332-
" \n",
328+
"\n",
333329
" # Exclude Entity from Full name\n",
334330
" if len(df) > 0:\n",
335-
" df.insert(loc=2, column=\"DATE\", value=pd.to_datetime(df['DATE_INTERACTION'], format=\"%Y-%m-%d %H:%M:%S%z\").dt.strftime(\"%a. %d %b.\"))\n",
331+
" df.insert(loc=2, column=\"DATE\", value=pd.to_datetime(df['DATE_INTERACTION'].str[:19], format=\"%Y-%m-%d %H:%M:%S\").dt.strftime(\"%a. %d %b.\"))\n",
336332
" entity = df.loc[0 , \"ENTITY\"]\n",
337333
" df = df[df[\"FULLNAME\"] != entity]\n",
338334
" \n",
@@ -419,17 +415,13 @@
419415
},
420416
"outputs": [],
421417
"source": [
422-
"df_check = pd.concat([df_init.astype(str), df_interactions.astype(str)]).drop_duplicates(keep=False)\n",
423-
"if len(df_check) > 0:\n",
424-
" gsheet.connect(spreadsheet_url).send(sheet_name=sheet_interaction, data=df_interactions, append=False)\n",
425-
"else:\n",
426-
" print(\"Noting to update in Google Sheets!\")"
418+
"send_data_to_gsheet(df_interactions, df_init, spreadsheet_url, sheet_interaction)"
427419
]
428420
},
429421
{
430422
"cell_type": "code",
431423
"execution_count": null,
432-
"id": "29368ef4-de34-44bc-8d8a-346909b8ea30",
424+
"id": "c15fc896-7362-4ab9-8c99-6c45fc41c9d1",
433425
"metadata": {},
434426
"outputs": [],
435427
"source": []

‎models/growth-engine/core/domain/Google_Sheets_Update_organizations_db.ipynb‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@
172172
"id": "4a5d888e-9fa7-4fe9-983b-c028cd0bb2a9",
173173
"metadata": {},
174174
"source": [
175-
"### Get companies"
175+
"### Get organizations"
176176
]
177177
},
178178
{
@@ -611,12 +611,13 @@
611611
" # Update org name with company name from LinkedIn\n",
612612
" df = df_organizations.copy()\n",
613613
" df = df[(df[\"ORGANIZATION\"] != df[\"ORG_NAME\"]) & (df[\"ORG_LINKEDIN_ID\"] != \"TBD\")]\n",
614-
" org_names_1 = get_dict_from_df(df, \"ORG_NAME\", \"ORGANIZATION\", f\"organizations_names_{datetime.now().isoformat()}\", output_dir)\n",
614+
" org_names_1 = get_dict_from_df(df, \"ORG_NAME\", \"ORGANIZATION\", f\"organizations_names\", output_dir)\n",
615615
" print(\"-> New Organization to be updated in People db:\", len(org_names_1))\n",
616616
" if len(org_names_1) > 0:\n",
617617
" df_org, df_p = update_dfs_from_dict(org_names_1, df_p, df_org)\n",
618618
" org_lk_urls = pload(output_dir, \"org_lk_urls\")\n",
619619
" for key, value in org_names_1.items():\n",
620+
" key = key.replace(\"Not Found\", \"\")\n",
620621
" org_lk_urls[value] = org_lk_urls[key]\n",
621622
" pdump(output_dir, org_lk_urls, \"org_lk_urls\")\n",
622623
" \n",
@@ -626,7 +627,7 @@
626627
" # Get org name with data tbd with same LinkedIn URL as org found\n",
627628
" df1 = df_org.copy()\n",
628629
" df1 = df1[(df1[\"LINKEDIN_URL\"] != \"NA\") & (df1[\"ORG_LINKEDIN_ID\"] != \"TBD\")]\n",
629-
" organizations_org_urls = get_dict_from_df(df1, \"ORGANIZATION\", \"LINKEDIN_URL\", f\"organizations_org_urls_{datetime.now().isoformat()}\", output_dir)\n",
630+
" organizations_org_urls = get_dict_from_df(df1, \"ORGANIZATION\", \"LINKEDIN_URL\", f\"organizations_org_urls\", output_dir)\n",
630631
"\n",
631632
" # Get org name with data tbd with similar LinkedIn URL\n",
632633
" df2 = df_org.copy()\n",
@@ -717,18 +718,13 @@
717718
{
718719
"cell_type": "code",
719720
"execution_count": null,
720-
"id": "32e91b35-c1f8-4fe6-ae78-a2d4b79c8be7",
721+
"id": "5098f953-0607-4225-b3f3-39e06dd301db",
721722
"metadata": {
722-
"papermill": {},
723723
"tags": []
724724
},
725725
"outputs": [],
726726
"source": [
727-
"df_check = pd.concat([df_init.astype(str), df_organizations_u.astype(str)]).drop_duplicates(keep=False)\n",
728-
"if len(df_check) > 0:\n",
729-
" gsheet.connect(spreadsheet_url).send(data=df_organizations_u, sheet_name=sheet_people_organizations, append=False)\n",
730-
"else:\n",
731-
" print(\"Noting to update in Google Sheets!\")"
727+
"send_data_to_gsheet(df_organizations_u, df_init, spreadsheet_url, sheet_people_organizations)"
732728
]
733729
},
734730
{
@@ -742,24 +738,19 @@
742738
{
743739
"cell_type": "code",
744740
"execution_count": null,
745-
"id": "f3c0277a-239f-4d5d-b34f-655c9f7c0028",
741+
"id": "23bcfff6-e962-4191-b79c-164ab9f65ff6",
746742
"metadata": {
747743
"tags": []
748744
},
749745
"outputs": [],
750746
"source": [
751-
"df_check = pd.concat([df_people.astype(str), df_people_u.astype(str)]).drop_duplicates(keep=False)\n",
752-
"if len(df_check) > 0:\n",
753-
" pdump(output_dir, df_people_u, file_people)\n",
754-
" gsheet.connect(spreadsheet_url).send(data=df_people_u, sheet_name=sheet_people, append=False)\n",
755-
"else:\n",
756-
" print(\"Noting to update in Google Sheets!\")"
747+
"send_data_to_gsheet(df_people_u, df_people, spreadsheet_url, sheet_people)"
757748
]
758749
},
759750
{
760751
"cell_type": "code",
761752
"execution_count": null,
762-
"id": "23bcfff6-e962-4191-b79c-164ab9f65ff6",
753+
"id": "f0ca1da1-617f-459e-a698-c5bb695a83be",
763754
"metadata": {},
764755
"outputs": [],
765756
"source": []

‎models/growth-engine/core/domain/Google_Sheets_Update_people_db.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@
143143
"# Inputs\n",
144144
"entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"entity_dir\")\n",
145145
"input_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
146-
"file_interactions = \"linkedin_interactions\"\n",
146+
"file_interactions = \"interactions\"\n",
147147
"api_key = os.environ.get(\"NAAS_API_TOKEN\") or naas.secret.get('NAAS_API_TOKEN')\n",
148148
"li_at = os.environ.get(\"LINKEDIN_LI_AT\") or naas.secret.get(\"LINKEDIN_LI_AT\")\n",
149149
"JSESSIONID = os.environ.get(\"LINKEDIN_JSESSIONID\") or naas.secret.get(\"LINKEDIN_JSESSIONID\")\n",
@@ -676,23 +676,19 @@
676676
{
677677
"cell_type": "code",
678678
"execution_count": null,
679-
"id": "59a498dd-a1d3-46fd-8b81-efa4df1813f1",
679+
"id": "6219b07a-9052-4f7f-8754-b2aa22eeabc7",
680680
"metadata": {
681681
"tags": []
682682
},
683683
"outputs": [],
684684
"source": [
685-
"df_check = pd.concat([df_init.astype(str), df_people.astype(str)]).drop_duplicates(keep=False)\n",
686-
"if len(df_check) > 0:\n",
687-
" gsheet.connect(spreadsheet_url).send(data=df_people, sheet_name=sheet_people, append=False)\n",
688-
"else:\n",
689-
" print(\"Noting to update in Google Sheets!\") "
685+
"send_data_to_gsheet(df_people, df_init, spreadsheet_url, sheet_people)"
690686
]
691687
},
692688
{
693689
"cell_type": "code",
694690
"execution_count": null,
695-
"id": "6219b07a-9052-4f7f-8754-b2aa22eeabc7",
691+
"id": "5f9806ef-87dc-4582-8d32-772eb036c8ad",
696692
"metadata": {},
697693
"outputs": [],
698694
"source": []

‎models/growth-engine/core/domain/LinkedIn_Get_interactions_from_posts.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@
173173
"\n",
174174
"# Outputs\n",
175175
"output_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
176-
"file_posts = \"linkedin_posts\"\n",
176+
"file_posts = \"posts\"\n",
177177
"file_reactions = \"linkedin_post_reactions\"\n",
178178
"file_comments = \"linkedin_post_comments\""
179179
]

‎models/sales-engine/core/domain/Google_Sheets_Update_deals_db.ipynb‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -392,30 +392,15 @@
392392
"### Send data to Google Sheets spreadsheet"
393393
]
394394
},
395-
{
396-
"cell_type": "code",
397-
"execution_count": null,
398-
"id": "32e91b35-c1f8-4fe6-ae78-a2d4b79c8be7",
399-
"metadata": {
400-
"papermill": {},
401-
"tags": []
402-
},
403-
"outputs": [],
404-
"source": [
405-
"df_check = pd.concat([db_deals.astype(str), df_init.astype(str)]).drop_duplicates(keep=False)\n",
406-
"if len(df_check) > 0:\n",
407-
" gsheet.connect(spreadsheet_url).send(sheet_name=sheet_name_output, data=db_deals, append=False)\n",
408-
"else:\n",
409-
" print(\"Noting to update in Google Sheets!\") "
410-
]
411-
},
412395
{
413396
"cell_type": "code",
414397
"execution_count": null,
415398
"id": "13cc4241-9a56-4495-9369-bf3de2bcfb42",
416399
"metadata": {},
417400
"outputs": [],
418-
"source": []
401+
"source": [
402+
"send_data_to_gsheet(db_deals, df_init, spreadsheet_url, sheet_name_output)"
403+
]
419404
}
420405
],
421406
"metadata": {

‎utils/data.ipynb‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@
5858
"import naas\n",
5959
"from naas_drivers import linkedin\n",
6060
"import naas_data_product\n",
61-
"import shutil"
61+
"import shutil\n",
62+
"import pandas as pd"
6263
]
6364
},
6465
{
@@ -136,6 +137,64 @@
136137
" return None"
137138
]
138139
},
140+
{
141+
"cell_type": "markdown",
142+
"id": "8cee202d-2085-46ff-ba31-54baee04f0be",
143+
"metadata": {},
144+
"source": [
145+
"### Send data to Gsheet"
146+
]
147+
},
148+
{
149+
"cell_type": "code",
150+
"execution_count": null,
151+
"id": "c3baae15-fcf9-4c22-9940-fedbeb20b262",
152+
"metadata": {},
153+
"outputs": [],
154+
"source": [
155+
"def send_data_to_gsheet(\n",
156+
" df: pd.DataFrame,\n",
157+
" df_init: pd.DataFrame, \n",
158+
" spreadsheet_url: str, \n",
159+
" sheet_name: str, \n",
160+
" chunck_size: int = 100000\n",
161+
"):\n",
162+
" \"\"\"\n",
163+
" This function compares two dataframes and if they are different, sends the data from the second dataframe to a Google Sheet.\n",
164+
"\n",
165+
" :param df: The main dataframe to be sent to Google Sheet.\n",
166+
" :param df_init: The initial dataframe to be compared with the main dataframe.\n",
167+
" :param spreadsheet_url: The URL of the Google Sheet to send data to.\n",
168+
" :param sheet_name: The name of the sheet in the Google Sheet to send data to.\n",
169+
" :param chunck_size: The size of the chunks to split the data into for sending. Default is 100000.\n",
170+
" \"\"\"\n",
171+
" # Compare dataframes\n",
172+
" df_check = pd.concat([df.astype(str), df_init.astype(str)]).drop_duplicates(keep=False)\n",
173+
" \n",
174+
" # Update or Do nothing\n",
175+
" if len(df_check) > 0:\n",
176+
" df_size = len(df) * len(df.columns)\n",
177+
" if df_size < chunck_size:\n",
178+
" gsheet.connect(spreadsheet_url).send(sheet_name=sheet_name, data=df, append=False)\n",
179+
" print(f\"✅ DataFrame successfully sent to Google Sheets!\")\n",
180+
" else:\n",
181+
" max_rows = int(chunck_size / len(df.columns))\n",
182+
" start = 0\n",
183+
" limit = start + max_rows\n",
184+
" gsheet.connect(spreadsheet_url).send(sheet_name=sheet_name, data=df[start:limit], append=False)\n",
185+
" print(f\"✅ Rows {start} to {limit} successfully added to Google Sheets!\")\n",
186+
" start += max_rows\n",
187+
" while start < len(df):\n",
188+
" limit = start + max_rows\n",
189+
" if limit > len(df):\n",
190+
" limit = len(df)\n",
191+
" gsheet.connect(spreadsheet_url).send(sheet_name=sheet_name, data=df[start:limit], append=True)\n",
192+
" print(f\"✅ Rows {start} to {limit} successfully added to Google Sheets!\")\n",
193+
" start += max_rows\n",
194+
" else:\n",
195+
" print(\"Noting to update in Google Sheets!\")"
196+
]
197+
},
139198
{
140199
"cell_type": "markdown",
141200
"id": "e929759d-1080-40df-912a-873fa21deb92",

0 commit comments

Comments
 (0)