Skip to content

Commit 7d7a349

Browse files
TomBurschTomBursch
committed
Initial implementation
1 parent ec26408 commit 7d7a349

3 files changed

Lines changed: 206 additions & 102 deletions

File tree

‎backend/app/service/ingredient_parsing.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ def parseNLPSingle(ingredient: str) -> IngredientParsingResult:
5252
def parseLLM(
5353
ingredients: list[str], targetLanguageCode: str | None = None
5454
) -> list[IngredientParsingResult] | None:
55+
if not LLM_MODEL:
56+
return None
57+
5558
systemMessage = """
5659
You are a tool that returns only JSON in the form of [{"name": name, "description": description}, ...]. Split every string from the list into these two properties. You receive recipe ingredients and fill the name field with the singular name of the ingredient and everything else is the description. Translate the response into the specified language.
5760

‎backend/app/service/recipe_public_scraping.py‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
import json
2+
import re
3+
import os
4+
5+
from litellm import completion
6+
from recipe_scrapers import scrape_html
7+
from recipe_scrapers._exceptions import SchemaOrgException
8+
from app.service.ingredient_parsing import parseIngredients
9+
from app.models import Recipe, Item, Household
10+
from app.config import SUPPORTED_LANGUAGES
11+
12+
LLM_MODEL = os.getenv("LLM_MODEL")
13+
LLM_API_URL = os.getenv("LLM_API_URL")
14+
15+
16+
def scrapeHTML(url: str, html: str, household: Household) -> dict | None:
17+
try:
18+
scraper = scrape_html(html, url, supported_only=False)
19+
except Exception:
20+
return None
21+
recipe = Recipe()
22+
try:
23+
recipe.name = scraper.title().strip()[:128]
24+
except (
25+
NotImplementedError,
26+
ValueError,
27+
TypeError,
28+
AttributeError,
29+
SchemaOrgException,
30+
):
31+
return None # Unsupported if title cannot be scraped
32+
try:
33+
recipe.time = int(scraper.total_time())
34+
except (
35+
NotImplementedError,
36+
ValueError,
37+
TypeError,
38+
AttributeError,
39+
SchemaOrgException,
40+
):
41+
pass
42+
try:
43+
recipe.cook_time = int(scraper.cook_time())
44+
except (
45+
NotImplementedError,
46+
ValueError,
47+
TypeError,
48+
AttributeError,
49+
SchemaOrgException,
50+
):
51+
pass
52+
try:
53+
recipe.prep_time = int(scraper.prep_time())
54+
except (
55+
NotImplementedError,
56+
ValueError,
57+
TypeError,
58+
AttributeError,
59+
SchemaOrgException,
60+
):
61+
pass
62+
try:
63+
yields = re.search(r"\d*", scraper.yields())
64+
if yields:
65+
recipe.yields = int(yields.group())
66+
except (
67+
NotImplementedError,
68+
ValueError,
69+
TypeError,
70+
AttributeError,
71+
SchemaOrgException,
72+
):
73+
pass
74+
description = ""
75+
try:
76+
description = scraper.description() + "\n\n"
77+
except (
78+
NotImplementedError,
79+
ValueError,
80+
TypeError,
81+
AttributeError,
82+
SchemaOrgException,
83+
):
84+
pass
85+
try:
86+
description = description + scraper.instructions()
87+
except (
88+
NotImplementedError,
89+
ValueError,
90+
TypeError,
91+
AttributeError,
92+
SchemaOrgException,
93+
):
94+
pass
95+
recipe.description = description
96+
recipe.photo = scraper.image()
97+
recipe.source = url
98+
items = {}
99+
for ingredient in parseIngredients(scraper.ingredients(), household.language):
100+
name = ingredient.name if ingredient.name else ingredient.originalText or ""
101+
item = Item.find_name_starts_with(household.id, name)
102+
if item:
103+
items[ingredient.originalText] = item.obj_to_dict() | {
104+
"description": ingredient.description,
105+
"optional": False,
106+
}
107+
else:
108+
items[ingredient.originalText] = None
109+
return {
110+
"recipe": recipe.obj_to_dict(),
111+
"items": items,
112+
}
113+
114+
115+
def scrapeHTMLLLM(url: str, html: str, household: Household) -> dict | None:
116+
if not LLM_MODEL:
117+
return None
118+
119+
systemMessage = """
120+
You are a tool that returns only JSON and nothing else. You get a html page of a recipe as an input. You return the json in the form {"name": "recipe name", "photo": "url to photo", "description": "description with instructions in markdown", "yields": "servings count", "time": integer total cooking time in minutes, "ingredients" ["list of string ingrediends including amount name and description"]}.
121+
122+
For example a result in English:
123+
{
124+
"name": "Pumpkin Soup",
125+
"photo": "https://recipes.com/photos/1283010293.jpg",
126+
"description": "Delicious soup.
127+
- First wash the pumpkin
128+
- Cut Pumpkin
129+
- Make soup",
130+
"yields": 4,
131+
"time": 20,
132+
"ingredients": [
133+
"1 Pumpkin",
134+
"50g red Onions"
135+
]
136+
}
137+
138+
Return only JSON and nothing else.
139+
"""
140+
141+
messages = [
142+
{
143+
"role": "system",
144+
"content": systemMessage,
145+
}
146+
]
147+
if household.language in SUPPORTED_LANGUAGES:
148+
messages.append(
149+
{
150+
"role": "user",
151+
"content": f"Translate the response to {SUPPORTED_LANGUAGES[household.language]}. Translate the JSON content to {SUPPORTED_LANGUAGES[household.language]}. Your target language is {SUPPORTED_LANGUAGES[household.language]}. Respond in {SUPPORTED_LANGUAGES[household.language]} from the start.",
152+
}
153+
)
154+
155+
messages.append(
156+
{
157+
"role": "user",
158+
"content": html,
159+
}
160+
)
161+
162+
response = completion(
163+
model=LLM_MODEL,
164+
api_base=LLM_API_URL,
165+
# response_format={"type": "json_object"},
166+
messages=messages,
167+
)
168+
169+
print(response.choices[0].message.content)
170+
try:
171+
llmResponse = json.loads(response.choices[0].message.content)
172+
except Exception as e:
173+
print(e)
174+
return None
175+
176+
items = {}
177+
for ingredient in parseIngredients(llmResponse["ingredients"], household.language):
178+
name = ingredient.name if ingredient.name else ingredient.originalText or ""
179+
item = Item.find_name_starts_with(household.id, name)
180+
if item:
181+
items[ingredient.originalText] = item.obj_to_dict() | {
182+
"description": ingredient.description,
183+
"optional": False,
184+
}
185+
else:
186+
items[ingredient.originalText] = None
187+
188+
return {
189+
"recipe": llmResponse
190+
| {
191+
"id": None,
192+
"public": False,
193+
"source": url,
194+
},
195+
"items": items,
196+
}

‎backend/app/service/recipe_scraping.py‎

Lines changed: 7 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -1,113 +1,18 @@
11
import re
22
from typing import Any
3-
from recipe_scrapers import scrape_html
4-
from recipe_scrapers._exceptions import SchemaOrgException
53
import requests
64
from app.config import FRONT_URL
75
from app.errors import ForbiddenRequest
86
from app.models.recipe import RecipeVisibility
9-
from app.service.ingredient_parsing import parseIngredients
7+
from app.models import Recipe, Household
8+
from app.service.recipe_public_scraping import scrapeHTML, scrapeHTMLLLM
109

11-
from app.models import Recipe, Item, Household
1210

13-
14-
def scrapePublic(url: str, html: str, household: Household) -> dict[str, Any] | None:
15-
try:
16-
scraper = scrape_html(html, url, supported_only=False)
17-
except Exception:
18-
return None
19-
recipe = Recipe()
20-
try:
21-
recipe.name = scraper.title().strip()[:128]
22-
except (
23-
NotImplementedError,
24-
ValueError,
25-
TypeError,
26-
AttributeError,
27-
SchemaOrgException,
28-
):
29-
return None # Unsupported if title cannot be scraped
30-
try:
31-
recipe.time = int(scraper.total_time())
32-
except (
33-
NotImplementedError,
34-
ValueError,
35-
TypeError,
36-
AttributeError,
37-
SchemaOrgException,
38-
):
39-
pass
40-
try:
41-
recipe.cook_time = int(scraper.cook_time())
42-
except (
43-
NotImplementedError,
44-
ValueError,
45-
TypeError,
46-
AttributeError,
47-
SchemaOrgException,
48-
):
49-
pass
50-
try:
51-
recipe.prep_time = int(scraper.prep_time())
52-
except (
53-
NotImplementedError,
54-
ValueError,
55-
TypeError,
56-
AttributeError,
57-
SchemaOrgException,
58-
):
59-
pass
60-
try:
61-
yields = re.search(r"\d*", scraper.yields())
62-
if yields:
63-
recipe.yields = int(yields.group())
64-
except (
65-
NotImplementedError,
66-
ValueError,
67-
TypeError,
68-
AttributeError,
69-
SchemaOrgException,
70-
):
71-
pass
72-
description = ""
73-
try:
74-
description = scraper.description() + "\n\n"
75-
except (
76-
NotImplementedError,
77-
ValueError,
78-
TypeError,
79-
AttributeError,
80-
SchemaOrgException,
81-
):
82-
pass
83-
try:
84-
description = description + scraper.instructions()
85-
except (
86-
NotImplementedError,
87-
ValueError,
88-
TypeError,
89-
AttributeError,
90-
SchemaOrgException,
91-
):
92-
pass
93-
recipe.description = description
94-
recipe.photo = scraper.image()
95-
recipe.source = url
96-
items = {}
97-
for ingredient in parseIngredients(scraper.ingredients(), household.language):
98-
name = ingredient.name if ingredient.name else ingredient.originalText or ""
99-
item = Item.find_name_starts_with(household.id, name)
100-
if item:
101-
items[ingredient.originalText] = item.obj_to_dict() | {
102-
"description": ingredient.description,
103-
"optional": False,
104-
}
105-
else:
106-
items[ingredient.originalText] = None
107-
return {
108-
"recipe": recipe.obj_to_dict(),
109-
"items": items,
110-
}
11+
def scrapePublic(url: str, html: str, household: Household) -> dict | None:
12+
res = scrapeHTML(url, html, household)
13+
if not res:
14+
res = scrapeHTMLLLM(url, html, household)
15+
return res
11116

11217

11318
def scrapeLocal(recipe_id: int, household: Household):

0 commit comments

Comments
 (0)