diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/database/mongo.py | 12 | ||||
| -rw-r--r-- | scripts/recipes/handle_recipes.py | 9 | ||||
| -rw-r--r-- | scripts/scraping/scraper.py | 84 | ||||
| -rw-r--r-- | scripts/users/handle_users.py | 18 |
4 files changed, 78 insertions, 45 deletions
diff --git a/scripts/database/mongo.py b/scripts/database/mongo.py index cc4dffa..3e97b0b 100644 --- a/scripts/database/mongo.py +++ b/scripts/database/mongo.py @@ -1,5 +1,9 @@ -from pymongo import MongoClient +from pymongo.mongo_client import MongoClient +from pymongo.server_api import ServerApi +import Constants +import certifi -# Monogo -mongo_client = MongoClient("mongodb://localhost:27017/") -mongo_database = mongo_client["recipedb"] +uri = Constants.MONGO_CONNECTION_STRING + +client = MongoClient(uri, server_api=ServerApi('1'), tlsCAFile=certifi.where()) +mongo_database = client["index-cooking"]
\ No newline at end of file diff --git a/scripts/recipes/handle_recipes.py b/scripts/recipes/handle_recipes.py index 76faf86..22ba0a4 100644 --- a/scripts/recipes/handle_recipes.py +++ b/scripts/recipes/handle_recipes.py @@ -2,7 +2,7 @@ from bson.objectid import ObjectId from scripts.database.mongo import mongo_database from scripts.scraping.scraper import scrape -mongo_collection = mongo_database["recipes"] +mongo_collection = mongo_database.get_collection("recipes") def add_single_recipe(recipe): added_recipe = mongo_collection.insert_one(recipe) @@ -31,6 +31,13 @@ def delete_single_recipe(objectId): except: return {"success": False, "error": "Could not delete recipe"} +def delete_multiple_recipes(user): + try: + mongo_collection.delete_many({"user": user}) + return {"success": True} + except: + return {"success": False} + def search_recipes(query_data, user): query = query_data["q"] facets_list = [] diff --git a/scripts/scraping/scraper.py b/scripts/scraping/scraper.py index 50b46ae..8919d46 100644 --- a/scripts/scraping/scraper.py +++ b/scripts/scraping/scraper.py @@ -3,33 +3,59 @@ from urllib import parse import json from bs4 import BeautifulSoup +def extractInstructions(instructions): + returnedInstructions = [] + for inst in instructions: + + # If there are sub sections, parse them + if "itemListElement" in inst: + print("hitting sub") + subObj = {} + subInst = [] + subObj["name"] = inst["name"] + for el in inst["itemListElement"]: + subInst.append(el["text"]) + subObj["subInstructions"] = subInst + returnedInstructions.append(subObj) + + # Else we have a shallower array of instructions + else: + returnedInstructions.append(inst["text"]) + + return returnedInstructions + + def scrape(url, user_name): - data = requests.get(url, headers= {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}) - html = BeautifulSoup(data.text, 'html.parser') - inner_html = html.find('script', class_='yoast-schema-graph') - json_data = json.loads(inner_html.contents[0]) - graph_data = json_data["@graph"] - for i in graph_data: - if(i["@type"] == "Recipe"): - recipe = {} - instructions = [] - for instruction in i["recipeInstructions"]: - instructions.append(instruction["text"]) - keywords_list = i["keywords"].split(",") - tags = i["recipeCuisine"] + keywords_list - cleaned_tags = list(set([tag.strip().lower() for tag in tags])) - slug = parse.quote(i["name"]).lower() - - # The recipe - recipe["user"] = user_name - recipe["slug"] = slug - recipe["title"] = i["name"] - recipe["image"] = i["image"][0] - recipe["url"] = i["mainEntityOfPage"] - recipe["tags"] = cleaned_tags - recipe["ingredients"] = i["recipeIngredient"] - recipe["instructions"] = instructions - recipe["visible_by"] = ["jez"] - # recipe["encoded_url"] = urllib.parse.quote(i["name"]) - # Complete this all later!! - return recipe + try: + data = requests.get(url, headers= {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}) + html = BeautifulSoup(data.text, 'html.parser') + inner_html = html.find('script', class_='yoast-schema-graph') + json_data = json.loads(inner_html.contents[0]) + graph_data = json_data["@graph"] + # print(graph_data, "graph data") + for i in graph_data: + if(i["@type"] == "Recipe"): + recipe = {} + instructions = extractInstructions(i["recipeInstructions"]) + keywords_list = i["keywords"].split(",") + tags = i["recipeCuisine"] + keywords_list + cleaned_tags = list(set([tag.strip().lower() for tag in tags])) + slug = parse.quote(i["name"]).lower() + + # The recipe + recipe["user"] = user_name + recipe["slug"] = slug + recipe["title"] = i["name"] + recipe["image"] = i["image"][0] + recipe["url"] = i["mainEntityOfPage"] + recipe["tags"] = cleaned_tags + recipe["ingredients"] = i["recipeIngredient"] + recipe["instructions"] = instructions + recipe["visible_by"] = ["jez"] + # recipe["encoded_url"] = urllib.parse.quote(i["name"]) + + return {"success": True, "data": recipe} + + except Exception as e: + print(str(e)) + return {"success": False, "error": "couldn't scrape site metadata"} diff --git a/scripts/users/handle_users.py b/scripts/users/handle_users.py index 87ff7cd..250c716 100644 --- a/scripts/users/handle_users.py +++ b/scripts/users/handle_users.py @@ -1,5 +1,5 @@ from scripts.database.mongo import mongo_database -mongo_collection = mongo_database["users"] +mongo_collection = mongo_database.get_collection("users") def user_exists(user): try: @@ -11,16 +11,6 @@ def user_exists(user): except: return {"success": False, "error": "Request error, try again"} -def authenticate_user(user, password_matches): - try: - res = mongo_collection.find_one({"name": user}) - if res is None: - return {"success": False, "error": "Password doesn't match, try again"} - else: - return {"success": True, "user": user} - except: - return {"success": False, "error": "Something went wrong matching your password"} - def add_user(user, password): try: res = mongo_collection.insert_one({"name": user, "password": password}) @@ -28,3 +18,9 @@ def add_user(user, password): except: print("error") +def delete_single_user(user): + try: + mongo_collection.delete_one({"name": user}) + return {"success": True, "user": user} + except: + return {"success": False, "user": user}
\ No newline at end of file |