scraper route update
This commit is contained in:
parent
bc9e5692fa
commit
8c1b33ad0b
2 changed files with 9 additions and 116 deletions
|
@ -24,7 +24,7 @@ async def get_all_recipes(
|
|||
async def get_recipe(recipe_slug: str):
|
||||
""" Takes in a recipe slug, returns all data for a recipe """
|
||||
recipe = Recipe.get_by_slug(recipe_slug)
|
||||
|
||||
|
||||
return recipe
|
||||
|
||||
|
||||
|
@ -42,14 +42,14 @@ async def get_recipe_url(url: dict):
|
|||
""" Takes in a URL and Attempts to scrape data and load it into the database """
|
||||
|
||||
url = url.get("url")
|
||||
slug = create_from_url(url)
|
||||
|
||||
|
||||
try:
|
||||
slug = create_from_url(url)
|
||||
except:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=SnackResponse.error("Unable to Parse URL")
|
||||
)
|
||||
# try:
|
||||
# slug = create_from_url(url)
|
||||
# except:
|
||||
# raise HTTPException(
|
||||
# status_code=400, detail=SnackResponse.error("Unable to Parse URL")
|
||||
# )
|
||||
|
||||
return slug
|
||||
|
||||
|
@ -75,9 +75,8 @@ def update_image(
|
|||
@router.post("/api/recipe/{recipe_slug}/update/", tags=["Recipes"])
|
||||
async def update(recipe_slug: str, data: Recipe):
|
||||
""" Updates a recipe by existing slug and data. Data should containt """
|
||||
|
||||
data.update(recipe_slug)
|
||||
|
||||
data.update(recipe_slug)
|
||||
|
||||
return {"message": "PLACEHOLDER"}
|
||||
|
||||
|
|
|
@ -1,106 +0,0 @@
|
|||
import shutil
|
||||
from os.path import join
|
||||
from pathlib import Path
|
||||
from pprint import pprint
|
||||
|
||||
import git
|
||||
import yaml
|
||||
from git.util import join_path
|
||||
|
||||
from services.image_services import IMG_DIR
|
||||
from services.recipe_services import Recipe
|
||||
|
||||
try:
|
||||
from yaml import CDumper as Dumper
|
||||
from yaml import CLoader as Loader
|
||||
except ImportError:
|
||||
from yaml import Dumper, Loader
|
||||
|
||||
CWD = Path(__file__).parent
|
||||
|
||||
file = f"/home/hayden/Projects/mealie-fastAPI/mealie/chowdown.md"
|
||||
|
||||
repo = "https://github.com/clarklab/chowdown"
|
||||
|
||||
|
||||
def pull_repo(repo):
|
||||
dest_dir = CWD.joinpath("data/temp/migration/git_pull")
|
||||
if dest_dir.exists():
|
||||
shutil.rmtree(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
git.Git(dest_dir).clone(repo)
|
||||
|
||||
repo_name = repo.split("/")[-1]
|
||||
recipe_dir = dest_dir.joinpath(repo_name, "_recipes")
|
||||
image_dir = dest_dir.joinpath(repo_name, "images")
|
||||
|
||||
return recipe_dir, image_dir
|
||||
|
||||
|
||||
def read_chowdown_file(recipe_file: Path) -> Recipe:
|
||||
with open(recipe_file, "r") as stream:
|
||||
recipe_description: str = str
|
||||
recipe_data: dict = {}
|
||||
try:
|
||||
for x, item in enumerate(yaml.load_all(stream, Loader=Loader)):
|
||||
print(item)
|
||||
if x == 0:
|
||||
recipe_data = item
|
||||
|
||||
elif x == 1:
|
||||
recipe_description = str(item)
|
||||
|
||||
except yaml.YAMLError as exc:
|
||||
print(exc)
|
||||
return
|
||||
|
||||
reformat_data = {
|
||||
"name": recipe_data.get("title"),
|
||||
"description": recipe_description,
|
||||
"image": recipe_data.get("image", ""),
|
||||
"recipeIngredient": recipe_data.get("ingredients"),
|
||||
"recipeInstructions": recipe_data.get("directions"),
|
||||
"tags": recipe_data.get("tags").split(","),
|
||||
}
|
||||
|
||||
pprint(reformat_data)
|
||||
new_recipe = Recipe(**reformat_data)
|
||||
|
||||
reformated_list = []
|
||||
for instruction in new_recipe.recipeInstructions:
|
||||
reformated_list.append({"text": instruction})
|
||||
|
||||
new_recipe.recipeInstructions = reformated_list
|
||||
|
||||
return new_recipe
|
||||
|
||||
|
||||
def main():
|
||||
from db.mongo_setup import global_init
|
||||
|
||||
global_init()
|
||||
recipe_dir, image_dir = pull_repo(repo)
|
||||
|
||||
failed_images = []
|
||||
for image in image_dir.iterdir():
|
||||
try:
|
||||
image.rename(IMG_DIR.joinpath(image.name))
|
||||
except:
|
||||
failed_images.append(image.name)
|
||||
|
||||
failed_recipes = []
|
||||
for recipe in recipe_dir.glob("*.md"):
|
||||
print(recipe.name)
|
||||
try:
|
||||
new_recipe = read_chowdown_file(recipe)
|
||||
new_recipe.save_to_db()
|
||||
|
||||
except:
|
||||
failed_recipes.append(recipe.name)
|
||||
|
||||
report = {"failedImages": failed_images, "failedRecipes": failed_recipes}
|
||||
|
||||
print(report)
|
||||
|
||||
|
||||
main()
|
Loading…
Reference in a new issue