From d0034e723bd033c099e0e74b92798152b898cd86 Mon Sep 17 00:00:00 2001 From: Thephaseless Date: Wed, 24 Jul 2024 15:16:46 +0000 Subject: [PATCH] added default logger --- main.py | 8 +++---- src/models/requests.py | 11 ++++++++-- src/utils/{misc.py => browser.py} | 8 ++++--- src/utils/consts.py | 2 ++ src/utils/extentions.py | 35 ++++++++++++++++++++++--------- src/utils/utils.py | 5 ----- 6 files changed, 44 insertions(+), 25 deletions(-) rename src/utils/{misc.py => browser.py} (73%) diff --git a/main.py b/main.py index d12c01b..f5c55e9 100644 --- a/main.py +++ b/main.py @@ -10,8 +10,8 @@ from fastapi.responses import RedirectResponse from src.models.requests import LinkRequest, LinkResponse from src.utils import logger +from src.utils.browser import bypass_cloudflare, new_browser from src.utils.extentions import download_extentions -from src.utils.misc import bypass_cloudflare, new_browser download_extentions() app = FastAPI(debug=True) @@ -35,10 +35,8 @@ async def read_item(request: LinkRequest): challenged = await asyncio.wait_for( bypass_cloudflare(page), timeout=request.maxTimeout ) - if challenged: - logger.info("Challenged") - else: - logger.info("Not challenged") + + logger.info(f"Got webpage: {request.url}") response = await LinkResponse.create( page=page, diff --git a/src/models/requests.py b/src/models/requests.py index 35b1f06..2b1ef60 100644 --- a/src/models/requests.py +++ b/src/models/requests.py @@ -5,6 +5,7 @@ import time from typing import Any from nodriver import Tab +from nodriver.cdp import storage from pydantic import BaseModel @@ -55,11 +56,13 @@ class LinkResponse(BaseModel): # # Convert cookies to json # cookies = [cookie.to_json() for cookie in cookies] - cookies = [] + cookies = storage.get_cookies() + + cookies = await page.send(cookies) solution = Solution( url=page.url, status=200, - cookies=cookies, + cookies=cookies if cookies else [], userAgent=user_agent, headers={}, response=await page.get_content(), @@ -70,3 +73,7 @@ class LinkResponse(BaseModel): solution=solution, startTimestamp=start_timestamp, ) + + +class NoChromeExtentionError(Exception): + """No chrome extention found.""" diff --git a/src/utils/misc.py b/src/utils/browser.py similarity index 73% rename from src/utils/misc.py rename to src/utils/browser.py index 729f700..cd44ada 100644 --- a/src/utils/misc.py +++ b/src/utils/browser.py @@ -2,12 +2,14 @@ import nodriver as webdriver from nodriver.core.element import Element from src.utils import logger +from src.utils.extentions import downloaded_extentions from src.utils.utils import CHALLENGE_TITLES async def new_browser(): config: webdriver.Config = webdriver.Config() config.sandbox = False + config.add_argument(f"--load-extension={','.join(downloaded_extentions)}") return await webdriver.start(config=config) @@ -15,13 +17,13 @@ async def new_browser(): async def bypass_cloudflare(page: webdriver.Tab): challenged = False while True: - logger.info("Bypassing cloudflare") await page await page.wait(0.5) if page.target.title not in CHALLENGE_TITLES: return challenged - challenged = True + if not challenged: + logger.info("Found challenge") + challenged = True elem = await page.query_selector(".cf-turnstile-wrapper") if isinstance(elem, Element): - logger.info(f"Clicking element: {elem}") await elem.mouse_click() diff --git a/src/utils/consts.py b/src/utils/consts.py index 610276a..f222974 100644 --- a/src/utils/consts.py +++ b/src/utils/consts.py @@ -11,3 +11,5 @@ EXTENTION_REPOSITIORIES = [ ] SLEEP_SECONDS = 1 + +EXTENTIONS_PATH = "./.extentions" diff --git a/src/utils/extentions.py b/src/utils/extentions.py index b761d2e..847bf9d 100644 --- a/src/utils/extentions.py +++ b/src/utils/extentions.py @@ -1,12 +1,15 @@ from __future__ import annotations +import json from pathlib import Path from zipfile import ZipFile import httpx from src.models.github import GithubResponse -from src.utils.consts import EXTENTION_REPOSITIORIES +from src.models.requests import NoChromeExtentionError +from src.utils import logger +from src.utils.consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH downloaded_extentions: list[str] = [] # Do not modify, use download_extentions() @@ -15,27 +18,39 @@ def get_latest_github_release(url: str): """Get the latest release from GitHub.""" url = "https://api.github.com/repos/" + url url += "/releases/latest" - response = GithubResponse.model_validate_json(httpx.get(url).text) + + response = httpx.get(url) + if response.status_code == httpx.codes.FORBIDDEN: + error = json.loads(response.text)["message"] + logger.error(error) + raise httpx.NetworkError(error) + response = GithubResponse(**response.json()) for asset in response.assets: if "chrom" in asset.name: return asset - raise ValueError("Couldn't find chrome verions of the release") + raise NoChromeExtentionError def download_extentions(): """Download the extention.""" for repository in EXTENTION_REPOSITIORIES: - extention = get_latest_github_release(repository) extention_name = repository.split("/")[-1] + path = Path(f"{EXTENTIONS_PATH}/{extention_name}") + try: + extention = get_latest_github_release(repository) + except httpx.NetworkError: + if path.is_dir(): + downloaded_extentions.append(path.as_posix()) + continue response = httpx.get(extention.browser_download_url) - Path("extentions").mkdir(exist_ok=True) - if not Path(f"extentions/{extention.name}").exists(): - with Path(f"extentions/{extention.name}").open("wb") as f: + Path("{EXTENTIONS_PATH}").mkdir(exist_ok=True) + if not Path(f"{EXTENTIONS_PATH}/{extention.name}").is_file(): + with Path(f"{EXTENTIONS_PATH}/{extention.name}").open("wb") as f: f.write(response.content) - with ZipFile(f"extentions/{extention.name}", "r") as zip_obj: - zip_obj.extractall(f"extentions/{extention_name}") + with ZipFile(f"{EXTENTIONS_PATH}/{extention.name}", "r") as zip_obj: + zip_obj.extractall(f"{EXTENTIONS_PATH}/{extention_name}") - downloaded_extentions.append(f"extentions/{extention_name}") + downloaded_extentions.append(path.as_posix()) diff --git a/src/utils/utils.py b/src/utils/utils.py index bd76989..1d906fe 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -1,4 +1,3 @@ -import logging from venv import logger import nodriver as webdriver @@ -7,10 +6,6 @@ from nodriver.core.element import Element from src.utils.consts import CHALLENGE_TITLES -def get_logger(): - return logging.getLogger("uvicorn.error") - - async def new_browser(): config: webdriver.Config = webdriver.Config() config.sandbox = False