From d0034e723bd033c099e0e74b92798152b898cd86 Mon Sep 17 00:00:00 2001
From: Thephaseless <kukubaorch@gmail.com>
Date: Wed, 24 Jul 2024 15:16:46 +0000
Subject: [PATCH] added default logger
---
main.py | 8 +++----
src/models/requests.py | 11 ++++++++--
src/utils/{misc.py => browser.py} | 8 ++++---
src/utils/consts.py | 2 ++
src/utils/extentions.py | 35 ++++++++++++++++++++++---------
src/utils/utils.py | 5 -----
6 files changed, 44 insertions(+), 25 deletions(-)
rename src/utils/{misc.py => browser.py} (73%)
diff --git a/main.py b/main.py
index d12c01b..f5c55e9 100644
--- a/main.py
+++ b/main.py
@@ -10,8 +10,8 @@ from fastapi.responses import RedirectResponse
from src.models.requests import LinkRequest, LinkResponse
from src.utils import logger
+from src.utils.browser import bypass_cloudflare, new_browser
from src.utils.extentions import download_extentions
-from src.utils.misc import bypass_cloudflare, new_browser
download_extentions()
app = FastAPI(debug=True)
@@ -35,10 +35,8 @@ async def read_item(request: LinkRequest):
challenged = await asyncio.wait_for(
bypass_cloudflare(page), timeout=request.maxTimeout
)
- if challenged:
- logger.info("Challenged")
- else:
- logger.info("Not challenged")
+
+ logger.info(f"Got webpage: {request.url}")
response = await LinkResponse.create(
page=page,
diff --git a/src/models/requests.py b/src/models/requests.py
index 35b1f06..2b1ef60 100644
--- a/src/models/requests.py
+++ b/src/models/requests.py
@@ -5,6 +5,7 @@ import time
from typing import Any
from nodriver import Tab
+from nodriver.cdp import storage
from pydantic import BaseModel
@@ -55,11 +56,13 @@ class LinkResponse(BaseModel):
# # Convert cookies to json
# cookies = [cookie.to_json() for cookie in cookies]
- cookies = []
+ cookies = storage.get_cookies()
+
+ cookies = await page.send(cookies)
solution = Solution(
url=page.url,
status=200,
- cookies=cookies,
+ cookies=cookies if cookies else [],
userAgent=user_agent,
headers={},
response=await page.get_content(),
@@ -70,3 +73,7 @@ class LinkResponse(BaseModel):
solution=solution,
startTimestamp=start_timestamp,
)
+
+
+class NoChromeExtentionError(Exception):
+ """No chrome extention found."""
diff --git a/src/utils/misc.py b/src/utils/browser.py
similarity index 73%
rename from src/utils/misc.py
rename to src/utils/browser.py
index 729f700..cd44ada 100644
--- a/src/utils/misc.py
+++ b/src/utils/browser.py
@@ -2,12 +2,14 @@ import nodriver as webdriver
from nodriver.core.element import Element
from src.utils import logger
+from src.utils.extentions import downloaded_extentions
from src.utils.utils import CHALLENGE_TITLES
async def new_browser():
config: webdriver.Config = webdriver.Config()
config.sandbox = False
+ config.add_argument(f"--load-extension={','.join(downloaded_extentions)}")
return await webdriver.start(config=config)
@@ -15,13 +17,13 @@ async def new_browser():
async def bypass_cloudflare(page: webdriver.Tab):
challenged = False
while True:
- logger.info("Bypassing cloudflare")
await page
await page.wait(0.5)
if page.target.title not in CHALLENGE_TITLES:
return challenged
- challenged = True
+ if not challenged:
+ logger.info("Found challenge")
+ challenged = True
elem = await page.query_selector(".cf-turnstile-wrapper")
if isinstance(elem, Element):
- logger.info(f"Clicking element: {elem}")
await elem.mouse_click()
diff --git a/src/utils/consts.py b/src/utils/consts.py
index 610276a..f222974 100644
--- a/src/utils/consts.py
+++ b/src/utils/consts.py
@@ -11,3 +11,5 @@ EXTENTION_REPOSITIORIES = [
]
SLEEP_SECONDS = 1
+
+EXTENTIONS_PATH = "./.extentions"
diff --git a/src/utils/extentions.py b/src/utils/extentions.py
index b761d2e..847bf9d 100644
--- a/src/utils/extentions.py
+++ b/src/utils/extentions.py
@@ -1,12 +1,15 @@
from __future__ import annotations
+import json
from pathlib import Path
from zipfile import ZipFile
import httpx
from src.models.github import GithubResponse
-from src.utils.consts import EXTENTION_REPOSITIORIES
+from src.models.requests import NoChromeExtentionError
+from src.utils import logger
+from src.utils.consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH
downloaded_extentions: list[str] = [] # Do not modify, use download_extentions()
@@ -15,27 +18,39 @@ def get_latest_github_release(url: str):
"""Get the latest release from GitHub."""
url = "https://api.github.com/repos/" + url
url += "/releases/latest"
- response = GithubResponse.model_validate_json(httpx.get(url).text)
+
+ response = httpx.get(url)
+ if response.status_code == httpx.codes.FORBIDDEN:
+ error = json.loads(response.text)["message"]
+ logger.error(error)
+ raise httpx.NetworkError(error)
+ response = GithubResponse(**response.json())
for asset in response.assets:
if "chrom" in asset.name:
return asset
- raise ValueError("Couldn't find chrome verions of the release")
+ raise NoChromeExtentionError
def download_extentions():
"""Download the extention."""
for repository in EXTENTION_REPOSITIORIES:
- extention = get_latest_github_release(repository)
extention_name = repository.split("/")[-1]
+ path = Path(f"{EXTENTIONS_PATH}/{extention_name}")
+ try:
+ extention = get_latest_github_release(repository)
+ except httpx.NetworkError:
+ if path.is_dir():
+ downloaded_extentions.append(path.as_posix())
+ continue
response = httpx.get(extention.browser_download_url)
- Path("extentions").mkdir(exist_ok=True)
- if not Path(f"extentions/{extention.name}").exists():
- with Path(f"extentions/{extention.name}").open("wb") as f:
+ Path("{EXTENTIONS_PATH}").mkdir(exist_ok=True)
+ if not Path(f"{EXTENTIONS_PATH}/{extention.name}").is_file():
+ with Path(f"{EXTENTIONS_PATH}/{extention.name}").open("wb") as f:
f.write(response.content)
- with ZipFile(f"extentions/{extention.name}", "r") as zip_obj:
- zip_obj.extractall(f"extentions/{extention_name}")
+ with ZipFile(f"{EXTENTIONS_PATH}/{extention.name}", "r") as zip_obj:
+ zip_obj.extractall(f"{EXTENTIONS_PATH}/{extention_name}")
- downloaded_extentions.append(f"extentions/{extention_name}")
+ downloaded_extentions.append(path.as_posix())
diff --git a/src/utils/utils.py b/src/utils/utils.py
index bd76989..1d906fe 100644
--- a/src/utils/utils.py
+++ b/src/utils/utils.py
@@ -1,4 +1,3 @@
-import logging
from venv import logger
import nodriver as webdriver
@@ -7,10 +6,6 @@ from nodriver.core.element import Element
from src.utils.consts import CHALLENGE_TITLES
-def get_logger():
- return logging.getLogger("uvicorn.error")
-
-
async def new_browser():
config: webdriver.Config = webdriver.Config()
config.sandbox = False