Merge pull request #37 from ThePhaseless/uv

Uv
This commit is contained in:
Jakub Orchowski 2024-12-10 23:50:34 +01:00 committed by GitHub
commit 629248d67e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 1529 additions and 2341 deletions

View File

@ -1,5 +1,6 @@
FROM python:3.13-bullseye FROM debian:latest
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb git
RUN curl -sSL https://install.python-poetry.org | python3 - RUN curl -LsSf https://astral.sh/uv/install.sh | sh
ENV PATH="/root/.local/bin:$PATH"

View File

@ -23,53 +23,7 @@ env:
IMAGE_NAME: ${{ github.repository }} IMAGE_NAME: ${{ github.repository }}
jobs: jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.13"
cache: "pip"
- name: Set up Poetry
run: pip install poetry
- name: Setup a local virtual environment (if no poetry.toml file)
run: |
poetry config virtualenvs.create true --local
poetry config virtualenvs.in-project true --local
- uses: actions/cache@v4
name: Define a cache for the virtual environment based on the dependencies lock file
with:
path: ./.venv
key: venv-${{ hashFiles('poetry.lock') }}
- name: Install dependencies
run: |
poetry install
sudo apt update
sudo apt install -y xvfb scrot python3-tk
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
sudo apt install -y ./google-chrome-stable_current_amd64.deb
rm ./google-chrome-stable_current_amd64.deb
- name: Run Ruff
run: poetry run ruff check .
- name: Run tests
run: poetry run pytest --retries 2 -n auto --xvfb
- name: Upload screenshots if tests fail
if: failure()
uses: actions/upload-artifact@v4
with:
path: screenshots
build: build:
needs: test
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
contents: read contents: read
@ -119,6 +73,16 @@ jobs:
type=raw,enable=${{ github.event_name != 'pull_request' && github.ref_name == 'main' }}, value=latest type=raw,enable=${{ github.event_name != 'pull_request' && github.ref_name == 'main' }}, value=latest
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Test
id: test
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
target: test
# Build and push Docker image with Buildx (don't push on PR) # Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action # https://github.com/docker/build-push-action
- name: Build and push Docker image - name: Build and push Docker image

View File

@ -1,35 +1,37 @@
FROM python:3.13-slim-bullseye FROM debian:bullseye-slim AS base
# Inspired by https://github.com/Hudrolax/uc-docker-alpine/ # Inspired by https://github.com/Hudrolax/uc-docker-alpine/
ARG GITHUB_BUILD=false ARG GITHUB_BUILD=false
ENV GITHUB_BUILD=${GITHUB_BUILD} ENV GITHUB_BUILD=${GITHUB_BUILD}
ARG VERSION
ENV VERSION=${VERSION}
ENV HOME=/root ENV HOME=/root
ENV \ ENV \
DEBIAN_FRONTEND=noninteractive \ DEBIAN_FRONTEND=noninteractive \
PYTHONUNBUFFERED=1 \ PYTHONUNBUFFERED=1 \
# prevents python creating .pyc files # prevents python creating .pyc files
PYTHONDONTWRITEBYTECODE=1 \ PYTHONDONTWRITEBYTECODE=1 \
# do not ask any interactive question
POETRY_NO_INTERACTION=1 \
POETRY_VIRTUALENVS_IN_PROJECT=true \
DISPLAY=:0 DISPLAY=:0
WORKDIR /app WORKDIR /app
EXPOSE 8191
RUN apt update &&\ RUN apt update &&\
apt install -y xvfb scrot python3-tk curl chromium chromium-driver apt install -y xvfb scrot python3-tk curl chromium chromium-driver
RUN curl -sSL https://install.python-poetry.org | python3 - RUN curl -LsSf https://astral.sh/uv/install.sh | sh
ENV PATH="${HOME}/.local/bin:$PATH" ENV PATH="${HOME}/.local/bin:$PATH"
COPY pyproject.toml poetry.lock ./ COPY pyproject.toml uv.lock ./
RUN poetry install RUN uv sync
COPY . . COPY . .
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
RUN cd .venv/lib/*/site-packages/seleniumbase/drivers && ln -s /usr/bin/chromedriver uc_driver RUN cd .venv/lib/*/site-packages/seleniumbase/drivers && ln -s /usr/bin/chromedriver uc_driver
FROM base AS test
RUN uv sync --group test
RUN uv run pytest --retries 2 -n auto --xvfb
FROM base
EXPOSE 8191
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
CMD ["./cmd.sh"] CMD ["./cmd.sh"]

View File

@ -14,14 +14,14 @@ An alternative to [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) a
## Troubleshooting ## Troubleshooting
1. Clone repo to the host that has the container has issues on. 1. Clone repo to the host that has the container has issues on.
2. Using vscode and `SSH extention`, connect to the host and open repo in it. 2. Using vscode and `SSH extension`, connect to the host and open repo in it.
3. Download `devcontainers` extention and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`) 3. Download `Dev Containers` extension and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
4. Forward port 6080 from devcontainer (port of noVNC server) to the host. 4. Forward port 6080 from devcontainer (port of noVNC server) to the host.
5. Open `http://localhost:6080` and connect to the virtual desktop. 5. Open `http://localhost:6080` and connect to the virtual desktop.
6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`. 6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`.
7. If chromium works, run (or debug) tests from VS Code. 7. If chromium works, run (or debug) tests from VS Code.
1. If code works, congrats! (/s) You are on your own. 1. If code works, congrats! (not really) You are on your own.
2. If it does not, try another host or network, try again and create issue about the problem. 2. If it does not, try on another host or network and create an issue if problem persists.
## Usage ## Usage
@ -37,11 +37,23 @@ services:
- "8191:8191" # Optional if needed to make make requests/check docs on host - "8191:8191" # Optional if needed to make make requests/check docs on host
``` ```
### Docker
```bash
docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
```
### Local
```bash
uv sync && ./cmd.sh
```
## Need help with / TODO ## Need help with / TODO
- [x] Slimming container (only ~650 MB now!) - [x] Slimming container (only ~650 MB now!)
- [x] Add more anti-bot challenges - [x] Add more anti-bot challenges
- [x] Add docstrings - [x] Add doc strings
- [x] Implement versioning - [x] Implement versioning
- [ ] Proxy support - [ ] Proxy support
- [x] Add more architectures support - [x] Add more architectures support

16
main.py
View File

@ -45,7 +45,7 @@ async def health_check():
@app.post("/v1") @app.post("/v1")
def read_item(request: LinkRequest): def read_item(request: LinkRequest) -> LinkResponse:
"""Handle POST requests.""" """Handle POST requests."""
start_time = int(time.time() * 1000) start_time = int(time.time() * 1000)
# request.url = "https://nowsecure.nl" # request.url = "https://nowsecure.nl"
@ -53,8 +53,8 @@ def read_item(request: LinkRequest):
response: LinkResponse response: LinkResponse
# start_time = int(time.time() * 1000) # start_time = int(time.time() * 1000)
try: with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb: try:
sb: BaseCase sb: BaseCase
sb.uc_open_with_reconnect(request.url) sb.uc_open_with_reconnect(request.url)
source = sb.get_page_source() source = sb.get_page_source()
@ -86,9 +86,13 @@ def read_item(request: LinkRequest):
), ),
startTimestamp=start_time, startTimestamp=start_time,
) )
except Exception as e: except Exception as e:
logger.error(f"Error: {e}") logger.error(f"Error: {e}")
raise HTTPException(status_code=500, detail="Unknown error, check logs") from e if sb.driver:
sb.driver.quit()
raise HTTPException(
status_code=500, detail="Unknown error, check logs"
) from e
return response return response

2238
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,38 +1,32 @@
[tool.poetry] # cspell:disable
[project]
authors = [{ name = "ThePhaseless", email = "kukubaorch@gmail.com" }]
license = { text = "LICENSE" }
requires-python = "<4.0,>=3.12"
name = "Byparr" name = "Byparr"
version = "0.1.0" version = "0.1.0"
description = "API for getting cookies for Cloudflare challenges" description = "API for getting cookies for Cloudflare challenges"
package-mode = false
authors = ["ThePhaseless <kukubaorch@gmail.com>"]
readme = "README.md" readme = "README.md"
license = "LICENSE" dependencies = [
repository = "https://github.com/ThePhaseless/Byparr" "beautifulsoup4>=4.12.3",
"fastapi[standard]>=0.115.6",
"pyautogui>=0.9.54",
"pydantic>=2.10.3",
"seleniumbase>=4.33.7",
"uvicorn>=0.32.1",
]
urls = { repository = "https://github.com/ThePhaseless/Byparr" }
# cspell:disable [dependency-groups]
[tool.poetry.dependencies] test = [
python = "^3.12" "httpx>=0.28.1",
fastapi = { extras = ["standard"], version = "^0" } "pytest>=8.3.4",
seleniumbase = "^4.33.1" "pytest-asyncio>=0.24.0",
beautifulsoup4 = "^4.12.3" "pytest-progress>=1.3.0",
uvicorn = "^0.32.1" "pytest-retry>=1.6.3",
pydantic = "^2.10.2" "pytest-xdist>=3.6.1",
pyautogui = "^0.9.54" ]
dev = ["deptry>=0.21.1", "ruff>=0.8.2"]
[tool.poetry.group.dev.dependencies]
ruff = "^0.8.0"
deptry = "^0.21.1"
[tool.poetry.group.test.dependencies]
pytest = "^8.3.3"
httpx = "^0.28.0"
pytest-retry = "^1.6.3"
pytest-progress = "^1.3.0"
pytest-asyncio = "^0"
pytest-xdist = "^3.6.1"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.deptry.per_rule_ignores] [tool.deptry.per_rule_ignores]
DEP002 = ["pyautogui"] DEP002 = ["pyautogui"]

View File

@ -4,7 +4,6 @@
"packageRules": [ "packageRules": [
{ {
"automerge": true, "automerge": true,
"matchPackageNames": ["^(?!.*\bpython\b).*"],
"matchUpdateTypes": ["minor", "patch"] "matchUpdateTypes": ["minor", "patch"]
} }
] ]

1450
uv.lock generated Normal file

File diff suppressed because it is too large Load Diff