mirror of
https://github.com/ThePhaseless/Byparr.git
synced 2025-03-15 09:50:20 +08:00
commit
629248d67e
@ -1,5 +1,6 @@
|
||||
FROM python:3.13-bullseye
|
||||
FROM debian:latest
|
||||
|
||||
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb
|
||||
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb git
|
||||
|
||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
56
.github/workflows/docker-publish.yml
vendored
56
.github/workflows/docker-publish.yml
vendored
@ -23,53 +23,7 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
cache: "pip"
|
||||
|
||||
- name: Set up Poetry
|
||||
run: pip install poetry
|
||||
|
||||
- name: Setup a local virtual environment (if no poetry.toml file)
|
||||
run: |
|
||||
poetry config virtualenvs.create true --local
|
||||
poetry config virtualenvs.in-project true --local
|
||||
|
||||
- uses: actions/cache@v4
|
||||
name: Define a cache for the virtual environment based on the dependencies lock file
|
||||
with:
|
||||
path: ./.venv
|
||||
key: venv-${{ hashFiles('poetry.lock') }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
poetry install
|
||||
sudo apt update
|
||||
sudo apt install -y xvfb scrot python3-tk
|
||||
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
|
||||
sudo apt install -y ./google-chrome-stable_current_amd64.deb
|
||||
rm ./google-chrome-stable_current_amd64.deb
|
||||
|
||||
- name: Run Ruff
|
||||
run: poetry run ruff check .
|
||||
|
||||
- name: Run tests
|
||||
run: poetry run pytest --retries 2 -n auto --xvfb
|
||||
|
||||
- name: Upload screenshots if tests fail
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
path: screenshots
|
||||
|
||||
build:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@ -119,6 +73,16 @@ jobs:
|
||||
type=raw,enable=${{ github.event_name != 'pull_request' && github.ref_name == 'main' }}, value=latest
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
|
||||
- name: Test
|
||||
id: test
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
target: test
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
|
26
Dockerfile
26
Dockerfile
@ -1,35 +1,37 @@
|
||||
FROM python:3.13-slim-bullseye
|
||||
FROM debian:bullseye-slim AS base
|
||||
|
||||
# Inspired by https://github.com/Hudrolax/uc-docker-alpine/
|
||||
|
||||
ARG GITHUB_BUILD=false
|
||||
ENV GITHUB_BUILD=${GITHUB_BUILD}
|
||||
|
||||
ARG VERSION
|
||||
ENV VERSION=${VERSION}
|
||||
|
||||
ENV HOME=/root
|
||||
ENV \
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
# prevents python creating .pyc files
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
# do not ask any interactive question
|
||||
POETRY_NO_INTERACTION=1 \
|
||||
POETRY_VIRTUALENVS_IN_PROJECT=true \
|
||||
DISPLAY=:0
|
||||
|
||||
WORKDIR /app
|
||||
EXPOSE 8191
|
||||
RUN apt update &&\
|
||||
apt install -y xvfb scrot python3-tk curl chromium chromium-driver
|
||||
|
||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
ENV PATH="${HOME}/.local/bin:$PATH"
|
||||
COPY pyproject.toml poetry.lock ./
|
||||
RUN poetry install
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN uv sync
|
||||
|
||||
COPY . .
|
||||
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
|
||||
RUN cd .venv/lib/*/site-packages/seleniumbase/drivers && ln -s /usr/bin/chromedriver uc_driver
|
||||
|
||||
FROM base AS test
|
||||
|
||||
RUN uv sync --group test
|
||||
RUN uv run pytest --retries 2 -n auto --xvfb
|
||||
|
||||
FROM base
|
||||
|
||||
EXPOSE 8191
|
||||
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
|
||||
CMD ["./cmd.sh"]
|
22
README.md
22
README.md
@ -14,14 +14,14 @@ An alternative to [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) a
|
||||
## Troubleshooting
|
||||
|
||||
1. Clone repo to the host that has the container has issues on.
|
||||
2. Using vscode and `SSH extention`, connect to the host and open repo in it.
|
||||
3. Download `devcontainers` extention and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
|
||||
2. Using vscode and `SSH extension`, connect to the host and open repo in it.
|
||||
3. Download `Dev Containers` extension and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
|
||||
4. Forward port 6080 from devcontainer (port of noVNC server) to the host.
|
||||
5. Open `http://localhost:6080` and connect to the virtual desktop.
|
||||
6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`.
|
||||
7. If chromium works, run (or debug) tests from VS Code.
|
||||
1. If code works, congrats! (/s) You are on your own.
|
||||
2. If it does not, try another host or network, try again and create issue about the problem.
|
||||
1. If code works, congrats! (not really) You are on your own.
|
||||
2. If it does not, try on another host or network and create an issue if problem persists.
|
||||
|
||||
## Usage
|
||||
|
||||
@ -37,11 +37,23 @@ services:
|
||||
- "8191:8191" # Optional if needed to make make requests/check docs on host
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
|
||||
```
|
||||
|
||||
### Local
|
||||
|
||||
```bash
|
||||
uv sync && ./cmd.sh
|
||||
```
|
||||
|
||||
## Need help with / TODO
|
||||
|
||||
- [x] Slimming container (only ~650 MB now!)
|
||||
- [x] Add more anti-bot challenges
|
||||
- [x] Add docstrings
|
||||
- [x] Add doc strings
|
||||
- [x] Implement versioning
|
||||
- [ ] Proxy support
|
||||
- [x] Add more architectures support
|
||||
|
10
main.py
10
main.py
@ -45,7 +45,7 @@ async def health_check():
|
||||
|
||||
|
||||
@app.post("/v1")
|
||||
def read_item(request: LinkRequest):
|
||||
def read_item(request: LinkRequest) -> LinkResponse:
|
||||
"""Handle POST requests."""
|
||||
start_time = int(time.time() * 1000)
|
||||
# request.url = "https://nowsecure.nl"
|
||||
@ -53,8 +53,8 @@ def read_item(request: LinkRequest):
|
||||
response: LinkResponse
|
||||
|
||||
# start_time = int(time.time() * 1000)
|
||||
try:
|
||||
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
|
||||
try:
|
||||
sb: BaseCase
|
||||
sb.uc_open_with_reconnect(request.url)
|
||||
source = sb.get_page_source()
|
||||
@ -88,7 +88,11 @@ def read_item(request: LinkRequest):
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Unknown error, check logs") from e
|
||||
if sb.driver:
|
||||
sb.driver.quit()
|
||||
raise HTTPException(
|
||||
status_code=500, detail="Unknown error, check logs"
|
||||
) from e
|
||||
|
||||
return response
|
||||
|
||||
|
2238
poetry.lock
generated
2238
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@ -1,38 +1,32 @@
|
||||
[tool.poetry]
|
||||
# cspell:disable
|
||||
[project]
|
||||
authors = [{ name = "ThePhaseless", email = "kukubaorch@gmail.com" }]
|
||||
license = { text = "LICENSE" }
|
||||
requires-python = "<4.0,>=3.12"
|
||||
name = "Byparr"
|
||||
version = "0.1.0"
|
||||
description = "API for getting cookies for Cloudflare challenges"
|
||||
package-mode = false
|
||||
authors = ["ThePhaseless <kukubaorch@gmail.com>"]
|
||||
readme = "README.md"
|
||||
license = "LICENSE"
|
||||
repository = "https://github.com/ThePhaseless/Byparr"
|
||||
dependencies = [
|
||||
"beautifulsoup4>=4.12.3",
|
||||
"fastapi[standard]>=0.115.6",
|
||||
"pyautogui>=0.9.54",
|
||||
"pydantic>=2.10.3",
|
||||
"seleniumbase>=4.33.7",
|
||||
"uvicorn>=0.32.1",
|
||||
]
|
||||
urls = { repository = "https://github.com/ThePhaseless/Byparr" }
|
||||
|
||||
# cspell:disable
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.12"
|
||||
fastapi = { extras = ["standard"], version = "^0" }
|
||||
seleniumbase = "^4.33.1"
|
||||
beautifulsoup4 = "^4.12.3"
|
||||
uvicorn = "^0.32.1"
|
||||
pydantic = "^2.10.2"
|
||||
pyautogui = "^0.9.54"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
ruff = "^0.8.0"
|
||||
deptry = "^0.21.1"
|
||||
|
||||
[tool.poetry.group.test.dependencies]
|
||||
pytest = "^8.3.3"
|
||||
httpx = "^0.28.0"
|
||||
pytest-retry = "^1.6.3"
|
||||
pytest-progress = "^1.3.0"
|
||||
pytest-asyncio = "^0"
|
||||
pytest-xdist = "^3.6.1"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
[dependency-groups]
|
||||
test = [
|
||||
"httpx>=0.28.1",
|
||||
"pytest>=8.3.4",
|
||||
"pytest-asyncio>=0.24.0",
|
||||
"pytest-progress>=1.3.0",
|
||||
"pytest-retry>=1.6.3",
|
||||
"pytest-xdist>=3.6.1",
|
||||
]
|
||||
dev = ["deptry>=0.21.1", "ruff>=0.8.2"]
|
||||
|
||||
[tool.deptry.per_rule_ignores]
|
||||
DEP002 = ["pyautogui"]
|
||||
|
@ -4,7 +4,6 @@
|
||||
"packageRules": [
|
||||
{
|
||||
"automerge": true,
|
||||
"matchPackageNames": ["^(?!.*\bpython\b).*"],
|
||||
"matchUpdateTypes": ["minor", "patch"]
|
||||
}
|
||||
]
|
||||
|
Loading…
x
Reference in New Issue
Block a user