mirror of
https://github.com/ThePhaseless/Byparr.git
synced 2025-03-15 09:50:20 +08:00
commit
629248d67e
@ -1,5 +1,6 @@
|
|||||||
FROM python:3.13-bullseye
|
FROM debian:latest
|
||||||
|
|
||||||
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb
|
RUN apt update && apt upgrade -y && apt install -y chromium chromium-driver xvfb git
|
||||||
|
|
||||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
|
ENV PATH="/root/.local/bin:$PATH"
|
56
.github/workflows/docker-publish.yml
vendored
56
.github/workflows/docker-publish.yml
vendored
@ -23,53 +23,7 @@ env:
|
|||||||
IMAGE_NAME: ${{ github.repository }}
|
IMAGE_NAME: ${{ github.repository }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: "3.13"
|
|
||||||
cache: "pip"
|
|
||||||
|
|
||||||
- name: Set up Poetry
|
|
||||||
run: pip install poetry
|
|
||||||
|
|
||||||
- name: Setup a local virtual environment (if no poetry.toml file)
|
|
||||||
run: |
|
|
||||||
poetry config virtualenvs.create true --local
|
|
||||||
poetry config virtualenvs.in-project true --local
|
|
||||||
|
|
||||||
- uses: actions/cache@v4
|
|
||||||
name: Define a cache for the virtual environment based on the dependencies lock file
|
|
||||||
with:
|
|
||||||
path: ./.venv
|
|
||||||
key: venv-${{ hashFiles('poetry.lock') }}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: |
|
|
||||||
poetry install
|
|
||||||
sudo apt update
|
|
||||||
sudo apt install -y xvfb scrot python3-tk
|
|
||||||
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
|
|
||||||
sudo apt install -y ./google-chrome-stable_current_amd64.deb
|
|
||||||
rm ./google-chrome-stable_current_amd64.deb
|
|
||||||
|
|
||||||
- name: Run Ruff
|
|
||||||
run: poetry run ruff check .
|
|
||||||
|
|
||||||
- name: Run tests
|
|
||||||
run: poetry run pytest --retries 2 -n auto --xvfb
|
|
||||||
|
|
||||||
- name: Upload screenshots if tests fail
|
|
||||||
if: failure()
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
path: screenshots
|
|
||||||
|
|
||||||
build:
|
build:
|
||||||
needs: test
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@ -119,6 +73,16 @@ jobs:
|
|||||||
type=raw,enable=${{ github.event_name != 'pull_request' && github.ref_name == 'main' }}, value=latest
|
type=raw,enable=${{ github.event_name != 'pull_request' && github.ref_name == 'main' }}, value=latest
|
||||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
|
|
||||||
|
- name: Test
|
||||||
|
id: test
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
cache-from: type=gha
|
||||||
|
cache-to: type=gha,mode=max
|
||||||
|
target: test
|
||||||
|
|
||||||
# Build and push Docker image with Buildx (don't push on PR)
|
# Build and push Docker image with Buildx (don't push on PR)
|
||||||
# https://github.com/docker/build-push-action
|
# https://github.com/docker/build-push-action
|
||||||
- name: Build and push Docker image
|
- name: Build and push Docker image
|
||||||
|
26
Dockerfile
26
Dockerfile
@ -1,35 +1,37 @@
|
|||||||
FROM python:3.13-slim-bullseye
|
FROM debian:bullseye-slim AS base
|
||||||
|
|
||||||
# Inspired by https://github.com/Hudrolax/uc-docker-alpine/
|
# Inspired by https://github.com/Hudrolax/uc-docker-alpine/
|
||||||
|
|
||||||
ARG GITHUB_BUILD=false
|
ARG GITHUB_BUILD=false
|
||||||
ENV GITHUB_BUILD=${GITHUB_BUILD}
|
ENV GITHUB_BUILD=${GITHUB_BUILD}
|
||||||
|
|
||||||
ARG VERSION
|
|
||||||
ENV VERSION=${VERSION}
|
|
||||||
|
|
||||||
ENV HOME=/root
|
ENV HOME=/root
|
||||||
ENV \
|
ENV \
|
||||||
DEBIAN_FRONTEND=noninteractive \
|
DEBIAN_FRONTEND=noninteractive \
|
||||||
PYTHONUNBUFFERED=1 \
|
PYTHONUNBUFFERED=1 \
|
||||||
# prevents python creating .pyc files
|
# prevents python creating .pyc files
|
||||||
PYTHONDONTWRITEBYTECODE=1 \
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
# do not ask any interactive question
|
|
||||||
POETRY_NO_INTERACTION=1 \
|
|
||||||
POETRY_VIRTUALENVS_IN_PROJECT=true \
|
|
||||||
DISPLAY=:0
|
DISPLAY=:0
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
EXPOSE 8191
|
|
||||||
RUN apt update &&\
|
RUN apt update &&\
|
||||||
apt install -y xvfb scrot python3-tk curl chromium chromium-driver
|
apt install -y xvfb scrot python3-tk curl chromium chromium-driver
|
||||||
|
|
||||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
ENV PATH="${HOME}/.local/bin:$PATH"
|
ENV PATH="${HOME}/.local/bin:$PATH"
|
||||||
COPY pyproject.toml poetry.lock ./
|
COPY pyproject.toml uv.lock ./
|
||||||
RUN poetry install
|
RUN uv sync
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
|
|
||||||
RUN cd .venv/lib/*/site-packages/seleniumbase/drivers && ln -s /usr/bin/chromedriver uc_driver
|
RUN cd .venv/lib/*/site-packages/seleniumbase/drivers && ln -s /usr/bin/chromedriver uc_driver
|
||||||
|
|
||||||
|
FROM base AS test
|
||||||
|
|
||||||
|
RUN uv sync --group test
|
||||||
|
RUN uv run pytest --retries 2 -n auto --xvfb
|
||||||
|
|
||||||
|
FROM base
|
||||||
|
|
||||||
|
EXPOSE 8191
|
||||||
|
HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
|
||||||
CMD ["./cmd.sh"]
|
CMD ["./cmd.sh"]
|
22
README.md
22
README.md
@ -14,14 +14,14 @@ An alternative to [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) a
|
|||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
1. Clone repo to the host that has the container has issues on.
|
1. Clone repo to the host that has the container has issues on.
|
||||||
2. Using vscode and `SSH extention`, connect to the host and open repo in it.
|
2. Using vscode and `SSH extension`, connect to the host and open repo in it.
|
||||||
3. Download `devcontainers` extention and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
|
3. Download `Dev Containers` extension and reopen repo in container (with `CTRL + SHIFT + P` -> `Reopen in devcontainer`)
|
||||||
4. Forward port 6080 from devcontainer (port of noVNC server) to the host.
|
4. Forward port 6080 from devcontainer (port of noVNC server) to the host.
|
||||||
5. Open `http://localhost:6080` and connect to the virtual desktop.
|
5. Open `http://localhost:6080` and connect to the virtual desktop.
|
||||||
6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`.
|
6. Check if `chromium` works by running in VNC's terminal command `chromium --no-sandbox`.
|
||||||
7. If chromium works, run (or debug) tests from VS Code.
|
7. If chromium works, run (or debug) tests from VS Code.
|
||||||
1. If code works, congrats! (/s) You are on your own.
|
1. If code works, congrats! (not really) You are on your own.
|
||||||
2. If it does not, try another host or network, try again and create issue about the problem.
|
2. If it does not, try on another host or network and create an issue if problem persists.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@ -37,11 +37,23 @@ services:
|
|||||||
- "8191:8191" # Optional if needed to make make requests/check docs on host
|
- "8191:8191" # Optional if needed to make make requests/check docs on host
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Docker
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -p 8191:8191 ghcr.io/thephaseless/byparr:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
### Local
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv sync && ./cmd.sh
|
||||||
|
```
|
||||||
|
|
||||||
## Need help with / TODO
|
## Need help with / TODO
|
||||||
|
|
||||||
- [x] Slimming container (only ~650 MB now!)
|
- [x] Slimming container (only ~650 MB now!)
|
||||||
- [x] Add more anti-bot challenges
|
- [x] Add more anti-bot challenges
|
||||||
- [x] Add docstrings
|
- [x] Add doc strings
|
||||||
- [x] Implement versioning
|
- [x] Implement versioning
|
||||||
- [ ] Proxy support
|
- [ ] Proxy support
|
||||||
- [x] Add more architectures support
|
- [x] Add more architectures support
|
||||||
|
10
main.py
10
main.py
@ -45,7 +45,7 @@ async def health_check():
|
|||||||
|
|
||||||
|
|
||||||
@app.post("/v1")
|
@app.post("/v1")
|
||||||
def read_item(request: LinkRequest):
|
def read_item(request: LinkRequest) -> LinkResponse:
|
||||||
"""Handle POST requests."""
|
"""Handle POST requests."""
|
||||||
start_time = int(time.time() * 1000)
|
start_time = int(time.time() * 1000)
|
||||||
# request.url = "https://nowsecure.nl"
|
# request.url = "https://nowsecure.nl"
|
||||||
@ -53,8 +53,8 @@ def read_item(request: LinkRequest):
|
|||||||
response: LinkResponse
|
response: LinkResponse
|
||||||
|
|
||||||
# start_time = int(time.time() * 1000)
|
# start_time = int(time.time() * 1000)
|
||||||
try:
|
|
||||||
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
|
with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
|
||||||
|
try:
|
||||||
sb: BaseCase
|
sb: BaseCase
|
||||||
sb.uc_open_with_reconnect(request.url)
|
sb.uc_open_with_reconnect(request.url)
|
||||||
source = sb.get_page_source()
|
source = sb.get_page_source()
|
||||||
@ -88,7 +88,11 @@ def read_item(request: LinkRequest):
|
|||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error: {e}")
|
logger.error(f"Error: {e}")
|
||||||
raise HTTPException(status_code=500, detail="Unknown error, check logs") from e
|
if sb.driver:
|
||||||
|
sb.driver.quit()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=500, detail="Unknown error, check logs"
|
||||||
|
) from e
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
2238
poetry.lock
generated
2238
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@ -1,38 +1,32 @@
|
|||||||
[tool.poetry]
|
# cspell:disable
|
||||||
|
[project]
|
||||||
|
authors = [{ name = "ThePhaseless", email = "kukubaorch@gmail.com" }]
|
||||||
|
license = { text = "LICENSE" }
|
||||||
|
requires-python = "<4.0,>=3.12"
|
||||||
name = "Byparr"
|
name = "Byparr"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "API for getting cookies for Cloudflare challenges"
|
description = "API for getting cookies for Cloudflare challenges"
|
||||||
package-mode = false
|
|
||||||
authors = ["ThePhaseless <kukubaorch@gmail.com>"]
|
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = "LICENSE"
|
dependencies = [
|
||||||
repository = "https://github.com/ThePhaseless/Byparr"
|
"beautifulsoup4>=4.12.3",
|
||||||
|
"fastapi[standard]>=0.115.6",
|
||||||
|
"pyautogui>=0.9.54",
|
||||||
|
"pydantic>=2.10.3",
|
||||||
|
"seleniumbase>=4.33.7",
|
||||||
|
"uvicorn>=0.32.1",
|
||||||
|
]
|
||||||
|
urls = { repository = "https://github.com/ThePhaseless/Byparr" }
|
||||||
|
|
||||||
# cspell:disable
|
[dependency-groups]
|
||||||
[tool.poetry.dependencies]
|
test = [
|
||||||
python = "^3.12"
|
"httpx>=0.28.1",
|
||||||
fastapi = { extras = ["standard"], version = "^0" }
|
"pytest>=8.3.4",
|
||||||
seleniumbase = "^4.33.1"
|
"pytest-asyncio>=0.24.0",
|
||||||
beautifulsoup4 = "^4.12.3"
|
"pytest-progress>=1.3.0",
|
||||||
uvicorn = "^0.32.1"
|
"pytest-retry>=1.6.3",
|
||||||
pydantic = "^2.10.2"
|
"pytest-xdist>=3.6.1",
|
||||||
pyautogui = "^0.9.54"
|
]
|
||||||
|
dev = ["deptry>=0.21.1", "ruff>=0.8.2"]
|
||||||
[tool.poetry.group.dev.dependencies]
|
|
||||||
ruff = "^0.8.0"
|
|
||||||
deptry = "^0.21.1"
|
|
||||||
|
|
||||||
[tool.poetry.group.test.dependencies]
|
|
||||||
pytest = "^8.3.3"
|
|
||||||
httpx = "^0.28.0"
|
|
||||||
pytest-retry = "^1.6.3"
|
|
||||||
pytest-progress = "^1.3.0"
|
|
||||||
pytest-asyncio = "^0"
|
|
||||||
pytest-xdist = "^3.6.1"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["poetry-core"]
|
|
||||||
build-backend = "poetry.core.masonry.api"
|
|
||||||
|
|
||||||
[tool.deptry.per_rule_ignores]
|
[tool.deptry.per_rule_ignores]
|
||||||
DEP002 = ["pyautogui"]
|
DEP002 = ["pyautogui"]
|
||||||
|
@ -4,7 +4,6 @@
|
|||||||
"packageRules": [
|
"packageRules": [
|
||||||
{
|
{
|
||||||
"automerge": true,
|
"automerge": true,
|
||||||
"matchPackageNames": ["^(?!.*\bpython\b).*"],
|
|
||||||
"matchUpdateTypes": ["minor", "patch"]
|
"matchUpdateTypes": ["minor", "patch"]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
Loading…
x
Reference in New Issue
Block a user