11 Commits

Author SHA1 Message Date
17c3859e9e feat(logging): Implement structured logging and debug mode
fix: Windows cannot download novels correctly
2025-10-17 01:36:23 +11:00
11fccdb05f ci(goreleaser): Install templ before generating templates
Adds a `go install` hook to `.goreleaser.yaml` to ensure the `templ` binary is
installed and up-to-date before `templ generate` is executed. This prevents
potential build failures in CI/CD environments where `templ` might not be
pre-installed or could be an outdated version, making the release process
more robust and self-contained.
2025-10-06 18:20:47 +11:00
af968cbc9a ci(workflow): Upgrade GitHub Actions in release workflow
Updated the major versions of several GitHub Actions used in the release workflow:
- actions/checkout from v4 to v5
- actions/setup-go from v4 to v5
- goreleaser/goreleaser-action from v5 to v6

This ensures we are using the latest features, bug fixes, and security updates provided by these actions.
2025-10-06 18:11:27 +11:00
08e6280c34 feat: Add NFPM packaging and defer Playwright installation
This commit introduces NFPM configuration in `.goreleaser.yaml` to
generate native packages for various Linux distributions (e.g., .deb,
.rpm, .apk). This provides a more streamlined installation experience
for Linux users.

The Playwright browser installation logic has been moved from `main.go`
to the `Run` function of the `download` command. This change ensures
that Playwright binaries are only downloaded and installed when the
`download` command is actually invoked, improving initial application
startup performance and reducing unnecessary overhead for other commands.

The Goreleaser configuration has also been updated to version 2 syntax
and the `arm` architecture has been removed from builds.
2025-10-06 18:07:54 +11:00
34179b4dc0 Create LICENSE 2025-10-06 18:03:04 +11:00
b0f8f31dcc feat: Add concurrency and headless options for downloads
This commit introduces new features for controlling the download process:

-   **Concurrency**: Users can now specify the number of concurrent volume downloads using the `--concurrency` flag. This significantly speeds up the download of entire novels.
-   **Headless Mode**: A `--headless` flag has been added to control whether the browser operates in headless mode (without a visible UI). This is useful for debugging or running in environments without a display.

**Changes include:**

-   Updated `download` command to accept `--concurrency` and `--headless` flags.
-   Refactored `bilinovel` downloader to support `BilinovelNewOption` for configuring headless mode and concurrency.
-   Implemented a page pool and concurrency control mechanism within the `bilinovel` downloader to manage concurrent browser page usage.
-   Added `DownloadNovel` and `DownloadVolume` methods to the `bilinovel` downloader, utilizing goroutines and wait groups for parallel processing.
-   Updated `.vscode/launch.json` with new configurations for testing novel and volume downloads with the new options.
2025-10-06 10:20:36 +11:00
6084386989 refactor(bilinovel): Migrate browser automation from Chromedp to Playwright
This commit replaces the `chromedp` library with `playwright-go` for browser automation within the Bilinovel downloader.

Changes include:
*   Updated `Bilinovel` struct to manage Playwright browser, context, and page instances.
*   Rewrote `initBrowser` and `Close` methods to use Playwright's API for browser lifecycle management.
*   Refactored `processContentWithChromedp` to `processContentWithPlaywright`, adapting the logic to use Playwright's page evaluation capabilities.
*   Removed unused `context` and `time` imports.
*   Added HTML cleanup in `getChapterByPage` to remove `class` attributes from images and `data-k` attributes from all elements, improving content consistency.
2025-10-06 07:58:31 +11:00
f1320cb978 Merge pull request #2 from sarymo/patch-3
fix: normalize path separators in wrapper.go
2025-09-03 13:02:07 +10:00
sarymo
434d5f54bd Update wrapper.go 2025-09-03 08:39:30 +08:00
b8cd053b00 refactor: improve network event handling and cleanup of hidden elements in Bilinovel processing 2025-08-24 20:51:09 +10:00
560cdfdec9 refactor: streamline download process and enhance browser handling in Bilinovel 2025-08-24 19:04:00 +10:00
14 changed files with 538 additions and 286 deletions

28
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,28 @@
name: release
on:
push:
tags:
- "v*"
permissions:
contents: write
jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v5
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v6
with:
distribution: goreleaser
version: latest
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -1,6 +1,8 @@
version: 2
project_name: bilinovel-downloader
before:
hooks:
- go install github.com/a-h/templ/cmd/templ@latest
- templ generate
builds:
- env:
@@ -12,16 +14,15 @@ builds:
goarch:
- amd64
- arm64
- arm
- "386"
ldflags:
- -s -w -X bilinovel-downloader/cmd.Version={{ .Version }}
flags:
- -trimpath
archives:
- format: tar.gz
- formats: ["tar.gz"]
format_overrides:
- format: zip
- formats: ["zip"]
goos: windows
wrap_in_directory: true
release:
@@ -29,3 +30,17 @@ release:
upx:
- enabled: true
compress: best
nfpms:
- id: bilinovel-downloader
homepage: https://github.com/bestnite/bilinovel-downloader
maintainer: Nite <admin@nite07.com>
license: "MIT"
formats:
- apk
- deb
- rpm
- termux.deb
- archlinux
provides:
- bilinovel-downloader

23
.vscode/launch.json vendored
View File

@@ -2,7 +2,7 @@
"version": "0.2.0",
"configurations": [
{
"name": "download",
"name": "novel",
"type": "go",
"request": "launch",
"mode": "auto",
@@ -10,11 +10,22 @@
"args": [
"download",
"-n",
"1410",
"-v",
"52748",
"-t",
"epub"
"2727",
"--concurrency",
"5"
]
},
{
"name": "volume",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}",
"args": [
"download",
"-n=2388",
"-v=84522",
"--debug=true"
]
}
]

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 Nite
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -1,16 +1,19 @@
package cmd
import (
"bilinovel-downloader/downloader"
"bilinovel-downloader/downloader/bilinovel"
"bilinovel-downloader/epub"
"bilinovel-downloader/model"
"bilinovel-downloader/text"
"encoding/json"
"fmt"
"log"
"io"
"log/slog"
"os"
"path/filepath"
"github.com/playwright-community/playwright-go"
"github.com/spf13/cobra"
)
@@ -19,18 +22,31 @@ var downloadCmd = &cobra.Command{
Short: "Download a novel or volume",
Long: "Download a novel or volume",
Run: func(cmd *cobra.Command, args []string) {
err := runDownloadNovel()
slog.Info("Installing playwright")
err := playwright.Install(&playwright.RunOptions{
Browsers: []string{"chromium"},
Stdout: io.Discard,
})
if err != nil {
log.Printf("failed to download novel: %v", err)
slog.Error("failed to install playwright")
return
}
err = runDownloadNovel()
if err != nil {
slog.Error("failed to download novel", slog.Any("error", err))
return
}
},
}
type downloadCmdArgs struct {
NovelId int `validate:"required"`
VolumeId int `validate:"required"`
outputPath string
outputType string
NovelId int `validate:"required"`
VolumeId int `validate:"required"`
outputPath string
outputType string
concurrency int
debug bool
}
var (
@@ -42,43 +58,82 @@ func init() {
downloadCmd.Flags().IntVarP(&downloadArgs.VolumeId, "volume-id", "v", 0, "volume id")
downloadCmd.Flags().StringVarP(&downloadArgs.outputPath, "output-path", "o", "novels", "output path")
downloadCmd.Flags().StringVarP(&downloadArgs.outputType, "output-type", "t", "epub", "output type, epub or text")
downloadCmd.Flags().BoolVar(&downloadArgs.debug, "debug", false, "debug mode")
downloadCmd.Flags().IntVar(&downloadArgs.concurrency, "concurrency", 3, "concurrency of downloading volumes")
RootCmd.AddCommand(downloadCmd)
}
func runDownloadNovel() error {
downloader, err := bilinovel.New()
downloader, err := bilinovel.New(bilinovel.BilinovelNewOption{
Concurrency: downloadArgs.concurrency,
Debug: downloadArgs.debug,
})
if err != nil {
return fmt.Errorf("failed to create downloader: %v", err)
}
// 确保在函数结束时关闭资源
defer func() {
if closeErr := downloader.Close(); closeErr != nil {
slog.Info("Failed to close downloader", slog.Any("error", closeErr))
}
}()
if downloadArgs.NovelId == 0 {
return fmt.Errorf("novel id is required")
}
if downloadArgs.VolumeId == 0 {
novel, err := downloadNovel(downloader)
// 下载整本小说
err := downloadNovel(downloader, downloadArgs.NovelId)
if err != nil {
return fmt.Errorf("failed to get novel: %v", err)
}
switch downloadArgs.outputType {
case "epub":
for _, volume := range novel.Volumes {
err = epub.PackVolumeToEpub(volume, downloadArgs.outputPath, downloader.GetStyleCSS(), downloader.GetExtraFiles())
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
}
case "text":
for _, volume := range novel.Volumes {
err = text.PackVolumeToText(volume, downloadArgs.outputPath)
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
}
}
} else {
// 下载单卷
volume, err := downloadVolume(downloader)
err = downloadVolume(downloader, downloadArgs.VolumeId)
if err != nil {
return fmt.Errorf("failed to get volume: %v", err)
return fmt.Errorf("failed to download volume: %v", err)
}
}
return nil
}
func downloadNovel(downloader downloader.Downloader, novelId int) error {
novelInfo, err := downloader.GetNovel(novelId, true, nil)
if err != nil {
return fmt.Errorf("failed to get novel info: %w", err)
}
skipVolumes := make([]int, 0)
for _, volume := range novelInfo.Volumes {
jsonPath := filepath.Join(downloadArgs.outputPath, fmt.Sprintf("volume-%d-%d.json", downloadArgs.NovelId, volume.Id))
err = os.MkdirAll(filepath.Dir(jsonPath), 0755)
if err != nil {
return fmt.Errorf("failed to create directory: %v", err)
}
_, err = os.Stat(jsonPath)
if err == nil {
// 已经下载
skipVolumes = append(skipVolumes, volume.Id)
}
}
novel, err := downloader.GetNovel(novelId, false, skipVolumes)
if err != nil {
return fmt.Errorf("failed to download novel: %w", err)
}
for _, volume := range novel.Volumes {
jsonPath := filepath.Join(downloadArgs.outputPath, fmt.Sprintf("volume-%d-%d.json", downloadArgs.NovelId, volume.Id))
err = os.MkdirAll(filepath.Dir(jsonPath), 0755)
if err != nil {
return fmt.Errorf("failed to create directory: %v", err)
}
jsonFile, err := os.Create(jsonPath)
if err != nil {
return fmt.Errorf("failed to create json file: %v", err)
}
err = json.NewEncoder(jsonFile).Encode(volume)
if err != nil {
return fmt.Errorf("failed to encode json file: %v", err)
}
switch downloadArgs.outputType {
case "epub":
@@ -93,85 +148,57 @@ func runDownloadNovel() error {
}
}
}
return nil
}
func downloadNovel(downloader model.Downloader) (*model.Novel, error) {
jsonPath := filepath.Join(downloadArgs.outputPath, fmt.Sprintf("novel-%d.json", downloadArgs.NovelId))
func downloadVolume(downloader downloader.Downloader, volumeId int) error {
jsonPath := filepath.Join(downloadArgs.outputPath, fmt.Sprintf("volume-%d-%d.json", downloadArgs.NovelId, volumeId))
err := os.MkdirAll(filepath.Dir(jsonPath), 0755)
if err != nil {
return nil, fmt.Errorf("failed to create directory: %v", err)
}
_, err = os.Stat(jsonPath)
novel := &model.Novel{}
if err != nil {
if os.IsNotExist(err) {
novel, err = downloader.GetNovel(downloadArgs.NovelId)
if err != nil {
return nil, fmt.Errorf("failed to get novel: %v", err)
}
jsonFile, err := os.Create(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to create json file: %v", err)
}
defer jsonFile.Close()
err = json.NewEncoder(jsonFile).Encode(novel)
if err != nil {
return nil, fmt.Errorf("failed to encode json file: %v", err)
}
} else {
return nil, fmt.Errorf("failed to get novel: %v", err)
}
} else {
jsonFile, err := os.Open(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to open json file: %v", err)
}
defer jsonFile.Close()
err = json.NewDecoder(jsonFile).Decode(novel)
if err != nil {
return nil, fmt.Errorf("failed to decode json file: %v", err)
}
}
return novel, nil
}
func downloadVolume(downloader model.Downloader) (*model.Volume, error) {
jsonPath := filepath.Join(downloadArgs.outputPath, fmt.Sprintf("volume-%d-%d.json", downloadArgs.NovelId, downloadArgs.VolumeId))
err := os.MkdirAll(filepath.Dir(jsonPath), 0755)
if err != nil {
return nil, fmt.Errorf("failed to create directory: %v", err)
return fmt.Errorf("failed to create directory: %v", err)
}
_, err = os.Stat(jsonPath)
volume := &model.Volume{}
if err != nil {
if os.IsNotExist(err) {
volume, err = downloader.GetVolume(downloadArgs.NovelId, downloadArgs.VolumeId)
volume, err = downloader.GetVolume(downloadArgs.NovelId, volumeId, false)
if err != nil {
return nil, fmt.Errorf("failed to get volume: %v", err)
return fmt.Errorf("failed to get volume: %v", err)
}
jsonFile, err := os.Create(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to create json file: %v", err)
return fmt.Errorf("failed to create json file: %v", err)
}
err = json.NewEncoder(jsonFile).Encode(volume)
if err != nil {
return nil, fmt.Errorf("failed to encode json file: %v", err)
return fmt.Errorf("failed to encode json file: %v", err)
}
} else {
return nil, fmt.Errorf("failed to get volume: %v", err)
return fmt.Errorf("failed to get volume: %v", err)
}
} else {
jsonFile, err := os.Open(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to open json file: %v", err)
return fmt.Errorf("failed to open json file: %v", err)
}
defer jsonFile.Close()
err = json.NewDecoder(jsonFile).Decode(volume)
if err != nil {
return nil, fmt.Errorf("failed to decode json file: %v", err)
return fmt.Errorf("failed to decode json file: %v", err)
}
}
return volume, nil
switch downloadArgs.outputType {
case "epub":
err = epub.PackVolumeToEpub(volume, downloadArgs.outputPath, downloader.GetStyleCSS(), downloader.GetExtraFiles())
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
case "text":
err = text.PackVolumeToText(volume, downloadArgs.outputPath)
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
}
return nil
}

View File

@@ -4,4 +4,6 @@ import (
"github.com/spf13/cobra"
)
var RootCmd = &cobra.Command{}
var RootCmd = &cobra.Command{
Use: "bilinovel-downloader",
}

View File

@@ -4,25 +4,23 @@ import (
"bilinovel-downloader/model"
"bilinovel-downloader/utils"
"bytes"
"context"
"crypto/sha256"
_ "embed"
"fmt"
"log"
"log/slog"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
mapper "github.com/bestnite/font-mapper"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/chromedp"
"github.com/playwright-community/playwright-go"
)
//go:embed read.ttf
@@ -35,31 +33,100 @@ type Bilinovel struct {
fontMapper *mapper.GlyphOutlineMapper
textOnly bool
restyClient *utils.RestyClient
debug bool
// 浏览器实例复用
browser playwright.Browser
browserContext playwright.BrowserContext
pages map[string]playwright.Page
concurrency int
concurrentChan chan any
logger *slog.Logger
}
func New() (*Bilinovel, error) {
type BilinovelNewOption struct {
Concurrency int
Debug bool
}
func New(option BilinovelNewOption) (*Bilinovel, error) {
fontMapper, err := mapper.NewGlyphOutlineMapper(readTTF, miLantingTTF)
if err != nil {
return nil, fmt.Errorf("failed to create font mapper: %v", err)
}
restyClient := utils.NewRestyClient(10)
return &Bilinovel{
fontMapper: fontMapper,
textOnly: false,
restyClient: restyClient,
}, nil
restyClient := utils.NewRestyClient(50)
var logLevel slog.Level
if option.Debug {
logLevel = slog.LevelDebug
} else {
logLevel = slog.LevelInfo
}
handlerOptions := &slog.HandlerOptions{
Level: logLevel,
}
b := &Bilinovel{
fontMapper: fontMapper,
textOnly: false,
restyClient: restyClient,
pages: make(map[string]playwright.Page),
concurrency: option.Concurrency,
concurrentChan: make(chan any, option.Concurrency),
logger: slog.New(slog.NewTextHandler(os.Stdout, handlerOptions)),
}
// 初始化浏览器实例
err = b.initBrowser(option.Debug)
if err != nil {
return nil, fmt.Errorf("failed to init browser: %v", err)
}
return b, nil
}
func (b *Bilinovel) SetTextOnly(textOnly bool) {
b.textOnly = textOnly
}
func (b *Bilinovel) SetDebug(debug bool) {
b.debug = debug
func (b *Bilinovel) GetExtraFiles() []model.ExtraFile {
return nil
}
func (b *Bilinovel) GetExtraFiles() []model.ExtraFile {
// initBrowser 初始化浏览器实例
func (b *Bilinovel) initBrowser(debug bool) error {
pw, err := playwright.Run()
if err != nil {
return fmt.Errorf("could not start playwright: %w", err)
}
b.browser, err = pw.Chromium.Launch(playwright.BrowserTypeLaunchOptions{
Headless: playwright.Bool(!debug),
Devtools: playwright.Bool(debug),
})
if err != nil {
return fmt.Errorf("could not launch browser: %w", err)
}
b.browserContext, err = b.browser.NewContext()
if err != nil {
return fmt.Errorf("could not create browser context: %w", err)
}
b.logger.Info("Browser initialized successfully")
return nil
}
// Close 清理资源
func (b *Bilinovel) Close() error {
if b.browser != nil {
if err := b.browser.Close(); err != nil {
b.logger.Error("could not close browser", slog.Any("error", err))
}
b.browser = nil
b.browserContext = nil
}
return nil
}
@@ -70,10 +137,9 @@ func (b *Bilinovel) GetStyleCSS() string {
return string(styleCSS)
}
func (b *Bilinovel) GetNovel(novelId int) (*model.Novel, error) {
if b.debug {
log.Printf("Getting novel %v\n", novelId)
}
func (b *Bilinovel) GetNovel(novelId int, skipChapterContent bool, skipVolumes []int) (*model.Novel, error) {
b.logger.Info("Getting novel", slog.Int("novelId", novelId))
novelUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v.html", novelId)
resp, err := b.restyClient.R().Get(novelUrl)
if err != nil {
@@ -101,7 +167,7 @@ func (b *Bilinovel) GetNovel(novelId int) (*model.Novel, error) {
novel.Authors = append(novel.Authors, strings.TrimSpace(s.Text()))
})
volumes, err := b.getAllVolumes(novelId)
volumes, err := b.getAllVolumes(novelId, skipChapterContent, skipVolumes)
if err != nil {
return nil, fmt.Errorf("failed to get novel volumes: %v", err)
}
@@ -110,10 +176,9 @@ func (b *Bilinovel) GetNovel(novelId int) (*model.Novel, error) {
return novel, nil
}
func (b *Bilinovel) GetVolume(novelId int, volumeId int) (*model.Volume, error) {
if b.debug {
log.Printf("Getting volume %v of novel %v\n", volumeId, novelId)
}
func (b *Bilinovel) GetVolume(novelId int, volumeId int, skipChapterContent bool) (*model.Volume, error) {
b.logger.Info("Getting volume of novel", slog.Int("volumeId", volumeId), slog.Int("novelId", novelId))
novelUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/catalog", novelId)
resp, err := b.restyClient.R().Get(novelUrl)
if err != nil {
@@ -185,48 +250,33 @@ func (b *Bilinovel) GetVolume(novelId int, volumeId int) (*model.Volume, error)
})
idRegexp := regexp.MustCompile(`/novel/(\d+)/(\d+).html`)
wg := sync.WaitGroup{}
errChan := make(chan error, len(volume.Chapters))
for i := range volume.Chapters {
wg.Add(1)
go func(i int) {
defer wg.Done()
if !skipChapterContent {
for i := range volume.Chapters {
matches := idRegexp.FindStringSubmatch(volume.Chapters[i].Url)
if len(matches) > 0 {
chapterId, err := strconv.Atoi(matches[2])
if err != nil {
errChan <- fmt.Errorf("failed to convert chapter id: %v", err)
return
return nil, fmt.Errorf("failed to convert chapter id: %v", err)
}
chapter, err := b.GetChapter(novelId, volumeId, chapterId)
if err != nil {
errChan <- fmt.Errorf("failed to get chapter: %v", err)
return
return nil, fmt.Errorf("failed to get chapter: %v", err)
}
chapter.Id = chapterId
volume.Chapters[i] = chapter
} else {
errChan <- fmt.Errorf("failed to get chapter id: %v", volume.Chapters[i].Url)
return
return nil, fmt.Errorf("failed to get chapter id: %v", volume.Chapters[i].Url)
}
}(i)
}
wg.Wait()
close(errChan)
// 检查是否有错误
for err := range errChan {
if err != nil {
return nil, err
}
}
return volume, nil
}
func (b *Bilinovel) getAllVolumes(novelId int) ([]*model.Volume, error) {
if b.debug {
log.Printf("Getting all volumes of novel %v\n", novelId)
}
func (b *Bilinovel) getAllVolumes(novelId int, skipChapterContent bool, skipVolumes []int) ([]*model.Volume, error) {
b.logger.Info("Getting all volumes of novel", slog.Int("novelId", novelId))
catelogUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/catalog", novelId)
resp, err := b.restyClient.R().Get(catelogUrl)
if err != nil {
@@ -252,28 +302,63 @@ func (b *Bilinovel) getAllVolumes(novelId int) ([]*model.Volume, error) {
}
})
volumes := make([]*model.Volume, 0)
volumes := make([]*model.Volume, len(volumeIds))
var wg sync.WaitGroup
var mu sync.Mutex // 保护 volumes 写入的互斥锁
for i, volumeIdStr := range volumeIds {
volumeId, err := strconv.Atoi(volumeIdStr)
if err != nil {
return nil, fmt.Errorf("failed to convert volume id: %v", err)
}
volume, err := b.GetVolume(novelId, volumeId)
if err != nil {
return nil, fmt.Errorf("failed to get volume info: %v", err)
}
volume.SeriesIdx = i
volumes = append(volumes, volume)
wg.Add(1)
b.concurrentChan <- struct{}{} // 获取一个并发槽
go func(i int, volumeIdStr string) {
defer wg.Done()
defer func() { <-b.concurrentChan }() // 释放并发槽
volumeId, err := strconv.Atoi(volumeIdStr)
if err != nil {
b.logger.Error("failed to convert volume id", slog.String("volumeIdStr", volumeIdStr), slog.Any("error", err))
return
}
if slices.Contains(skipVolumes, volumeId) {
return
}
volume, err := b.GetVolume(novelId, volumeId, skipChapterContent)
if err != nil {
b.logger.Error("failed to get volume info", slog.Int("novelId", novelId), slog.Int("volumeId", volumeId), slog.Any("error", err))
return
}
volume.SeriesIdx = i
// 关闭浏览器标签页
pwPageKey := fmt.Sprintf("%v-%v", novelId, volumeId)
if pwPage, ok := b.pages[pwPageKey]; ok {
_ = pwPage.Close()
delete(b.pages, pwPageKey)
}
mu.Lock()
volumes[i] = volume
mu.Unlock()
}(i, volumeIdStr)
}
return volumes, nil
wg.Wait()
// 过滤掉获取失败的 nil volume
filteredVolumes := make([]*model.Volume, 0, len(volumes))
for _, vol := range volumes {
if vol != nil {
filteredVolumes = append(filteredVolumes, vol)
}
}
return filteredVolumes, nil
}
func (b *Bilinovel) GetChapter(novelId int, volumeId int, chapterId int) (*model.Chapter, error) {
if b.debug {
log.Printf("Getting chapter %v of novel %v\n", chapterId, novelId)
}
page := 1
b.logger.Info("Getting chapter of novel", slog.Int("chapterId", chapterId), slog.Int("novelId", novelId))
pageNum := 1
chapter := &model.Chapter{
Id: chapterId,
NovelId: novelId,
@@ -281,24 +366,33 @@ func (b *Bilinovel) GetChapter(novelId int, volumeId int, chapterId int) (*model
Url: fmt.Sprintf("https://www.bilinovel.com/novel/%v/%v.html", novelId, chapterId),
}
for {
hasNext, err := b.getChapterByPage(chapter, page)
pwPageKey := fmt.Sprintf("%v-%v", novelId, volumeId)
if _, ok := b.pages[pwPageKey]; !ok {
pwPage, err := b.browserContext.NewPage()
if err != nil {
return nil, fmt.Errorf("failed to create browser page: %w", err)
}
b.pages[pwPageKey] = pwPage
}
hasNext, err := b.getChapterByPage(b.pages[pwPageKey], chapter, pageNum)
if err != nil {
return nil, fmt.Errorf("failed to download chapter: %w", err)
}
if !hasNext {
break
}
page++
pageNum++
}
return chapter, nil
}
func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, error) {
if b.debug {
log.Printf("Getting chapter %v by page %v\n", chapter.Id, page)
}
var nextPageUrlRegexp = regexp.MustCompile(`url_next:\s?['"]([^'"]*?)['"]`)
var cleanNextPageUrlRegexp = regexp.MustCompile(`(_\d+)?\.html$`)
Url := strings.TrimSuffix(chapter.Url, ".html") + fmt.Sprintf("_%v.html", page)
func (b *Bilinovel) getChapterByPage(pwPage playwright.Page, chapter *model.Chapter, pageNum int) (bool, error) {
b.logger.Info("Getting chapter by page", slog.Int("chapter", chapter.Id), slog.Int("page", pageNum))
Url := strings.TrimSuffix(chapter.Url, ".html") + fmt.Sprintf("_%v.html", pageNum)
hasNext := false
headers := map[string]string{
@@ -319,8 +413,9 @@ func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, er
}
html := resp.Body()
// 解决乱序问题
resortedHtml, err := ProcessContentWithChromedp(string(html))
resortedHtml, err := b.processContentWithPlaywright(pwPage, string(html))
if err != nil {
return false, fmt.Errorf("failed to process html: %w", err)
}
@@ -329,7 +424,18 @@ func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, er
return false, fmt.Errorf("failed to parse html: %w", err)
}
if page == 1 {
// 判断章节是否有下一页
n := nextPageUrlRegexp.FindStringSubmatch(resortedHtml)
if len(n) != 2 {
return false, fmt.Errorf("failed to determine wether there is a next page")
}
s := cleanNextPageUrlRegexp.ReplaceAllString(n[1], "")
if strings.Contains(Url, s) {
hasNext = true
}
if pageNum == 1 {
chapter.Title = doc.Find("#atitle").Text()
}
content := doc.Find("#acontent").First()
@@ -337,7 +443,7 @@ func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, er
content.Find("center").Remove()
content.Find(".google-auto-placed").Remove()
if strings.Contains(resp.String(), `font-family: "read"`) {
if strings.Contains(resortedHtml, `font-family: "read"`) {
html, err := content.Find("p").Last().Html()
if err != nil {
return false, fmt.Errorf("failed to get html: %v", err)
@@ -368,6 +474,7 @@ func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, er
imageFilename := fmt.Sprintf("%x%s", string(imageHash[:]), path.Ext(imgUrl))
s.SetAttr("src", imageFilename)
s.SetAttr("alt", imgUrl)
s.RemoveAttr("class")
img, err := b.getImg(imgUrl)
if err != nil {
return
@@ -382,6 +489,19 @@ func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, er
})
}
doc.Find("*").Each(func(i int, s *goquery.Selection) {
if len(s.Nodes) > 0 && len(s.Nodes[0].Attr) > 0 {
// 遍历元素的所有属性
for _, attr := range s.Nodes[0].Attr {
// 3. 检查属性名是否以 "data-k" 开头,且属性值是否为空
if strings.HasPrefix(attr.Key, "data-k") {
// 4. 如果满足条件,就移除这个属性
s.RemoveAttr(attr.Key)
}
}
}
})
htmlStr, err := content.Html()
if err != nil {
return false, fmt.Errorf("failed to get html: %v", err)
@@ -396,9 +516,7 @@ func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, er
}
func (b *Bilinovel) getImg(url string) ([]byte, error) {
if b.debug {
log.Printf("Getting img %v\n", url)
}
b.logger.Info("Getting img", slog.String("url", url))
resp, err := b.restyClient.R().SetHeader("Referer", "https://www.bilinovel.com").Get(url)
if err != nil {
return nil, err
@@ -407,12 +525,22 @@ func (b *Bilinovel) getImg(url string) ([]byte, error) {
return resp.Body(), nil
}
func ProcessContentWithChromedp(htmlContent string) (string, error) {
tempFile, err := os.CreateTemp("", "bilinovel-temp-*.html")
// processContentWithPlaywright 使用复用的浏览器实例处理内容
func (b *Bilinovel) processContentWithPlaywright(page playwright.Page, htmlContent string) (string, error) {
// 替换 window.location.replace防止页面跳转
htmlContent = strings.ReplaceAll(htmlContent, "window.location.replace", "console.log")
tempPath := filepath.Join(os.TempDir(), "bilinovel-downloader")
err := os.MkdirAll(tempPath, 0755)
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %w", err)
}
tempFile, err := os.CreateTemp(tempPath, "temp-*.html")
if err != nil {
return "", fmt.Errorf("failed to create temp file: %w", err)
}
defer os.Remove(tempFile.Name())
_, err = tempFile.WriteString(htmlContent)
if err != nil {
return "", fmt.Errorf("failed to write temp file: %w", err)
@@ -420,70 +548,92 @@ func ProcessContentWithChromedp(htmlContent string) (string, error) {
tempFile.Close()
tempFilePath := tempFile.Name()
// 创建chromedp选项
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.Flag("headless", true),
chromedp.Flag("disable-gpu", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-extensions", true),
chromedp.Flag("no-sandbox", true),
)
// // 屏蔽请求
// googleAdsDomains := []string{
// "adtrafficquality.google",
// "doubleclick.net",
// "googlesyndication.com",
// "googletagmanager.com",
// "hm.baidu.com",
// "cloudflareinsights.com",
// "fsdoa.js", // adblock 检测
// "https://www.linovelib.com/novel/", // 阻止从本地文件跳转到在线页面
// }
// err = page.Route("**/*", func(route playwright.Route) {
// for _, d := range googleAdsDomains {
// if strings.Contains(route.Request().URL(), d) {
// b.logger.Debug("blocking request", slog.String("url", route.Request().URL()))
// err := route.Abort("aborted")
// if err != nil {
// b.logger.Debug("failed to block request", route.Request().URL(), err)
// }
// return
// }
// }
// _ = route.Continue()
// })
// if err != nil {
// return "", fmt.Errorf("failed to intercept requests: %w", err)
// }
allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
defer cancel()
_, err = page.ExpectResponse(func(url string) bool {
return strings.Contains(url, "chapterlog.js")
}, func() error {
_, err = page.Goto("file://" + filepath.ToSlash(tempFilePath))
if err != nil {
return fmt.Errorf("could not navigate to file: %w", err)
}
return nil
}, playwright.PageExpectResponseOptions{
Timeout: playwright.Float(10000),
})
if err != nil {
return "", fmt.Errorf("failed to wait for network request finish")
}
ctx, cancel := chromedp.NewContext(allocCtx)
defer cancel()
err = page.Locator("#acontent").WaitFor(playwright.LocatorWaitForOptions{
State: playwright.WaitForSelectorStateVisible,
Timeout: playwright.Float(10000),
})
if err != nil {
return "", fmt.Errorf("could not wait for #acontent: %w", err)
}
// 设置超时
ctx, cancel = context.WithTimeout(ctx, 30*time.Second)
defer cancel()
var processedHTML string
// 3. 执行chromedp任务并获取页面代码
err = chromedp.Run(ctx,
network.Enable(),
// 等待JavaScript执行完成
chromedp.ActionFunc(func(ctx context.Context) error {
// 监听网络事件
networkEventChan := make(chan bool, 1)
requestID := ""
chromedp.ListenTarget(ctx, func(ev interface{}) {
switch ev := ev.(type) {
case *network.EventRequestWillBeSent:
if strings.Contains(ev.Request.URL, "chapterlog.js") {
requestID = ev.RequestID.String()
}
case *network.EventLoadingFinished:
if ev.RequestID.String() == requestID {
networkEventChan <- true
}
// 遍历所有 #acontent 的子元素, 通过 window.getComputedStyle().display 检测是否是 none, 如果是 none 则从页面删除这个元素
result, err := page.Evaluate(`
(function() {
const acontent = document.getElementById('acontent');
if (!acontent) {
return 'acontent element not found';
}
let removedCount = 0;
const elements = acontent.querySelectorAll('*');
// 从后往前遍历,避免删除元素时影响索引
for (let i = elements.length - 1; i >= 0; i--) {
const element = elements[i];
const computedStyle = window.getComputedStyle(element);
if (computedStyle.display === 'none' || computedStyle.transform == 'matrix(0, 0, 0, 0, 0, 0)') {
element.remove();
removedCount++;
}
})
go func() {
select {
case <-networkEventChan:
case <-time.After(30 * time.Second):
log.Println("Timeout waiting for external script")
case <-ctx.Done():
log.Println("Context cancelled")
}
}()
return nil
}),
// 导航到本地文件
chromedp.Navigate("file://"+filepath.ToSlash(tempFilePath)),
// 等待页面加载完成
chromedp.WaitVisible(`#acontent`, chromedp.ByID),
// 获取页面的HTML代码
chromedp.OuterHTML("html", &processedHTML, chromedp.ByQuery),
)
}
return 'Removed ' + removedCount + ' hidden elements';
})()
`)
if err != nil {
return "", fmt.Errorf("chromedp execution failed: %w", err)
return "", fmt.Errorf("failed to remove hidden elements: %w", err)
}
b.logger.Debug("Hidden elements removal result", slog.Any("count", result))
processedHTML, err := page.Content()
if err != nil {
return "", fmt.Errorf("could not get page content: %w", err)
}
return processedHTML, nil

12
downloader/downloader.go Normal file
View File

@@ -0,0 +1,12 @@
package downloader
import "bilinovel-downloader/model"
type Downloader interface {
GetNovel(novelId int, skipChapterContent bool, skipVolumes []int) (*model.Novel, error)
GetVolume(novelId int, volumeId int, skipChapterContent bool) (*model.Volume, error)
GetChapter(novelId int, volumeId int, chapterId int) (*model.Chapter, error)
GetStyleCSS() string
GetExtraFiles() []model.ExtraFile
Close() error
}

View File

@@ -339,6 +339,8 @@ func addDirContentToZip(zipWriter *zip.Writer, dirPath string, method uint16) er
return err
}
relPath = filepath.ToSlash(relPath)
file, err := os.Open(filePath)
if err != nil {
return err

12
go.mod
View File

@@ -6,24 +6,20 @@ require (
github.com/PuerkitoBio/goquery v1.10.3
github.com/a-h/templ v0.3.943
github.com/bestnite/font-mapper v0.0.0-20250823155658-56c76d820267
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
github.com/chromedp/chromedp v0.14.1
github.com/go-resty/resty/v2 v2.16.5
github.com/google/uuid v1.6.0
github.com/playwright-community/playwright-go v0.5200.1
github.com/spf13/cobra v1.9.1
)
require (
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
github.com/deckarep/golang-set/v2 v2.8.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.4 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.7 // indirect
golang.org/x/image v0.30.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
)

48
go.sum
View File

@@ -1,49 +1,48 @@
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/a-h/templ v0.3.906 h1:ZUThc8Q9n04UATaCwaG60pB1AqbulLmYEAMnWV63svg=
github.com/a-h/templ v0.3.906/go.mod h1:FFAu4dI//ESmEN7PQkJ7E7QfnSEMdcnu7QrAY8Dn334=
github.com/a-h/templ v0.3.943 h1:o+mT/4yqhZ33F3ootBiHwaY4HM5EVaOJfIshvd5UNTY=
github.com/a-h/templ v0.3.943/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/bestnite/font-mapper v0.0.0-20250823155658-56c76d820267 h1:nmUTJV2u/0XmVjQ++VIy/Hu+MtxdpQvOevvcSZtUATA=
github.com/bestnite/font-mapper v0.0.0-20250823155658-56c76d820267/go.mod h1:cfB1e9YhoI/QWrXPp3h6QVAKU6iCI2ifbjRPHP3xf/0=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
github.com/chromedp/chromedp v0.14.1 h1:0uAbnxewy/Q+Bg7oafVePE/6EXEho9hnaC38f+TTENg=
github.com/chromedp/chromedp v0.14.1/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b h1:6Q4zRHXS/YLOl9Ng1b1OOOBWMidAQZR3Gel0UKPC/KU=
github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ=
github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
github.com/playwright-community/playwright-go v0.5200.1 h1:Sm2oOuhqt0M5Y4kUi/Qh9w4cyyi3ZIWTBeGKImc2UVo=
github.com/playwright-community/playwright-go v0.5200.1/go.mod h1:UnnyQZaqUOO5ywAZu60+N4EiWReUqX1MQBBA3Oofvf8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M=
github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -67,8 +66,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -84,14 +81,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -120,4 +114,6 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -1,15 +0,0 @@
package model
type ExtraFile struct {
Data []byte
Path string
ManifestItem ManifestItem
}
type Downloader interface {
GetNovel(novelId int) (*Novel, error)
GetVolume(novelId int, volumeId int) (*Volume, error)
GetChapter(novelId int, volumeId int, chapterId int) (*Chapter, error)
GetStyleCSS() string
GetExtraFiles() []ExtraFile
}

View File

@@ -1,6 +1,14 @@
package model
import "encoding/xml"
import (
"encoding/xml"
)
type ExtraFile struct {
Data []byte
Path string
ManifestItem ManifestItem
}
type DublinCoreMetadata struct {
XMLName xml.Name `xml:"metadata"`

View File

@@ -8,13 +8,12 @@ import (
)
func TestBilinovel_GetNovel(t *testing.T) {
bilinovel, err := bilinovel.New()
bilinovel, err := bilinovel.New(bilinovel.BilinovelNewOption{Concurrency: 5})
bilinovel.SetTextOnly(true)
bilinovel.SetDebug(true)
if err != nil {
t.Fatalf("failed to create bilinovel: %v", err)
}
novel, err := bilinovel.GetNovel(4519)
novel, err := bilinovel.GetNovel(2727, false, nil)
if err != nil {
t.Fatalf("failed to get novel: %v", err)
}
@@ -26,12 +25,12 @@ func TestBilinovel_GetNovel(t *testing.T) {
}
func TestBilinovel_GetVolume(t *testing.T) {
bilinovel, err := bilinovel.New()
bilinovel, err := bilinovel.New(bilinovel.BilinovelNewOption{Concurrency: 1})
bilinovel.SetTextOnly(true)
if err != nil {
t.Fatalf("failed to create bilinovel: %v", err)
}
volume, err := bilinovel.GetVolume(1410, 52748)
volume, err := bilinovel.GetVolume(2727, 129092, false)
if err != nil {
t.Fatalf("failed to get volume: %v", err)
}
@@ -43,12 +42,12 @@ func TestBilinovel_GetVolume(t *testing.T) {
}
func TestBilinovel_GetChapter(t *testing.T) {
bilinovel, err := bilinovel.New()
bilinovel, err := bilinovel.New(bilinovel.BilinovelNewOption{Concurrency: 1})
bilinovel.SetTextOnly(true)
if err != nil {
t.Fatalf("failed to create bilinovel: %v", err)
}
bilinovel.SetDebug(true)
chapter, err := bilinovel.GetChapter(1410, 52748, 52752)
chapter, err := bilinovel.GetChapter(2727, 129092, 129094)
if err != nil {
t.Fatalf("failed to get chapter: %v", err)
}