nite/bilinovel-downloader
1
0
Fork 0
mirror of https://github.com/bestnite/bilinovel-downloader.git synced 2025-10-25 16:51:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
nite
2025-08-24 16:53:05 +10:00
parent 75745b9431
commit e9fbe5c5db
22 changed files with 1005 additions and 642 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
.vscode/launch.json vendored
View File

@@ -2,20 +2,20 @@
"version": "0.2.0",
"configurations": [
{
"name": "volume",
"name": "download",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}",
"args": ["download", "volume", "-n", "2025", "-v", "72693"]
},
{
"name": "novel",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}",
"args": ["download", "novel", "-n", "4325"]
"args": [
"download",
"-n",
"1410",
"-v",
"52748",
"-t",
"text"
]
}
]
}
}

4
README.md
View File

@@ -8,13 +8,13 @@
1. 下载整本 `https://www.bilinovel.com/novel/2388.html`
```bash
bilinovel-downloader download novel -n 2388
bilinovel-downloader download -n 2388
```
2. 下载单卷 `https://www.bilinovel.com/novel/2388/vol_84522.html`
```bash
bilinovel-downloader download volume -n 2388 -v 84522
bilinovel-downloader download -n 2388 -v 84522
```
3. 对自动生成的 epub 格式不满意可以自行修改后使用命令打包

187
cmd/download.go
View File

@@ -2,7 +2,14 @@ package cmd
import (
"bilinovel-downloader/downloader/bilinovel"
"bilinovel-downloader/epub"
"bilinovel-downloader/model"
"bilinovel-downloader/text"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"github.com/spf13/cobra"
)
@@ -11,74 +18,160 @@ var downloadCmd = &cobra.Command{
Use: "download",
Short: "Download a novel or volume",
Long: "Download a novel or volume",
Run: func(cmd *cobra.Command, args []string) {
err := runDownloadNovel()
if err != nil {
log.Printf("failed to download novel: %v", err)
}
},
}
var downloadNovelCmd = &cobra.Command{
Use: "novel",
Short: "Download a novel, default download all volumes",
Long: "Download a novel, default download all volumes",
RunE: runDownloadNovel,
}
var downloadVolumeCmd = &cobra.Command{
Use: "volume",
Short: "Download a volume",
Long: "Download a volume",
RunE: runDownloadVolume,
}
type downloadNovelArgs struct {
NovelId int `validate:"required"`
outputPath string
}
type downloadVolumeArgs struct {
type downloadCmdArgs struct {
NovelId int `validate:"required"`
VolumeId int `validate:"required"`
outputPath string
outputType string
}
var (
novelArgs downloadNovelArgs
volumeArgs downloadVolumeArgs
downloadArgs downloadCmdArgs
)
func init() {
downloadNovelCmd.Flags().IntVarP(&novelArgs.NovelId, "novel-id", "n", 0, "novel id")
downloadNovelCmd.Flags().StringVarP(&novelArgs.outputPath, "output-path", "o", "./novels", "output path")
downloadVolumeCmd.Flags().IntVarP(&volumeArgs.NovelId, "novel-id", "n", 0, "novel id")
downloadVolumeCmd.Flags().IntVarP(&volumeArgs.VolumeId, "volume-id", "v", 0, "volume id")
downloadVolumeCmd.Flags().StringVarP(&volumeArgs.outputPath, "output-path", "o", "./novels", "output path")
downloadCmd.AddCommand(downloadNovelCmd)
downloadCmd.AddCommand(downloadVolumeCmd)
downloadCmd.Flags().IntVarP(&downloadArgs.NovelId, "novel-id", "n", 0, "novel id")
downloadCmd.Flags().IntVarP(&downloadArgs.VolumeId, "volume-id", "v", 0, "volume id")
downloadCmd.Flags().StringVarP(&downloadArgs.outputPath, "output-path", "o", "novels", "output path")
downloadCmd.Flags().StringVarP(&downloadArgs.outputType, "output-type", "t", "epub", "output type, epub or text")
RootCmd.AddCommand(downloadCmd)
}
func runDownloadNovel(cmd *cobra.Command, args []string) error {
if novelArgs.NovelId == 0 {
func runDownloadNovel() error {
downloader, err := bilinovel.New()
if err != nil {
return fmt.Errorf("failed to create downloader: %v", err)
}
if downloadArgs.NovelId == 0 {
return fmt.Errorf("novel id is required")
}
err := bilinovel.DownloadNovel(novelArgs.NovelId, novelArgs.outputPath)
if err != nil {
return fmt.Errorf("failed to download novel: %v", err)
if downloadArgs.VolumeId == 0 {
novel, err := downloadNovel(downloader)
if err != nil {
return fmt.Errorf("failed to get novel: %v", err)
}
switch downloadArgs.outputType {
case "epub":
for _, volume := range novel.Volumes {
err = epub.PackVolumeToEpub(volume, downloadArgs.outputPath, downloader.GetStyleCSS(), downloader.GetExtraFiles())
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
}
case "text":
for _, volume := range novel.Volumes {
err = text.PackVolumeToText(volume, downloadArgs.outputPath)
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
}
}
} else {
// 下载单卷
volume, err := downloadVolume(downloader)
if err != nil {
return fmt.Errorf("failed to get volume: %v", err)
}
switch downloadArgs.outputType {
case "epub":
err = epub.PackVolumeToEpub(volume, downloadArgs.outputPath, downloader.GetStyleCSS(), downloader.GetExtraFiles())
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
case "text":
err = text.PackVolumeToText(volume, downloadArgs.outputPath)
if err != nil {
return fmt.Errorf("failed to pack volume: %v", err)
}
}
}
return nil
}
func runDownloadVolume(cmd *cobra.Command, args []string) error {
if volumeArgs.NovelId == 0 {
return fmt.Errorf("novel id is required")
}
if volumeArgs.VolumeId == 0 {
return fmt.Errorf("volume id is required")
}
err := bilinovel.DownloadVolume(volumeArgs.NovelId, volumeArgs.VolumeId, volumeArgs.outputPath)
func downloadNovel(downloader model.Downloader) (*model.Novel, error) {
jsonPath := filepath.Join(downloadArgs.outputPath, fmt.Sprintf("novel-%d.json", downloadArgs.NovelId))
err := os.MkdirAll(filepath.Dir(jsonPath), 0755)
if err != nil {
return fmt.Errorf("failed to download volume: %v", err)
return nil, fmt.Errorf("failed to create directory: %v", err)
}
return nil
_, err = os.Stat(jsonPath)
novel := &model.Novel{}
if err != nil {
if os.IsNotExist(err) {
novel, err = downloader.GetNovel(downloadArgs.NovelId)
if err != nil {
return nil, fmt.Errorf("failed to get novel: %v", err)
}
jsonFile, err := os.Create(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to create json file: %v", err)
}
defer jsonFile.Close()
err = json.NewEncoder(jsonFile).Encode(novel)
if err != nil {
return nil, fmt.Errorf("failed to encode json file: %v", err)
}
} else {
return nil, fmt.Errorf("failed to get novel: %v", err)
}
} else {
jsonFile, err := os.Open(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to open json file: %v", err)
}
defer jsonFile.Close()
err = json.NewDecoder(jsonFile).Decode(novel)
if err != nil {
return nil, fmt.Errorf("failed to decode json file: %v", err)
}
}
return novel, nil
}
func downloadVolume(downloader model.Downloader) (*model.Volume, error) {
jsonPath := filepath.Join(downloadArgs.outputPath, fmt.Sprintf("volume-%d-%d.json", downloadArgs.NovelId, downloadArgs.VolumeId))
err := os.MkdirAll(filepath.Dir(jsonPath), 0755)
if err != nil {
return nil, fmt.Errorf("failed to create directory: %v", err)
}
_, err = os.Stat(jsonPath)
volume := &model.Volume{}
if err != nil {
if os.IsNotExist(err) {
volume, err = downloader.GetVolume(downloadArgs.NovelId, downloadArgs.VolumeId)
if err != nil {
return nil, fmt.Errorf("failed to get volume: %v", err)
}
jsonFile, err := os.Create(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to create json file: %v", err)
}
err = json.NewEncoder(jsonFile).Encode(volume)
if err != nil {
return nil, fmt.Errorf("failed to encode json file: %v", err)
}
} else {
return nil, fmt.Errorf("failed to get volume: %v", err)
}
} else {
jsonFile, err := os.Open(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to open json file: %v", err)
}
defer jsonFile.Close()
err = json.NewDecoder(jsonFile).Decode(volume)
if err != nil {
return nil, fmt.Errorf("failed to decode json file: %v", err)
}
}
return volume, nil
}

4
cmd/package.go
View File

@@ -1,7 +1,7 @@
package cmd
import (
"bilinovel-downloader/downloader/bilinovel"
"bilinovel-downloader/epub"
"fmt"
"github.com/spf13/cobra"
@@ -28,7 +28,7 @@ func init() {
}
func runPackage(cmd *cobra.Command, args []string) error {
err := bilinovel.CreateEpub(pArgs.DirPath)
err := epub.PackEpub(pArgs.DirPath)
if err != nil {
return fmt.Errorf("failed to create epub: %v", err)
}

BIN
downloader/bilinovel/MI LANTING.ttf Normal file
View File

Binary file not shown.

671
downloader/bilinovel/bilinovel.go
View File

@@ -2,12 +2,11 @@ package bilinovel
import (
"bilinovel-downloader/model"
"bilinovel-downloader/template"
"bilinovel-downloader/utils"
"bytes"
"context"
"crypto/sha256"
_ "embed"
"encoding/json"
"fmt"
"log"
"net/http"
@@ -17,17 +16,68 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/google/uuid"
mapper "github.com/bestnite/font-mapper"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/chromedp"
)
func GetNovel(novelId int) (*model.Novel, error) {
novelUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v.html", novelId)
resp, err := utils.Request().Get(novelUrl)
//go:embed read.ttf
var readTTF []byte
//go:embed "MI LANTING.ttf"
var miLantingTTF []byte
type Bilinovel struct {
fontMapper *mapper.GlyphOutlineMapper
textOnly bool
restyClient *utils.RestyClient
debug bool
}
func New() (*Bilinovel, error) {
fontMapper, err := mapper.NewGlyphOutlineMapper(readTTF, miLantingTTF)
if err != nil {
return nil, fmt.Errorf("failed to get novel info: %v", err)
return nil, fmt.Errorf("failed to create font mapper: %v", err)
}
restyClient := utils.NewRestyClient(10)
return &Bilinovel{
fontMapper: fontMapper,
textOnly: false,
restyClient: restyClient,
}, nil
}
func (b *Bilinovel) SetTextOnly(textOnly bool) {
b.textOnly = textOnly
}
func (b *Bilinovel) SetDebug(debug bool) {
b.debug = debug
}
func (b *Bilinovel) GetExtraFiles() []model.ExtraFile {
return nil
}
//go:embed style.css
var styleCSS []byte
func (b *Bilinovel) GetStyleCSS() string {
return string(styleCSS)
}
func (b *Bilinovel) GetNovel(novelId int) (*model.Novel, error) {
if b.debug {
log.Printf("Getting novel %v\n", novelId)
}
novelUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v.html", novelId)
resp, err := b.restyClient.R().Get(novelUrl)
if err != nil {
return nil, fmt.Errorf("failed to get novel info: %w", err)
}
if resp.StatusCode() != http.StatusOK {
return nil, fmt.Errorf("failed to get novel info: %v", resp.Status())
@@ -51,7 +101,7 @@ func GetNovel(novelId int) (*model.Novel, error) {
novel.Authors = append(novel.Authors, strings.TrimSpace(s.Text()))
})
volumes, err := getNovelVolumes(novelId)
volumes, err := b.getAllVolumes(novelId)
if err != nil {
return nil, fmt.Errorf("failed to get novel volumes: %v", err)
}
@@ -60,11 +110,14 @@ func GetNovel(novelId int) (*model.Novel, error) {
return novel, nil
}
func GetVolume(novelId int, volumeId int) (*model.Volume, error) {
func (b *Bilinovel) GetVolume(novelId int, volumeId int) (*model.Volume, error) {
if b.debug {
log.Printf("Getting volume %v of novel %v\n", volumeId, novelId)
}
novelUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/catalog", novelId)
resp, err := utils.Request().Get(novelUrl)
resp, err := b.restyClient.R().Get(novelUrl)
if err != nil {
return nil, fmt.Errorf("failed to get novel info: %v", err)
return nil, fmt.Errorf("failed to get novel info: %w", err)
}
if resp.StatusCode() != http.StatusOK {
return nil, fmt.Errorf("failed to get novel info: %v", resp.Status())
@@ -89,7 +142,7 @@ func GetVolume(novelId int, volumeId int) (*model.Volume, error) {
}
volumeUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/vol_%v.html", novelId, volumeId)
resp, err = utils.Request().Get(volumeUrl)
resp, err = b.restyClient.R().Get(volumeUrl)
if err != nil {
return nil, fmt.Errorf("failed to get novel info: %v", err)
}
@@ -109,9 +162,14 @@ func GetVolume(novelId int, volumeId int) (*model.Volume, error) {
volume.SeriesIdx = seriesIdx
volume.Title = strings.TrimSpace(doc.Find(".book-title").First().Text())
volume.Description = strings.TrimSpace(doc.Find(".book-summary>content").First().Text())
volume.Cover = doc.Find(".book-cover").First().AttrOr("src", "")
volume.Url = volumeUrl
volume.Chapters = make([]*model.Chapter, 0)
volume.CoverUrl = doc.Find(".book-cover").First().AttrOr("src", "")
cover, err := b.getImg(volume.CoverUrl)
if err != nil {
return nil, fmt.Errorf("failed to get cover: %v", err)
}
volume.Cover = cover
doc.Find(".authorname>a").Each(func(i int, s *goquery.Selection) {
volume.Authors = append(volume.Authors, strings.TrimSpace(s.Text()))
@@ -119,7 +177,6 @@ func GetVolume(novelId int, volumeId int) (*model.Volume, error) {
doc.Find(".illname>a").Each(func(i int, s *goquery.Selection) {
volume.Authors = append(volume.Authors, strings.TrimSpace(s.Text()))
})
doc.Find(".chapter-li.jsChapter").Each(func(i int, s *goquery.Selection) {
volume.Chapters = append(volume.Chapters, &model.Chapter{
Title: s.Find("a").Text(),
@@ -127,12 +184,51 @@ func GetVolume(novelId int, volumeId int) (*model.Volume, error) {
})
})
idRegexp := regexp.MustCompile(`/novel/(\d+)/(\d+).html`)
wg := sync.WaitGroup{}
errChan := make(chan error, len(volume.Chapters))
for i := range volume.Chapters {
wg.Add(1)
go func(i int) {
defer wg.Done()
matches := idRegexp.FindStringSubmatch(volume.Chapters[i].Url)
if len(matches) > 0 {
chapterId, err := strconv.Atoi(matches[2])
if err != nil {
errChan <- fmt.Errorf("failed to convert chapter id: %v", err)
return
}
chapter, err := b.GetChapter(novelId, volumeId, chapterId)
if err != nil {
errChan <- fmt.Errorf("failed to get chapter: %v", err)
return
}
chapter.Id = chapterId
volume.Chapters[i] = chapter
} else {
errChan <- fmt.Errorf("failed to get chapter id: %v", volume.Chapters[i].Url)
return
}
}(i)
}
wg.Wait()
close(errChan)
// 检查是否有错误
for err := range errChan {
if err != nil {
return nil, err
}
}
return volume, nil
}
func getNovelVolumes(novelId int) ([]*model.Volume, error) {
func (b *Bilinovel) getAllVolumes(novelId int) ([]*model.Volume, error) {
if b.debug {
log.Printf("Getting all volumes of novel %v\n", novelId)
}
catelogUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/catalog", novelId)
resp, err := utils.Request().Get(catelogUrl)
resp, err := b.restyClient.R().Get(catelogUrl)
if err != nil {
return nil, fmt.Errorf("failed to get catelog: %v", err)
}
@@ -162,7 +258,7 @@ func getNovelVolumes(novelId int) ([]*model.Volume, error) {
if err != nil {
return nil, fmt.Errorf("failed to convert volume id: %v", err)
}
volume, err := GetVolume(novelId, volumeId)
volume, err := b.GetVolume(novelId, volumeId)
if err != nil {
return nil, fmt.Errorf("failed to get volume info: %v", err)
}
@@ -173,207 +269,36 @@ func getNovelVolumes(novelId int) ([]*model.Volume, error) {
return volumes, nil
}
func DownloadNovel(novelId int, outputPath string) error {
log.Printf("Downloading Novel: %v", novelId)
novel, err := GetNovel(novelId)
if err != nil {
return fmt.Errorf("failed to get novel info: %v", err)
func (b *Bilinovel) GetChapter(novelId int, volumeId int, chapterId int) (*model.Chapter, error) {
if b.debug {
log.Printf("Getting chapter %v of novel %v\n", chapterId, novelId)
}
outputPath = filepath.Join(outputPath, utils.CleanDirName(novel.Title))
err = os.MkdirAll(outputPath, 0755)
if err != nil {
return fmt.Errorf("failed to create output directory: %v", err)
}
for _, volume := range novel.Volumes {
err := downloadVolume(volume, outputPath)
if err != nil {
return fmt.Errorf("failed to download volume: %v", err)
}
}
return nil
}
func DownloadVolume(novelId, volumeId int, outputPath string) error {
volume, err := GetVolume(novelId, volumeId)
if err != nil {
return fmt.Errorf("failed to get volume info: %v", err)
}
err = downloadVolume(volume, outputPath)
if err != nil {
return fmt.Errorf("failed to download volume: %v", err)
}
return nil
}
func downloadVolume(volume *model.Volume, outputPath string) error {
log.Printf("Downloading Volume: %s", volume.Title)
outputPath = filepath.Join(outputPath, utils.CleanDirName(volume.Title))
err := os.MkdirAll(outputPath, 0755)
if err != nil {
return fmt.Errorf("failed to create output directory: %v", err)
}
_, err = os.Stat(filepath.Join(outputPath, "volume.json"))
if os.IsNotExist(err) {
for idx, chapter := range volume.Chapters {
err := DownloadChapter(idx, chapter, outputPath)
if err != nil {
return fmt.Errorf("failed to download chapter: %v", err)
}
}
} else {
jsonBytes, err := os.ReadFile(filepath.Join(outputPath, "volume.json"))
if err != nil {
return fmt.Errorf("failed to read volume: %v", err)
}
err = json.Unmarshal(jsonBytes, volume)
if err != nil {
return fmt.Errorf("failed to unmarshal volume: %v", err)
}
for idx, chapter := range volume.Chapters {
file, err := os.Create(filepath.Join(outputPath, fmt.Sprintf("OEBPS/Text/chapter-%03v.xhtml", idx+1)))
if err != nil {
return fmt.Errorf("failed to create chapter file: %v", err)
}
err = template.ContentXHTML(chapter).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render text file: %v", err)
}
}
}
for i := range volume.Chapters {
volume.Chapters[i].ImageFullPaths = utils.Unique(volume.Chapters[i].ImageFullPaths)
volume.Chapters[i].ImageOEBPSPaths = utils.Unique(volume.Chapters[i].ImageOEBPSPaths)
}
jsonBytes, err := json.Marshal(volume)
if err != nil {
return fmt.Errorf("failed to marshal volume: %v", err)
}
err = os.WriteFile(filepath.Join(outputPath, "volume.json"), jsonBytes, 0644)
if err != nil {
return fmt.Errorf("failed to write volume: %v", err)
}
coverPath := filepath.Join(outputPath, "cover.jpeg")
err = os.MkdirAll(path.Dir(coverPath), 0755)
if err != nil {
return fmt.Errorf("failed to create cover directory: %v", err)
}
err = DownloadImg(volume.Cover, coverPath)
if err != nil {
return fmt.Errorf("failed to download cover: %v", err)
}
coverXHTMLPath := filepath.Join(outputPath, "OEBPS/Text/cover.xhtml")
err = os.MkdirAll(path.Dir(coverXHTMLPath), 0755)
if err != nil {
return fmt.Errorf("failed to create cover directory: %v", err)
}
file, err := os.Create(coverXHTMLPath)
if err != nil {
return fmt.Errorf("failed to create cover file: %v", err)
}
err = template.CoverXHTML(fmt.Sprintf(`../../cover%s`, strings.ReplaceAll(path.Ext(volume.Cover), "jpg", "jpeg"))).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render cover: %v", err)
}
err = DownloadFont(filepath.Join(outputPath, "OEBPS/Fonts"))
if err != nil {
return fmt.Errorf("failed to download font: %v", err)
}
contentsXHTMLPath := filepath.Join(outputPath, "OEBPS/Text/contents.xhtml")
err = os.MkdirAll(path.Dir(contentsXHTMLPath), 0755)
if err != nil {
return fmt.Errorf("failed to create contents directory: %v", err)
}
file, err = os.Create(contentsXHTMLPath)
if err != nil {
return fmt.Errorf("failed to create contents file: %v", err)
}
contents := strings.Builder{}
contents.WriteString(`<nav epub:type="toc" id="toc">`)
contents.WriteString(`<ol>`)
for _, chapter := range volume.Chapters {
contents.WriteString(fmt.Sprintf(`<li><a href="%s">%s</a></li>`, strings.TrimPrefix(chapter.TextOEBPSPath, "Text/"), chapter.Title))
}
contents.WriteString(`</ol>`)
contents.WriteString(`</nav>`)
err = template.ContentXHTML(&model.Chapter{
Title: "目录",
Content: contents.String(),
}).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render contents: %v", err)
}
err = CreateContainerXML(outputPath)
if err != nil {
return fmt.Errorf("failed to create container xml: %v", err)
}
u, err := uuid.NewV7()
if err != nil {
return fmt.Errorf("failed to generate uuid: %v", err)
}
err = CreateContentOPF(outputPath, u.String(), volume)
if err != nil {
return fmt.Errorf("failed to create content opf: %v", err)
}
err = CreateEpub(outputPath)
if err != nil {
return fmt.Errorf("failed to create epub: %v", err)
}
return nil
}
func DownloadChapter(chapterIdx int, chapter *model.Chapter, outputPath string) error {
chapter.TextFullPath = filepath.Join(outputPath, fmt.Sprintf("OEBPS/Text/chapter-%03v.xhtml", chapterIdx+1))
chapter.TextOEBPSPath = fmt.Sprintf("Text/chapter-%03v.xhtml", chapterIdx+1)
err := os.MkdirAll(path.Dir(chapter.TextFullPath), 0755)
if err != nil {
return fmt.Errorf("failed to create text directory: %v", err)
}
page := 1
chapter := &model.Chapter{
Id: chapterId,
NovelId: novelId,
VolumeId: volumeId,
Url: fmt.Sprintf("https://www.bilinovel.com/novel/%v/%v.html", novelId, chapterId),
}
for {
hasNext, err := downloadChapterByPage(page, chapterIdx, chapter, outputPath)
hasNext, err := b.getChapterByPage(chapter, page)
if err != nil {
return fmt.Errorf("failed to download chapter: %v", err)
return nil, fmt.Errorf("failed to download chapter: %w", err)
}
if !hasNext {
break
}
page++
time.Sleep(time.Second)
}
file, err := os.Create(chapter.TextFullPath)
if err != nil {
return fmt.Errorf("failed to create text file: %v", err)
}
err = template.ContentXHTML(chapter).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render text file: %v", err)
}
return nil
return chapter, nil
}
func downloadChapterByPage(page, chapterIdx int, chapter *model.Chapter, outputPath string) (bool, error) {
func (b *Bilinovel) getChapterByPage(chapter *model.Chapter, page int) (bool, error) {
if b.debug {
log.Printf("Getting chapter %v by page %v\n", chapter.Id, page)
}
Url := strings.TrimSuffix(chapter.Url, ".html") + fmt.Sprintf("_%v.html", page)
log.Printf("Downloading Chapter: %s", Url)
hasNext := false
headers := map[string]string{
@@ -381,251 +306,185 @@ func downloadChapterByPage(page, chapterIdx int, chapter *model.Chapter, outputP
"Accept-Language": "zh-CN,zh;q=0.9,en-GB;q=0.8,en;q=0.7,zh-TW;q=0.6",
"Cookie": "night=1;",
}
resp, err := utils.Request().SetHeaders(headers).Get(Url)
resp, err := b.restyClient.R().SetHeaders(headers).Get(Url)
if err != nil {
return hasNext, err
return false, fmt.Errorf("failed to get chapter: %w", err)
}
if resp.StatusCode() != http.StatusOK {
return hasNext, fmt.Errorf("failed to get chapter: %v", resp.Status())
return false, fmt.Errorf("failed to get chapter: %v", resp.Status())
}
if strings.Contains(resp.String(), `<a onclick="window.location.href = ReadParams.url_next;">下一頁</a>`) {
hasNext = true
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
html := resp.Body()
// 解决乱序问题
resortedHtml, err := ProcessContentWithChromedp(string(html))
if err != nil {
fmt.Println(err)
return hasNext, err
return false, fmt.Errorf("failed to process html: %w", err)
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(resortedHtml))
if err != nil {
return false, fmt.Errorf("failed to parse html: %w", err)
}
imgSavePath := fmt.Sprintf("OEBPS/Images/chapter-%03v", chapterIdx+1)
if page == 1 {
chapter.Title = doc.Find("#atitle").Text()
}
content := doc.Find("#acontent").First()
content.Find(".cgo").Remove()
content.Find("center").Remove()
content.Find(".google-auto-placed").Remove()
if strings.Contains(resp.String(), `font-family: "read"`) {
content.Find("p").Last().AddClass("read-font")
}
content.Find("img").Each(func(i int, s *goquery.Selection) {
if strings.Contains(resp.String(), `font-family: "read"`) {
html, err := content.Find("p").Last().Html()
if err != nil {
return
return false, fmt.Errorf("failed to get html: %v", err)
}
imgUrl := s.AttrOr("data-src", "")
if imgUrl == "" {
imgUrl = s.AttrOr("src", "")
if imgUrl == "" {
return
builder := strings.Builder{}
for _, r := range html {
_, newRune, ok := b.fontMapper.MappingRune(r)
if ok {
builder.WriteRune(newRune)
}
}
fileName := filepath.Join(imgSavePath, fmt.Sprintf("%03v%s", len(chapter.ImageFullPaths)+1, path.Ext(imgUrl)))
err = DownloadImg(imgUrl, filepath.Join(outputPath, fileName))
if err == nil {
s.SetAttr("src", "../"+strings.TrimPrefix(fileName, "OEBPS/"))
s.RemoveAttr("class")
s.RemoveAttr("data-src")
chapter.ImageFullPaths = append(chapter.ImageFullPaths, filepath.Join(outputPath, fileName))
chapter.ImageOEBPSPaths = append(chapter.ImageOEBPSPaths, strings.TrimPrefix(fileName, "OEBPS/"))
}
})
if err != nil {
return false, fmt.Errorf("failed to download img: %v", err)
content.Find("p").Last().SetHtml(builder.String())
}
html, err := content.Html()
if b.textOnly {
content.Find("img").Remove()
} else {
content.Find("img").Each(func(i int, s *goquery.Selection) {
imgUrl := s.AttrOr("data-src", "")
if imgUrl == "" {
imgUrl = s.AttrOr("src", "")
if imgUrl == "" {
return
}
}
imageHash := sha256.Sum256([]byte(imgUrl))
imageFilename := fmt.Sprintf("%x%s", string(imageHash[:]), path.Ext(imgUrl))
s.SetAttr("src", imageFilename)
s.SetAttr("alt", imgUrl)
img, err := b.getImg(imgUrl)
if err != nil {
return
}
if chapter.Content == nil {
chapter.Content = &model.ChaperContent{}
}
if chapter.Content.Images == nil {
chapter.Content.Images = make(map[string][]byte)
}
chapter.Content.Images[imageFilename] = img
})
}
htmlStr, err := content.Html()
if err != nil {
return false, fmt.Errorf("failed to get html: %v", err)
}
chapter.Content += strings.TrimSpace(html)
if chapter.Content == nil {
chapter.Content = &model.ChaperContent{}
}
chapter.Content.Html += strings.TrimSpace(htmlStr)
return hasNext, nil
}
func DownloadImg(url string, fileName string) error {
_, err := os.Stat(fileName)
if !os.IsNotExist(err) {
return nil
func (b *Bilinovel) getImg(url string) ([]byte, error) {
if b.debug {
log.Printf("Getting img %v\n", url)
}
log.Printf("Downloading Image: %s", url)
dir := filepath.Dir(fileName)
err = os.MkdirAll(dir, 0755)
resp, err := b.restyClient.R().SetHeader("Referer", "https://www.bilinovel.com").Get(url)
if err != nil {
return err
return nil, err
}
resp, err := utils.Request().SetHeader("Referer", "https://www.bilinovel.com").Get(url)
if err != nil {
return err
}
err = os.WriteFile(fileName, resp.Body(), 0644)
if err != nil {
return err
}
return nil
return resp.Body(), nil
}
func CreateContainerXML(dirPath string) error {
containerPath := filepath.Join(dirPath, "META-INF/container.xml")
err := os.MkdirAll(path.Dir(containerPath), 0755)
func ProcessContentWithChromedp(htmlContent string) (string, error) {
tempFile, err := os.CreateTemp("", "bilinovel-temp-*.html")
if err != nil {
return fmt.Errorf("failed to create container directory: %v", err)
return "", fmt.Errorf("failed to create temp file: %w", err)
}
file, err := os.Create(containerPath)
defer os.Remove(tempFile.Name())
_, err = tempFile.WriteString(htmlContent)
if err != nil {
return fmt.Errorf("failed to create container file: %v", err)
return "", fmt.Errorf("failed to write temp file: %w", err)
}
err = template.ContainerXML().Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render container: %v", err)
}
return nil
}
tempFile.Close()
tempFilePath := tempFile.Name()
func CreateContentOPF(dirPath string, uuid string, volume *model.Volume) error {
creators := make([]model.DCCreator, 0)
for _, author := range volume.Authors {
creators = append(creators, model.DCCreator{
Value: author,
})
}
dc := &model.DublinCoreMetadata{
Titles: []model.DCTitle{
{
Value: volume.Title,
},
},
Identifiers: []model.DCIdentifier{
{
Value: fmt.Sprintf("urn:uuid:%s", uuid),
ID: "book-id",
// Scheme: "UUID",
},
},
Languages: []model.DCLanguage{
{
Value: "zh-CN",
},
},
Descriptions: []model.DCDescription{
{
Value: volume.Description,
},
},
Creators: creators,
Metas: []model.DublinCoreMeta{
{
Name: "cover",
Content: "cover",
},
{
Property: "dcterms:modified",
Value: time.Now().UTC().Format("2006-01-02T15:04:05Z"),
},
{
Name: "calibre:series",
Content: volume.NovelTitle,
},
{
Name: "calibre:series_index",
Content: strconv.Itoa(volume.SeriesIdx),
},
},
}
manifest := &model.Manifest{
Items: make([]model.ManifestItem, 0),
}
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "cover.xhtml",
Link: "OEBPS/Text/cover.xhtml",
Media: "application/xhtml+xml",
})
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "contents.xhtml",
Link: "OEBPS/Text/contents.xhtml",
Media: "application/xhtml+xml",
Properties: "nav",
})
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "cover",
Link: fmt.Sprintf("cover%s", strings.ReplaceAll(path.Ext(volume.Cover), "jpg", "jpeg")),
Media: fmt.Sprintf("image/%s", strings.ReplaceAll(strings.TrimPrefix(path.Ext(volume.Cover), "."), "jpg", "jpeg")),
Properties: "cover-image",
})
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "read.ttf",
Link: "OEBPS/Fonts/read.ttf",
Media: "application/vnd.ms-opentype",
})
for _, chapter := range volume.Chapters {
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: path.Base(chapter.TextOEBPSPath),
Link: "OEBPS/" + chapter.TextOEBPSPath,
Media: "application/xhtml+xml",
})
for _, image := range chapter.ImageOEBPSPaths {
item := model.ManifestItem{
ID: strings.Join(strings.Split(strings.ToLower(image), string(filepath.Separator)), "-"),
Link: "OEBPS/" + image,
}
item.Media = fmt.Sprintf("image/%s", strings.ReplaceAll(strings.TrimPrefix(path.Ext(volume.Cover), "."), "jpg", "jpeg"))
manifest.Items = append(manifest.Items, item)
}
}
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "style",
Link: "style.css",
Media: "text/css",
})
// 创建chromedp选项
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.Flag("headless", true),
chromedp.Flag("disable-gpu", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-extensions", true),
chromedp.Flag("no-sandbox", true),
)
spine := &model.Spine{
Items: make([]model.SpineItem, 0),
}
for _, item := range manifest.Items {
if filepath.Ext(item.Link) == ".xhtml" {
spine.Items = append(spine.Items, model.SpineItem{
IDref: item.ID,
allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
defer cancel()
ctx, cancel := chromedp.NewContext(allocCtx)
defer cancel()
// 设置超时
ctx, cancel = context.WithTimeout(ctx, 30*time.Second)
defer cancel()
var processedHTML string
// 3. 执行chromedp任务并获取页面代码
err = chromedp.Run(ctx,
network.Enable(),
// 等待JavaScript执行完成
chromedp.ActionFunc(func(ctx context.Context) error {
// 监听网络事件
networkEventChan := make(chan bool, 1)
requestID := ""
chromedp.ListenTarget(ctx, func(ev interface{}) {
switch ev := ev.(type) {
case *network.EventRequestWillBeSent:
if strings.Contains(ev.Request.URL, "chapterlog.js") {
requestID = ev.RequestID.String()
}
case *network.EventLoadingFinished:
if ev.RequestID.String() == requestID {
networkEventChan <- true
}
}
})
}
}
contentOPFPath := filepath.Join(dirPath, "content.opf")
err := os.MkdirAll(path.Dir(contentOPFPath), 0755)
go func() {
select {
case <-networkEventChan:
case <-time.After(30 * time.Second):
log.Println("Timeout waiting for external script")
case <-ctx.Done():
log.Println("Context cancelled")
}
}()
return nil
}),
// 导航到本地文件
chromedp.Navigate("file://"+filepath.ToSlash(tempFilePath)),
// 等待页面加载完成
chromedp.WaitVisible(`#acontent`, chromedp.ByID),
// 获取页面的HTML代码
chromedp.OuterHTML("html", &processedHTML, chromedp.ByQuery),
)
if err != nil {
return fmt.Errorf("failed to create content directory: %v", err)
return "", fmt.Errorf("chromedp execution failed: %w", err)
}
file, err := os.Create(contentOPFPath)
if err != nil {
return fmt.Errorf("failed to create content file: %v", err)
}
err = template.ContentOPF("book-id", dc, manifest, spine, nil).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render content: %v", err)
}
return nil
}
//go:embed read.ttf
var readTTF []byte
func DownloadFont(outputPath string) error {
log.Printf("Writing Font: %s", outputPath)
fontPath := filepath.Join(outputPath, "read.ttf")
err := os.MkdirAll(path.Dir(fontPath), 0755)
if err != nil {
return fmt.Errorf("failed to create font directory: %v", err)
}
err = os.WriteFile(fontPath, readTTF, 0644)
if err != nil {
return fmt.Errorf("failed to write font: %v", err)
}
return nil
return processedHTML, nil
}

122
downloader/bilinovel/epub.go
View File

@@ -1,122 +0,0 @@
package bilinovel
import (
"archive/zip"
"io"
"log"
"os"
"path/filepath"
)
func CreateEpub(path string) error {
log.Printf("Creating epub for %s", path)
savePath := path + ".epub"
zipFile, err := os.Create(savePath)
if err != nil {
return err
}
defer zipFile.Close()
zipWriter := zip.NewWriter(zipFile)
defer zipWriter.Close()
err = addStringToZip(zipWriter, "mimetype", "application/epub+zip", zip.Store)
if err != nil {
return err
}
err = addDirContentToZip(zipWriter, path, zip.Deflate)
if err != nil {
return err
}
err = addStringToZip(zipWriter, "style.css", StyleCSS, zip.Deflate)
if err != nil {
return err
}
return nil
}
// func addFileToZip(zipWriter *zip.Writer, filename string, relPath string, method uint16) error {
// file, err := os.Open(filename)
// if err != nil {
// return err
// }
// defer file.Close()
// info, err := file.Stat()
// if err != nil {
// return err
// }
// header, err := zip.FileInfoHeader(info)
// if err != nil {
// return err
// }
// header.Name = relPath
// header.Method = method
// writer, err := zipWriter.CreateHeader(header)
// if err != nil {
// return err
// }
// _, err = io.Copy(writer, file)
// return err
// }
func addStringToZip(zipWriter *zip.Writer, relPath, content string, method uint16) error {
header := &zip.FileHeader{
Name: relPath,
Method: method,
}
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}
_, err = writer.Write([]byte(content))
return err
}
func addDirContentToZip(zipWriter *zip.Writer, dirPath string, method uint16) error {
return filepath.Walk(dirPath, func(filePath string, info os.FileInfo, err error) error {
if filepath.Base(filePath) == "volume.json" {
return nil
}
if err != nil {
return err
}
if info.IsDir() {
return nil
}
relPath, err := filepath.Rel(dirPath, filePath)
if err != nil {
return err
}
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
header.Name = relPath
header.Method = method
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(writer, file)
return err
})
}

17
downloader/bilinovel/style.css.go → downloader/bilinovel/style.css
View File

@@ -1,19 +1,3 @@
package bilinovel
const StyleCSS = `
@font-face {
font-family: "MI LANTING";
src: url(OEBPS/Fonts/read.ttf);
}
.read-font {
display: block;
font-family: "MI LANTING", serif;
font-size: 1.33333em;
text-indent: 2em;
margin: 0.8em 0;
}
body > div {
margin: 0 auto;
padding: 20px;
@@ -53,4 +37,3 @@ img {
margin-top: 1em;
margin-bottom: 1em;
}
`

363
epub/wrapper.go Normal file
View File

@@ -0,0 +1,363 @@
package epub
import (
"archive/zip"
"bilinovel-downloader/model"
"bilinovel-downloader/template"
"bilinovel-downloader/utils"
"context"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/google/uuid"
)
func PackVolumeToEpub(volume *model.Volume, outputPath string, styleCSS string, extraFiles []model.ExtraFile) error {
outputPath = filepath.Join(outputPath, utils.CleanDirName(volume.Title))
_, err := os.Stat(outputPath)
if err != nil {
if os.IsNotExist(err) {
err = os.MkdirAll(outputPath, 0755)
if err != nil {
return fmt.Errorf("failed to create output directory: %v", err)
}
} else {
return fmt.Errorf("failed to get output directory: %v", err)
}
} else {
err = os.RemoveAll(outputPath)
if err != nil {
return fmt.Errorf("failed to remove output directory: %v", err)
}
err = os.MkdirAll(outputPath, 0755)
if err != nil {
return fmt.Errorf("failed to create output directory: %v", err)
}
}
// 将文字写入 OEBPS/Text/chapter-%03v.xhtml
// 将图片写入 OEBPS/Images/chapter-%03v/
for i, chapter := range volume.Chapters {
imageNames := make([]string, 0)
for imgName, imgData := range chapter.Content.Images {
imageNames = append(imageNames, imgName)
imgPath := filepath.Join(outputPath, fmt.Sprintf("OEBPS/Images/chapter-%03v/%s", i, imgName))
err := os.MkdirAll(filepath.Dir(imgPath), 0755)
if err != nil {
return fmt.Errorf("failed to create image directory: %v", err)
}
err = os.WriteFile(imgPath, imgData, 0644)
if err != nil {
return fmt.Errorf("failed to write image: %v", err)
}
}
chapterPath := filepath.Join(outputPath, fmt.Sprintf("OEBPS/Text/chapter-%03v.xhtml", i))
err = os.MkdirAll(filepath.Dir(chapterPath), 0755)
if err != nil {
return fmt.Errorf("failed to create chapter directory: %v", err)
}
file, err := os.Create(chapterPath)
if err != nil {
return fmt.Errorf("failed to create chapter file: %v", err)
}
defer file.Close()
text := chapter.Content.Html
for _, imgName := range imageNames {
text = strings.ReplaceAll(text, imgName, fmt.Sprintf("../Images/chapter-%03v/%s", i, imgName))
}
err = template.ContentXHTML(chapter.Title, text).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to write chapter: %v", err)
}
}
// 将 Cover 写入
coverPath := filepath.Join(outputPath, fmt.Sprintf("cover%s", filepath.Ext(volume.CoverUrl)))
err = os.WriteFile(coverPath, volume.Cover, 0644)
if err != nil {
return fmt.Errorf("failed to write cover: %v", err)
}
// 将 CoverXHTML 写入 OEBPS/Text/cover.xhtml
coverXHTMLPath := filepath.Join(outputPath, "OEBPS/Text/cover.xhtml")
file, err := os.Create(coverXHTMLPath)
if err != nil {
return fmt.Errorf("failed to create cover XHTML file: %v", err)
}
defer file.Close()
err = template.CoverXHTML(fmt.Sprintf("../../%s", filepath.Base(coverPath))).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render cover XHTML: %v", err)
}
// OEBPS/Text/contents.xhtml 目录
contentsXHTMLPath := filepath.Join(outputPath, "OEBPS/Text/contents.xhtml")
file, err = os.Create(contentsXHTMLPath)
if err != nil {
return fmt.Errorf("failed to create contents XHTML file: %v", err)
}
defer file.Close()
contents := strings.Builder{}
contents.WriteString(`<nav epub:type="toc" id="toc">`)
contents.WriteString(`<ol>`)
for i, chapter := range volume.Chapters {
contents.WriteString(fmt.Sprintf(`<li><a href="chapter-%03v.xhtml">%s</a></li>`, i, chapter.Title))
}
contents.WriteString(`</ol>`)
contents.WriteString(`</nav>`)
err = template.ContentXHTML("目录", contents.String()).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render contents XHTML: %v", err)
}
// ContainerXML
containerPath := filepath.Join(outputPath, "META-INF/container.xml")
err = os.MkdirAll(filepath.Dir(containerPath), 0755)
if err != nil {
return fmt.Errorf("failed to create container directory: %v", err)
}
file, err = os.Create(containerPath)
if err != nil {
return fmt.Errorf("failed to create container file: %v", err)
}
defer file.Close()
err = template.ContainerXML().Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render container: %v", err)
}
// ContentOPF
u := uuid.New()
err = CreateContentOPF(outputPath, u.String(), volume, extraFiles)
if err != nil {
return fmt.Errorf("failed to create content OPF: %v", err)
}
// 写入 CSS
cssPath := filepath.Join(outputPath, "style.css")
err = os.WriteFile(cssPath, []byte(styleCSS), 0644)
if err != nil {
return fmt.Errorf("failed to write CSS: %v", err)
}
// 写入 extraFiles
for _, file := range extraFiles {
extraFilePath := filepath.Join(outputPath, file.Path)
err = os.WriteFile(extraFilePath, file.Data, 0644)
if err != nil {
return fmt.Errorf("failed to write extra file: %v", err)
}
}
// 打包成 epub 文件
err = PackEpub(outputPath)
if err != nil {
return fmt.Errorf("failed to pack epub: %v", err)
}
return nil
}
func CreateContentOPF(outputPath string, uuid string, volume *model.Volume, extraFiles []model.ExtraFile) error {
creators := make([]model.DCCreator, 0)
for _, author := range volume.Authors {
creators = append(creators, model.DCCreator{
Value: author,
})
}
dc := &model.DublinCoreMetadata{
Titles: []model.DCTitle{
{
Value: volume.Title,
},
},
Identifiers: []model.DCIdentifier{
{
Value: fmt.Sprintf("urn:uuid:%s", uuid),
ID: "book-id",
// Scheme: "UUID",
},
},
Languages: []model.DCLanguage{
{
Value: "zh-CN",
},
},
Descriptions: []model.DCDescription{
{
Value: volume.Description,
},
},
Creators: creators,
Metas: []model.DublinCoreMeta{
{
Name: "cover",
Content: "cover",
},
{
Property: "dcterms:modified",
Value: time.Now().UTC().Format("2006-01-02T15:04:05Z"),
},
{
Name: "calibre:series",
Content: volume.NovelTitle,
},
{
Name: "calibre:series_index",
Content: strconv.Itoa(volume.SeriesIdx),
},
},
}
manifest := &model.Manifest{
Items: make([]model.ManifestItem, 0),
}
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "cover.xhtml",
Link: "OEBPS/Text/cover.xhtml",
Media: "application/xhtml+xml",
})
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "contents.xhtml",
Link: "OEBPS/Text/contents.xhtml",
Media: "application/xhtml+xml",
Properties: "nav",
})
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "cover",
Link: fmt.Sprintf("cover%s", filepath.Ext(volume.CoverUrl)),
Media: fmt.Sprintf("image/%s", strings.ReplaceAll(strings.TrimPrefix(filepath.Ext(volume.CoverUrl), "."), "jpg", "jpeg")),
Properties: "cover-image",
})
for i, chapter := range volume.Chapters {
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: fmt.Sprintf("chapter-%03v.xhtml", i),
Link: fmt.Sprintf("OEBPS/Text/chapter-%03v.xhtml", i),
Media: "application/xhtml+xml",
})
for filename := range chapter.Content.Images {
item := model.ManifestItem{
ID: fmt.Sprintf("chapter-%03v-%s", i, filepath.Ext(filename)),
Link: fmt.Sprintf("OEBPS/Images/chapter-%03v/%s", i, filepath.Ext(filename)),
Media: fmt.Sprintf("image/%s", strings.ReplaceAll(strings.TrimPrefix(filepath.Ext(filename), "."), "jpg", "jpeg")),
}
manifest.Items = append(manifest.Items, item)
}
}
manifest.Items = append(manifest.Items, model.ManifestItem{
ID: "style",
Link: "style.css",
Media: "text/css",
})
// ExtraFiles
for _, file := range extraFiles {
manifest.Items = append(manifest.Items, file.ManifestItem)
}
spine := &model.Spine{
Items: make([]model.SpineItem, 0),
}
for _, item := range manifest.Items {
if filepath.Ext(item.Link) == ".xhtml" {
spine.Items = append(spine.Items, model.SpineItem{
IDref: item.ID,
})
}
}
contentOPFPath := filepath.Join(outputPath, "content.opf")
err := os.MkdirAll(path.Dir(contentOPFPath), 0755)
if err != nil {
return fmt.Errorf("failed to create content directory: %v", err)
}
file, err := os.Create(contentOPFPath)
if err != nil {
return fmt.Errorf("failed to create content file: %v", err)
}
err = template.ContentOPF("book-id", dc, manifest, spine, nil).Render(context.Background(), file)
if err != nil {
return fmt.Errorf("failed to render content: %v", err)
}
return nil
}
func PackEpub(dirPath string) error {
savePath := strings.TrimSuffix(dirPath, string(filepath.Separator)) + ".epub"
zipFile, err := os.Create(savePath)
if err != nil {
return err
}
defer zipFile.Close()
zipWriter := zip.NewWriter(zipFile)
defer zipWriter.Close()
err = addStringToZip(zipWriter, "mimetype", "application/epub+zip", zip.Store)
if err != nil {
return err
}
err = addDirContentToZip(zipWriter, dirPath, zip.Deflate)
if err != nil {
return err
}
return nil
}
func addStringToZip(zipWriter *zip.Writer, relPath, content string, method uint16) error {
header := &zip.FileHeader{
Name: relPath,
Method: method,
}
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}
_, err = writer.Write([]byte(content))
return err
}
func addDirContentToZip(zipWriter *zip.Writer, dirPath string, method uint16) error {
return filepath.Walk(dirPath, func(filePath string, info os.FileInfo, err error) error {
if filepath.Base(filePath) == "volume.json" {
return nil
}
if err != nil {
return err
}
if info.IsDir() {
return nil
}
relPath, err := filepath.Rel(dirPath, filePath)
if err != nil {
return err
}
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
header.Name = relPath
header.Method = method
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(writer, file)
return err
})
}

11
go.mod
View File

@@ -5,6 +5,9 @@ go 1.24.2
require (
github.com/PuerkitoBio/goquery v1.10.3
github.com/a-h/templ v0.3.906
github.com/bestnite/font-mapper v0.0.0-20250823155658-56c76d820267
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
github.com/chromedp/chromedp v0.14.1
github.com/go-resty/resty/v2 v2.16.5
github.com/google/uuid v1.6.0
github.com/spf13/cobra v1.9.1
@@ -12,7 +15,15 @@ require (
require (
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.6 // indirect
golang.org/x/image v0.30.0 // indirect
golang.org/x/net v0.39.0 // indirect
golang.org/x/sys v0.35.0 // indirect
)

29
go.sum
View File

@@ -1,20 +1,40 @@
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/a-h/templ v0.3.857 h1:6EqcJuGZW4OL+2iZ3MD+NnIcG7nGkaQeF2Zq5kf9ZGg=
github.com/a-h/templ v0.3.857/go.mod h1:qhrhAkRFubE7khxLZHsBFHfX+gWwVNKbzKeF9GlPV4M=
github.com/a-h/templ v0.3.906 h1:ZUThc8Q9n04UATaCwaG60pB1AqbulLmYEAMnWV63svg=
github.com/a-h/templ v0.3.906/go.mod h1:FFAu4dI//ESmEN7PQkJ7E7QfnSEMdcnu7QrAY8Dn334=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/bestnite/font-mapper v0.0.0-20250823155658-56c76d820267 h1:nmUTJV2u/0XmVjQ++VIy/Hu+MtxdpQvOevvcSZtUATA=
github.com/bestnite/font-mapper v0.0.0-20250823155658-56c76d820267/go.mod h1:cfB1e9YhoI/QWrXPp3h6QVAKU6iCI2ifbjRPHP3xf/0=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
github.com/chromedp/chromedp v0.14.1 h1:0uAbnxewy/Q+Bg7oafVePE/6EXEho9hnaC38f+TTENg=
github.com/chromedp/chromedp v0.14.1/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b h1:6Q4zRHXS/YLOl9Ng1b1OOOBWMidAQZR3Gel0UKPC/KU=
github.com/go-json-experiment/json v0.0.0-20250813233538-9b1f9ea2e11b/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
@@ -27,6 +47,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/image v0.30.0 h1:jD5RhkmVAnjqaCUXfbGBrn3lpxbknfN9w2UhHHU+5B4=
golang.org/x/image v0.30.0/go.mod h1:SAEUTxCCMWSrJcCy/4HwavEsfZZJlYxeHLc6tTiAe/c=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -56,11 +78,14 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

15
model/downloader.go Normal file
View File

@@ -0,0 +1,15 @@
package model
type ExtraFile struct {
Data []byte
Path string
ManifestItem ManifestItem
}
type Downloader interface {
GetNovel(novelId int) (*Novel, error)
GetVolume(novelId int, volumeId int) (*Volume, error)
GetChapter(novelId int, volumeId int, chapterId int) (*Chapter, error)
GetStyleCSS() string
GetExtraFiles() []ExtraFile
}

0
model/container_opf.go → model/epub.go
View File

21
model/struct.go → model/novel.go
View File

@@ -1,13 +1,17 @@
package model
type ChaperContent struct {
Html string
Images map[string][]byte
}
type Chapter struct {
Title string
Url string
Content string
ImageOEBPSPaths []string
ImageFullPaths []string
TextOEBPSPath string
TextFullPath string
Id int
NovelId int
VolumeId int
Title string
Url string
Content *ChaperContent
}
type Volume struct {
@@ -15,7 +19,8 @@ type Volume struct {
SeriesIdx int
Title string
Url string
Cover string
CoverUrl string
Cover []byte
Description string
Authors []string
Chapters []*Chapter

2
template/container.xml_templ.go
View File

@@ -1,6 +1,6 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.906
// templ: version: v0.3.865
package template
//lint:file-ignore SA4006 This context is only used if a nested component is present.

2
template/content.opf_templ.go
View File

@@ -1,6 +1,6 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.906
// templ: version: v0.3.865
package template
//lint:file-ignore SA4006 This context is only used if a nested component is present.

10
template/content.xhtml.templ
View File

@@ -1,21 +1,19 @@
package template
import "bilinovel-downloader/model"
templ ContentXHTML(content *model.Chapter) {
templ ContentXHTML(title, content string) {
@templ.Raw(`<?xml version='1.0' encoding='utf-8'?>`)
// @templ.Raw(`<!DOCTYPE html>`)
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" xml:lang="zh-CN">
<head>
<title>{ content.Title }</title>
<title>{ title }</title>
@templ.Raw(`<link href="../../style.css" rel="stylesheet" type="text/css"/>`)
</head>
<body>
<div class="chapter">
<h1>{ content.Title }</h1>
<h1>{ title }</h1>
@templ.Raw(`<hr/>`)
<div class="content">
@templ.Raw(content.Content)
@templ.Raw(content)
</div>
</div>
</body>

16
template/content.xhtml_templ.go
View File

@@ -1,6 +1,6 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.906
// templ: version: v0.3.865
package template
//lint:file-ignore SA4006 This context is only used if a nested component is present.
@@ -8,9 +8,7 @@ package template
import "github.com/a-h/templ"
import templruntime "github.com/a-h/templ/runtime"
import "bilinovel-downloader/model"
func ContentXHTML(content *model.Chapter) templ.Component {
func ContentXHTML(title, content string) templ.Component {
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
@@ -40,9 +38,9 @@ func ContentXHTML(content *model.Chapter) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(content.Title)
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `template/content.xhtml.templ`, Line: 10, Col: 25}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `template/content.xhtml.templ`, Line: 8, Col: 17}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
@@ -61,9 +59,9 @@ func ContentXHTML(content *model.Chapter) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(content.Title)
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `template/content.xhtml.templ`, Line: 15, Col: 23}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `template/content.xhtml.templ`, Line: 13, Col: 15}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
if templ_7745c5c3_Err != nil {
@@ -81,7 +79,7 @@ func ContentXHTML(content *model.Chapter) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templ.Raw(content.Content).Render(ctx, templ_7745c5c3_Buffer)
templ_7745c5c3_Err = templ.Raw(content).Render(ctx, templ_7745c5c3_Buffer)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}

2
template/cover.xhtml_templ.go
View File

@@ -1,6 +1,6 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.906
// templ: version: v0.3.865
package template
//lint:file-ignore SA4006 This context is only used if a nested component is present.

60
test/bilinovel_test.go Normal file
View File

@@ -0,0 +1,60 @@
package test
import (
"bilinovel-downloader/downloader/bilinovel"
"encoding/json"
"fmt"
"testing"
)
func TestBilinovel_GetNovel(t *testing.T) {
bilinovel, err := bilinovel.New()
bilinovel.SetTextOnly(true)
bilinovel.SetDebug(true)
if err != nil {
t.Fatalf("failed to create bilinovel: %v", err)
}
novel, err := bilinovel.GetNovel(4519)
if err != nil {
t.Fatalf("failed to get novel: %v", err)
}
jsonBytes, err := json.Marshal(novel)
if err != nil {
t.Fatalf("failed to marshal novel: %v", err)
}
fmt.Println(string(jsonBytes))
}
func TestBilinovel_GetVolume(t *testing.T) {
bilinovel, err := bilinovel.New()
bilinovel.SetTextOnly(true)
if err != nil {
t.Fatalf("failed to create bilinovel: %v", err)
}
volume, err := bilinovel.GetVolume(1410, 52748)
if err != nil {
t.Fatalf("failed to get volume: %v", err)
}
jsonBytes, err := json.Marshal(volume)
if err != nil {
t.Fatalf("failed to marshal volume: %v", err)
}
fmt.Println(string(jsonBytes))
}
func TestBilinovel_GetChapter(t *testing.T) {
bilinovel, err := bilinovel.New()
if err != nil {
t.Fatalf("failed to create bilinovel: %v", err)
}
bilinovel.SetDebug(true)
chapter, err := bilinovel.GetChapter(1410, 52748, 52752)
if err != nil {
t.Fatalf("failed to get chapter: %v", err)
}
jsonBytes, err := json.Marshal(chapter)
if err != nil {
t.Fatalf("failed to marshal chapter: %v", err)
}
fmt.Println(string(jsonBytes))
}

55
text/wrapper.go Normal file
View File

@@ -0,0 +1,55 @@
package text
import (
"bilinovel-downloader/model"
"bilinovel-downloader/utils"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/PuerkitoBio/goquery"
)
func PackVolumeToText(volume *model.Volume, outputPath string) error {
outputPath = filepath.Join(outputPath, utils.CleanDirName(volume.Title))
_, err := os.Stat(outputPath)
if err != nil {
if os.IsNotExist(err) {
err = os.MkdirAll(outputPath, 0755)
if err != nil {
return fmt.Errorf("failed to create output directory: %v", err)
}
} else {
return fmt.Errorf("failed to get output directory: %v", err)
}
} else {
err = os.RemoveAll(outputPath)
if err != nil {
return fmt.Errorf("failed to remove output directory: %v", err)
}
err = os.MkdirAll(outputPath, 0755)
if err != nil {
return fmt.Errorf("failed to create output directory: %v", err)
}
}
for i, chapter := range volume.Chapters {
chapterPath := filepath.Join(outputPath, fmt.Sprintf("%03d-%s.txt", i, chapter.Title))
chapterFile, err := os.Create(chapterPath)
if err != nil {
return fmt.Errorf("failed to create chapter file: %v", err)
}
defer chapterFile.Close()
doc, err := goquery.NewDocumentFromReader(strings.NewReader(chapter.Content.Html))
if err != nil {
return fmt.Errorf("failed to create chapter file: %v", err)
}
doc.Find("img").Remove()
text := doc.Text()
_, err = chapterFile.WriteString(text)
if err != nil {
return fmt.Errorf("failed to write chapter file: %v", err)
}
}
return nil
}

34
utils/request.go
View File

@@ -9,11 +9,19 @@ import (
"github.com/go-resty/resty/v2"
)
var client *resty.Client
type RestyClient struct {
client *resty.Client
concurrency int
sem chan struct{}
}
func init() {
client = resty.New()
client.SetTransport(&http.Transport{
func NewRestyClient(concurrency int) *RestyClient {
client := &RestyClient{
client: resty.New(),
concurrency: concurrency,
sem: make(chan struct{}, concurrency),
}
client.client.SetTransport(&http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
if addr == "www.bilinovel.com:443" {
addr = "64.140.161.52:443"
@@ -24,7 +32,16 @@ func init() {
},
TLSHandshakeTimeout: 10 * time.Second,
})
client.SetRetryCount(10).
client.client.
OnBeforeRequest(func(c *resty.Client, req *resty.Request) error {
client.sem <- struct{}{}
return nil
}).
OnAfterResponse(func(c *resty.Client, resp *resty.Response) error {
<-client.sem
return nil
})
client.client.SetRetryCount(10).
SetRetryWaitTime(3 * time.Second).
SetRetryAfter(func(client *resty.Client, resp *resty.Response) (time.Duration, error) {
if resp.StatusCode() == http.StatusTooManyRequests {
@@ -43,10 +60,13 @@ func init() {
AddRetryCondition(func(r *resty.Response, err error) bool {
return err != nil || r.StatusCode() == http.StatusTooManyRequests
})
client.client.SetLogger(disableLogger{}).SetHeader("Accept-Charset", "utf-8").SetHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0")
return client
}
func Request() *resty.Request {
return client.R().SetLogger(disableLogger{}).SetHeader("Accept-Charset", "utf-8").SetHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0")
func (c *RestyClient) R() *resty.Request {
return c.client.R()
}
type disableLogger struct{}