diff --git a/downloader/bilinovel/bilinovel.go b/downloader/bilinovel/bilinovel.go index 854b150..491a5e3 100644 --- a/downloader/bilinovel/bilinovel.go +++ b/downloader/bilinovel/bilinovel.go @@ -266,8 +266,7 @@ func (b *Bilinovel) GetVolume(novelId int, volumeId int, skipChapterContent bool if err != nil { return nil, fmt.Errorf("failed to get chapter: %v", err) } - chapter.Id = chapterId - volume.Chapters[i] = chapter + volume.Chapters[i] = mergeDownloadedChapter(volume.Chapters[i], chapter) } else { return nil, fmt.Errorf("failed to get chapter id: %v", volume.Chapters[i].Url) } @@ -277,6 +276,12 @@ func (b *Bilinovel) GetVolume(novelId int, volumeId int, skipChapterContent bool return volume, nil } +func mergeDownloadedChapter(listedChapter, downloadedChapter *model.Chapter) *model.Chapter { + downloadedChapter.Title = listedChapter.Title + downloadedChapter.Url = listedChapter.Url + return downloadedChapter +} + func (b *Bilinovel) getAllVolumes(novelId int, skipChapterContent bool, skipVolumes []int) ([]*model.Volume, error) { b.logger.Info("Getting all volumes of novel", slog.Int("novelId", novelId)) @@ -442,9 +447,7 @@ func (b *Bilinovel) getChapterByPage(pwPage playwright.Page, chapter *model.Chap chapter.Title = doc.Find("#atitle").Text() } content := doc.Find("#acontent").First() - content.Find(".cgo").Remove() - content.Find("center").Remove() - content.Find(".google-auto-placed").Remove() + cleanChapterContent(content) if strings.Contains(resortedHtml, `font-family: "read"`) { html, err := content.Find("p").Last().Html() @@ -518,6 +521,11 @@ func (b *Bilinovel) getChapterByPage(pwPage playwright.Page, chapter *model.Chap return hasNext, nil } +func cleanChapterContent(content *goquery.Selection) { + content.Find(".cgo, .csgo, .co, .google-auto-placed, ins.adsbygoogle").Remove() + content.Find("center").Remove() +} + func (b *Bilinovel) getImg(url string) ([]byte, error) { b.logger.Info("Getting img", slog.String("url", url)) resp, err := b.restyClient.R().SetHeader("Referer", "https://www.bilinovel.com").Get(url) diff --git a/downloader/bilinovel/bilinovel_test.go b/downloader/bilinovel/bilinovel_test.go new file mode 100644 index 0000000..c078b53 --- /dev/null +++ b/downloader/bilinovel/bilinovel_test.go @@ -0,0 +1,90 @@ +package bilinovel + +import ( + "bilinovel-downloader/model" + "strings" + "testing" + + "github.com/PuerkitoBio/goquery" +) + +func TestCleanChapterContentRemovesAdvertisementContainers(t *testing.T) { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(` +
+
+
+

保留正文

+ +
+
+
+ `)) + if err != nil { + t.Fatalf("failed to parse fixture: %v", err) + } + + content := doc.Find("#acontent").First() + cleanChapterContent(content) + + if remaining := content.Find(".cgo, .csgo, .co, .google-auto-placed, ins.adsbygoogle, iframe").Length(); remaining != 0 { + t.Fatalf("cleaned content still contains %d advertisement elements", remaining) + } + html, err := content.Html() + if err != nil { + t.Fatalf("failed to render cleaned content: %v", err) + } + if !strings.Contains(html, "

保留正文

") { + t.Fatalf("cleaned content lost chapter text: %s", html) + } +} + +func TestMergeDownloadedChapterPreservesVolumeTitle(t *testing.T) { + testCases := []struct { + name string + listedTitle string + downloadedTitle string + }{ + { + name: "circle placeholder", + listedTitle: "~第三败~ 欲擒主将,可是马儿太强", + downloadedTitle: "~第三败~ 〇擒主将,可是马儿太强", + }, + { + name: "heart placeholder", + listedTitle: "番外篇 真心话", + downloadedTitle: "番外篇 真♡话", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + listedChapter := &model.Chapter{ + Title: testCase.listedTitle, + Url: "https://www.bilinovel.com/novel/3095/186113.html", + } + downloadedChapter := &model.Chapter{ + Id: 186113, + Title: testCase.downloadedTitle, + Url: "https://www.bilinovel.com/novel/3095/186113_1.html", + Content: &model.ChaperContent{ + Html: "

正文

", + }, + } + + mergedChapter := mergeDownloadedChapter(listedChapter, downloadedChapter) + + if mergedChapter.Title != listedChapter.Title { + t.Fatalf("merged title = %q, want volume title %q", mergedChapter.Title, listedChapter.Title) + } + if mergedChapter.Content != downloadedChapter.Content { + t.Fatal("merged chapter did not retain downloaded content") + } + if mergedChapter.Url != listedChapter.Url { + t.Fatalf("merged URL = %q, want volume URL %q", mergedChapter.Url, listedChapter.Url) + } + if mergedChapter.Id != downloadedChapter.Id { + t.Fatalf("merged chapter id = %d, want downloaded id %d", mergedChapter.Id, downloadedChapter.Id) + } + }) + } +}