package bilinovel import ( "bilinovel-downloader/model" "bilinovel-downloader/utils" "bytes" "crypto/sha256" _ "embed" "fmt" "log" "net/http" "os" "path" "path/filepath" "regexp" "slices" "strconv" "strings" "sync" "github.com/PuerkitoBio/goquery" mapper "github.com/bestnite/font-mapper" "github.com/playwright-community/playwright-go" ) //go:embed read.ttf var readTTF []byte //go:embed "MI LANTING.ttf" var miLantingTTF []byte type Bilinovel struct { fontMapper *mapper.GlyphOutlineMapper textOnly bool restyClient *utils.RestyClient // 浏览器实例复用 browser playwright.Browser browserContext playwright.BrowserContext pages map[string]playwright.Page concurrency int concurrentChan chan any } type BilinovelNewOption struct { Headless bool Concurrency int } func New(option BilinovelNewOption) (*Bilinovel, error) { fontMapper, err := mapper.NewGlyphOutlineMapper(readTTF, miLantingTTF) if err != nil { return nil, fmt.Errorf("failed to create font mapper: %v", err) } restyClient := utils.NewRestyClient(50) b := &Bilinovel{ fontMapper: fontMapper, textOnly: false, restyClient: restyClient, pages: make(map[string]playwright.Page), concurrency: option.Concurrency, concurrentChan: make(chan any, option.Concurrency), } // 初始化浏览器实例 err = b.initBrowser(option.Headless) if err != nil { return nil, fmt.Errorf("failed to init browser: %v", err) } return b, nil } func (b *Bilinovel) SetTextOnly(textOnly bool) { b.textOnly = textOnly } func (b *Bilinovel) GetExtraFiles() []model.ExtraFile { return nil } // initBrowser 初始化浏览器实例 func (b *Bilinovel) initBrowser(headless bool) error { pw, err := playwright.Run() if err != nil { return fmt.Errorf("could not start playwright: %w", err) } b.browser, err = pw.Chromium.Launch(playwright.BrowserTypeLaunchOptions{ Headless: playwright.Bool(headless), }) if err != nil { return fmt.Errorf("could not launch browser: %w", err) } b.browserContext, err = b.browser.NewContext() if err != nil { return fmt.Errorf("could not create browser context: %w", err) } log.Println("Browser initialized successfully") return nil } // Close 清理资源 func (b *Bilinovel) Close() error { if b.browser != nil { if err := b.browser.Close(); err != nil { log.Printf("could not close browser: %v", err) } b.browser = nil b.browserContext = nil } return nil } //go:embed style.css var styleCSS []byte func (b *Bilinovel) GetStyleCSS() string { return string(styleCSS) } func (b *Bilinovel) GetNovel(novelId int, skipChapterContent bool, skipVolumes []int) (*model.Novel, error) { log.Printf("Getting novel %v\n", novelId) novelUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v.html", novelId) resp, err := b.restyClient.R().Get(novelUrl) if err != nil { return nil, fmt.Errorf("failed to get novel info: %w", err) } if resp.StatusCode() != http.StatusOK { return nil, fmt.Errorf("failed to get novel info: %v", resp.Status()) } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body())) if err != nil { return nil, fmt.Errorf("failed to parse html: %v", err) } novel := &model.Novel{} novel.Title = strings.TrimSpace(doc.Find(".book-title").First().Text()) novel.Description = strings.TrimSpace(doc.Find(".book-summary>content").First().Text()) novel.Id = novelId doc.Find(".authorname>a").Each(func(i int, s *goquery.Selection) { novel.Authors = append(novel.Authors, strings.TrimSpace(s.Text())) }) doc.Find(".illname>a").Each(func(i int, s *goquery.Selection) { novel.Authors = append(novel.Authors, strings.TrimSpace(s.Text())) }) volumes, err := b.getAllVolumes(novelId, skipChapterContent, skipVolumes) if err != nil { return nil, fmt.Errorf("failed to get novel volumes: %v", err) } novel.Volumes = volumes return novel, nil } func (b *Bilinovel) GetVolume(novelId int, volumeId int, skipChapterContent bool) (*model.Volume, error) { log.Printf("Getting volume %v of novel %v\n", volumeId, novelId) novelUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/catalog", novelId) resp, err := b.restyClient.R().Get(novelUrl) if err != nil { return nil, fmt.Errorf("failed to get novel info: %w", err) } if resp.StatusCode() != http.StatusOK { return nil, fmt.Errorf("failed to get novel info: %v", resp.Status()) } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body())) if err != nil { return nil, fmt.Errorf("failed to parse html: %v", err) } seriesIdx := 0 doc.Find("a.volume-cover-img").Each(func(i int, s *goquery.Selection) { if s.AttrOr("href", "") == fmt.Sprintf("/novel/%v/vol_%v.html", novelId, volumeId) { seriesIdx = i + 1 } }) novelTitle := strings.TrimSpace(doc.Find(".book-title").First().Text()) if seriesIdx == 0 { return nil, fmt.Errorf("volume not found: %v", volumeId) } volumeUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/vol_%v.html", novelId, volumeId) resp, err = b.restyClient.R().Get(volumeUrl) if err != nil { return nil, fmt.Errorf("failed to get novel info: %v", err) } if resp.StatusCode() != http.StatusOK { return nil, fmt.Errorf("failed to get novel info: %v", resp.Status()) } doc, err = goquery.NewDocumentFromReader(bytes.NewReader(resp.Body())) if err != nil { return nil, fmt.Errorf("failed to parse html: %v", err) } volume := &model.Volume{} volume.NovelId = novelId volume.NovelTitle = novelTitle volume.Id = volumeId volume.SeriesIdx = seriesIdx volume.Title = strings.TrimSpace(doc.Find(".book-title").First().Text()) volume.Description = strings.TrimSpace(doc.Find(".book-summary>content").First().Text()) volume.Url = volumeUrl volume.Chapters = make([]*model.Chapter, 0) volume.CoverUrl = doc.Find(".book-cover").First().AttrOr("src", "") cover, err := b.getImg(volume.CoverUrl) if err != nil { return nil, fmt.Errorf("failed to get cover: %v", err) } volume.Cover = cover doc.Find(".authorname>a").Each(func(i int, s *goquery.Selection) { volume.Authors = append(volume.Authors, strings.TrimSpace(s.Text())) }) doc.Find(".illname>a").Each(func(i int, s *goquery.Selection) { volume.Authors = append(volume.Authors, strings.TrimSpace(s.Text())) }) doc.Find(".chapter-li.jsChapter").Each(func(i int, s *goquery.Selection) { volume.Chapters = append(volume.Chapters, &model.Chapter{ Title: s.Find("a").Text(), Url: fmt.Sprintf("https://www.bilinovel.com%v", s.Find("a").AttrOr("href", "")), }) }) idRegexp := regexp.MustCompile(`/novel/(\d+)/(\d+).html`) if !skipChapterContent { for i := range volume.Chapters { matches := idRegexp.FindStringSubmatch(volume.Chapters[i].Url) if len(matches) > 0 { chapterId, err := strconv.Atoi(matches[2]) if err != nil { return nil, fmt.Errorf("failed to convert chapter id: %v", err) } chapter, err := b.GetChapter(novelId, volumeId, chapterId) if err != nil { return nil, fmt.Errorf("failed to get chapter: %v", err) } chapter.Id = chapterId volume.Chapters[i] = chapter } else { return nil, fmt.Errorf("failed to get chapter id: %v", volume.Chapters[i].Url) } } } return volume, nil } func (b *Bilinovel) getAllVolumes(novelId int, skipChapterContent bool, skipVolumes []int) ([]*model.Volume, error) { log.Printf("Getting all volumes of novel %v\n", novelId) catelogUrl := fmt.Sprintf("https://www.bilinovel.com/novel/%v/catalog", novelId) resp, err := b.restyClient.R().Get(catelogUrl) if err != nil { return nil, fmt.Errorf("failed to get catelog: %v", err) } if resp.StatusCode() != http.StatusOK { return nil, fmt.Errorf("failed to get catelog: %v", resp.Status()) } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body())) if err != nil { return nil, fmt.Errorf("failed to parse html: %v", err) } volumeRegexp := regexp.MustCompile(fmt.Sprintf(`/novel/%v/vol_(\d+).html`, novelId)) volumeIds := make([]string, 0) doc.Find("a.volume-cover-img").Each(func(i int, s *goquery.Selection) { link := s.AttrOr("href", "") matches := volumeRegexp.FindStringSubmatch(link) if len(matches) > 0 { volumeIds = append(volumeIds, matches[1]) } }) volumes := make([]*model.Volume, len(volumeIds)) var wg sync.WaitGroup var mu sync.Mutex // 保护 volumes 写入的互斥锁 for i, volumeIdStr := range volumeIds { wg.Add(1) b.concurrentChan <- struct{}{} // 获取一个并发槽 go func(i int, volumeIdStr string) { defer wg.Done() defer func() { <-b.concurrentChan }() // 释放并发槽 volumeId, err := strconv.Atoi(volumeIdStr) if err != nil { log.Printf("failed to convert volume id %s: %v", volumeIdStr, err) return } if slices.Contains(skipVolumes, volumeId) { return } volume, err := b.GetVolume(novelId, volumeId, skipChapterContent) if err != nil { log.Printf("failed to get volume info for novel %d, volume %d: %v", novelId, volumeId, err) return } volume.SeriesIdx = i // 关闭浏览器标签页 pwPageKey := fmt.Sprintf("%v-%v", novelId, volumeId) if pwPage, ok := b.pages[pwPageKey]; ok { _ = pwPage.Close() delete(b.pages, pwPageKey) } mu.Lock() volumes[i] = volume mu.Unlock() }(i, volumeIdStr) } wg.Wait() // 过滤掉获取失败的 nil volume filteredVolumes := make([]*model.Volume, 0, len(volumes)) for _, vol := range volumes { if vol != nil { filteredVolumes = append(filteredVolumes, vol) } } return filteredVolumes, nil } func (b *Bilinovel) GetChapter(novelId int, volumeId int, chapterId int) (*model.Chapter, error) { log.Printf("Getting chapter %v of novel %v\n", chapterId, novelId) pageNum := 1 chapter := &model.Chapter{ Id: chapterId, NovelId: novelId, VolumeId: volumeId, Url: fmt.Sprintf("https://www.bilinovel.com/novel/%v/%v.html", novelId, chapterId), } for { pwPageKey := fmt.Sprintf("%v-%v", novelId, volumeId) if _, ok := b.pages[pwPageKey]; !ok { pwPage, err := b.browserContext.NewPage() if err != nil { return nil, fmt.Errorf("failed to create browser page: %w", err) } b.pages[pwPageKey] = pwPage } hasNext, err := b.getChapterByPage(b.pages[pwPageKey], chapter, pageNum) if err != nil { return nil, fmt.Errorf("failed to download chapter: %w", err) } if !hasNext { break } pageNum++ } return chapter, nil } func (b *Bilinovel) getChapterByPage(pwPage playwright.Page, chapter *model.Chapter, pageNum int) (bool, error) { log.Printf("Getting chapter %v by page %v\n", chapter.Id, pageNum) Url := strings.TrimSuffix(chapter.Url, ".html") + fmt.Sprintf("_%v.html", pageNum) hasNext := false headers := map[string]string{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Accept-Language": "zh-CN,zh;q=0.9,en-GB;q=0.8,en;q=0.7,zh-TW;q=0.6", "Cookie": "night=1;", } resp, err := b.restyClient.R().SetHeaders(headers).Get(Url) if err != nil { return false, fmt.Errorf("failed to get chapter: %w", err) } if resp.StatusCode() != http.StatusOK { return false, fmt.Errorf("failed to get chapter: %v", resp.Status()) } if strings.Contains(resp.String(), `下一頁`) { hasNext = true } html := resp.Body() // 解决乱序问题 resortedHtml, err := b.processContentWithPlaywright(pwPage, string(html)) if err != nil { return false, fmt.Errorf("failed to process html: %w", err) } doc, err := goquery.NewDocumentFromReader(strings.NewReader(resortedHtml)) if err != nil { return false, fmt.Errorf("failed to parse html: %w", err) } if pageNum == 1 { chapter.Title = doc.Find("#atitle").Text() } content := doc.Find("#acontent").First() content.Find(".cgo").Remove() content.Find("center").Remove() content.Find(".google-auto-placed").Remove() if strings.Contains(resp.String(), `font-family: "read"`) { html, err := content.Find("p").Last().Html() if err != nil { return false, fmt.Errorf("failed to get html: %v", err) } builder := strings.Builder{} for _, r := range html { _, newRune, ok := b.fontMapper.MappingRune(r) if ok { builder.WriteRune(newRune) } } content.Find("p").Last().SetHtml(builder.String()) } if b.textOnly { content.Find("img").Remove() } else { content.Find("img").Each(func(i int, s *goquery.Selection) { imgUrl := s.AttrOr("data-src", "") if imgUrl == "" { imgUrl = s.AttrOr("src", "") if imgUrl == "" { return } } imageHash := sha256.Sum256([]byte(imgUrl)) imageFilename := fmt.Sprintf("%x%s", string(imageHash[:]), path.Ext(imgUrl)) s.SetAttr("src", imageFilename) s.SetAttr("alt", imgUrl) s.RemoveAttr("class") img, err := b.getImg(imgUrl) if err != nil { return } if chapter.Content == nil { chapter.Content = &model.ChaperContent{} } if chapter.Content.Images == nil { chapter.Content.Images = make(map[string][]byte) } chapter.Content.Images[imageFilename] = img }) } doc.Find("*").Each(func(i int, s *goquery.Selection) { if len(s.Nodes) > 0 && len(s.Nodes[0].Attr) > 0 { // 遍历元素的所有属性 for _, attr := range s.Nodes[0].Attr { // 3. 检查属性名是否以 "data-k" 开头,且属性值是否为空 if strings.HasPrefix(attr.Key, "data-k") { // 4. 如果满足条件,就移除这个属性 s.RemoveAttr(attr.Key) } } } }) htmlStr, err := content.Html() if err != nil { return false, fmt.Errorf("failed to get html: %v", err) } if chapter.Content == nil { chapter.Content = &model.ChaperContent{} } chapter.Content.Html += strings.TrimSpace(htmlStr) return hasNext, nil } func (b *Bilinovel) getImg(url string) ([]byte, error) { log.Printf("Getting img %v\n", url) resp, err := b.restyClient.R().SetHeader("Referer", "https://www.bilinovel.com").Get(url) if err != nil { return nil, err } return resp.Body(), nil } // processContentWithPlaywright 使用复用的浏览器实例处理内容 func (b *Bilinovel) processContentWithPlaywright(page playwright.Page, htmlContent string) (string, error) { tempFile, err := os.CreateTemp("", "bilinovel-temp-*.html") if err != nil { return "", fmt.Errorf("failed to create temp file: %w", err) } defer os.Remove(tempFile.Name()) _, err = tempFile.WriteString(htmlContent) if err != nil { return "", fmt.Errorf("failed to write temp file: %w", err) } tempFile.Close() tempFilePath := tempFile.Name() _, err = page.ExpectResponse(func(url string) bool { return strings.Contains(url, "chapterlog.js") }, func() error { _, err = page.Goto("file://" + filepath.ToSlash(tempFilePath)) if err != nil { return fmt.Errorf("could not navigate to file: %w", err) } return nil }, playwright.PageExpectResponseOptions{ Timeout: playwright.Float(5000), }) if err != nil { return "", fmt.Errorf("failed to wait for network request finish") } err = page.Locator("#acontent").WaitFor(playwright.LocatorWaitForOptions{ State: playwright.WaitForSelectorStateVisible, }) if err != nil { return "", fmt.Errorf("could not wait for #acontent: %w", err) } // 遍历所有 #acontent 的子元素, 通过 window.getComputedStyle().display 检测是否是 none, 如果是 none 则从页面删除这个元素 result, err := page.Evaluate(` (function() { const acontent = document.getElementById('acontent'); if (!acontent) { return 'acontent element not found'; } let removedCount = 0; const elements = acontent.querySelectorAll('*'); // 从后往前遍历,避免删除元素时影响索引 for (let i = elements.length - 1; i >= 0; i--) { const element = elements[i]; const computedStyle = window.getComputedStyle(element); if (computedStyle.display === 'none' || computedStyle.transform == 'matrix(0, 0, 0, 0, 0, 0)') { element.remove(); removedCount++; } } return 'Removed ' + removedCount + ' hidden elements'; })() `) if err != nil { return "", fmt.Errorf("failed to remove hidden elements: %w", err) } log.Printf("Hidden elements removal result: %s", result) processedHTML, err := page.Content() if err != nil { return "", fmt.Errorf("could not get page content: %w", err) } return processedHTML, nil }