game-crawler/crawler/chovka.go

package crawler

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"strconv"
	"strings"

	"game-crawler/constant"
	"game-crawler/db"
	"game-crawler/model"
	"game-crawler/utils"

	"github.com/PuerkitoBio/goquery"
	"go.uber.org/zap"
)

type ChovkaCrawler struct {
	*BaseLogger
	*BaseError
	logger *zap.Logger
}

func NewChovkaCrawler(logger *zap.Logger) *ChovkaCrawler {
	return &ChovkaCrawler{
		BaseLogger: &BaseLogger{
			logger: logger,
		},
		BaseError: &BaseError{},
		logger:    logger,
	}
}

func (c *ChovkaCrawler) Name() string {
	return "ChovkaCrawler"
}

func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
	c.LogCrawlByUrlStart(URL)
	resp, err := utils.Request().SetLogger(c.logger.Sugar()).Get(URL)
	if err != nil {
		return nil, c.ErrRequest(URL, err)
	}

	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
	if err != nil {
		return nil, c.ErrParseDoc(URL, err)
	}

	item, err := db.GetGameItemByUrl(URL)
	if err != nil {
		return nil, c.ErrDBQuery(err)
	}

	item.Url = URL
	item.RawName = doc.Find(".inner-entry__title").First().Text()
	item.Name = ChovkaFormatter(item.RawName)
	item.Author = "Chovka"
	item.Platform = "windows"

	downloadURL := doc.Find(".download-torrent").AttrOr("href", "")
	if downloadURL == "" {
		return nil, c.ErrGetGameItemDetail(URL, errors.New("failed to get download URL"))
	}

	resp, err = utils.Request().SetLogger(c.logger.Sugar()).SetHeader("Referer", URL).Get(downloadURL)
	if err != nil {
		return nil, c.ErrGetGameItemDetail(URL, fmt.Errorf("failed to get download URL %s: %w", downloadURL, c.ErrRequest(downloadURL, err)))
	}

	magnet, size, err := utils.ConvertTorrentToMagnet(resp.Body())
	if err != nil {
		return nil, c.ErrGetGameItemDetail(URL, err)
	}

	item.Size = size
	item.Downloads = map[string]string{
		"magnet": magnet,
	}
	return item, nil
}

func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameItem, error) {
	c.LogCrawlStart(page)
	resp, err := utils.Request().SetLogger(c.logger.Sugar()).Get(fmt.Sprintf(constant.RepackInfoURL, page))
	if err != nil {
		return nil, c.ErrRequest(fmt.Sprintf(constant.RepackInfoURL, page), err)
	}

	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
	if err != nil {
		return nil, c.ErrParseDoc(fmt.Sprintf(constant.RepackInfoURL, page), err)
	}

	var urls []string
	var updateFlags []string
	doc.Find(".entry").Each(func(i int, s *goquery.Selection) {
		u, exist := s.Find(".entry__title.h2 a").Attr("href")
		if !exist {
			return
		}
		urls = append(urls, u)
		updateFlags = append(updateFlags, base64.StdEncoding.EncodeToString([]byte(s.Find(".entry__title.h2 a").Text())))
	})

	var res []*model.GameItem
	for i, u := range urls {
		if db.IsChovkaCrawled(updateFlags[i]) {
			c.LogCrawlSkip(u)
			continue
		}

		item, err := c.CrawlByUrl(u)
		if err != nil {
			c.LogCrawlByUrlError(u, err)
			continue
		}

		item.UpdateFlag = updateFlags[i]
		if err := db.SaveGameItem(item); err != nil {
			c.LogSaveGameItemError(u, err)
			continue
		}

		res = append(res, item)

		if err := OrganizeGameItem(item); err != nil {
			c.LogOrganizeGameWarn(item.ID, item.Name, err)
			continue
		}
	}

	return res, nil
}

func (c *ChovkaCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
	totalPageNum, err := c.GetTotalPageNum()
	if err != nil {
		return nil, c.ErrGetTotalPageNum(err)
	}
	var res []*model.GameItem
	for _, page := range pages {
		if page > totalPageNum || page < 1 {
			c.LogPageExceedWarn(page)
			continue
		}
		items, err := c.Crawl(page)
		if err != nil {
			c.LogCrawlError(page, err)
			continue
		}
		res = append(res, items...)
	}

	return res, nil
}

func (c *ChovkaCrawler) CrawlAll() ([]*model.GameItem, error) {
	totalPageNum, err := c.GetTotalPageNum()
	if err != nil {
		return nil, c.ErrGetTotalPageNum(err)
	}

	var res []*model.GameItem
	for i := 1; i <= totalPageNum; i++ {
		items, err := c.Crawl(i)
		if err != nil {
			c.LogCrawlError(i, err)
			continue
		}
		res = append(res, items...)
	}

	return res, nil
}

func (c *ChovkaCrawler) GetTotalPageNum() (int, error) {
	resp, err := utils.Request().SetLogger(c.logger.Sugar()).Get(fmt.Sprintf(constant.RepackInfoURL, 1))
	if err != nil {
		return 0, c.ErrRequest(fmt.Sprintf(constant.RepackInfoURL, 1), err)
	}

	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
	if err != nil {
		return 0, c.ErrParseDoc(fmt.Sprintf(constant.RepackInfoURL, 1), err)
	}

	pageStr := doc.Find(".pagination>a").Last().Text()
	totalPageNum, err := strconv.Atoi(pageStr)
	if err != nil {
		return 0, c.ErrParseInt(pageStr, err)
	}

	return totalPageNum, nil
}

func ChovkaFormatter(name string) string {
	cutoffs := []string{"| RePack", "| GOG", "| Portable"}
	for _, cutoff := range cutoffs {
		if idx := strings.Index(name, cutoff); idx != -1 {
			name = name[:idx]
		}
	}
	return strings.TrimSpace(name)
}