pcgamedb/crawler/onlinefix.go

package crawler

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"pcgamedb/cache"
	"pcgamedb/config"
	"pcgamedb/constant"
	"pcgamedb/db"
	"pcgamedb/model"
	"pcgamedb/utils"

	"github.com/PuerkitoBio/goquery"
	"go.uber.org/zap"
)

type OnlineFixCrawler struct {
	logger *zap.Logger
}

func NewOnlineFixCrawler(logger *zap.Logger) *OnlineFixCrawler {
	return &OnlineFixCrawler{
		logger: logger,
	}
}

func (c *OnlineFixCrawler) Name() string {
	return "OnlineFixCrawler"
}

func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
	cookies, err := c.getCookies()
	if err != nil {
		return nil, err
	}
	requestURL := fmt.Sprintf("%s/page/%d/", constant.OnlineFixURL, page)
	resp, err := utils.Request().SetHeader("Referer", constant.OnlineFixURL).SetCookies(cookies).Get(requestURL)
	if err != nil {
		c.logger.Error("Failed to fetch", zap.Error(err))
		return nil, err
	}
	body := utils.Windows1251ToUTF8(resp.Body())
	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
	if err != nil {
		c.logger.Error("Failed to parse HTML", zap.Error(err))
		return nil, err
	}
	var urls []string
	var updateFlags []string //link+date
	doc.Find("article.news").Each(func(i int, s *goquery.Selection) {
		urls = append(urls, s.Find(".big-link").First().AttrOr("href", ""))
		updateFlags = append(
			updateFlags,
			s.Find(".big-link").First().AttrOr("href", "")+
				s.Find("time").Text(),
		)
	})

	var res []*model.GameItem
	for i, u := range urls {
		if db.IsOnlineFixCrawled(updateFlags[i]) {
			continue
		}
		c.logger.Info("Crawling", zap.String("URL", u))
		item, err := c.CrawlByUrl(u)
		if err != nil {
			c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
			continue
		}
		item.UpdateFlag = updateFlags[i]
		err = db.SaveGameItem(item)
		if err != nil {
			c.logger.Warn("Failed to save", zap.Error(err))
			continue
		}
		res = append(res, item)
		if err := OrganizeGameItem(item); err != nil {
			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
			continue
		}
	}
	return res, nil
}

func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
	cookies, err := c.getCookies()
	if err != nil {
		return nil, err
	}
	resp, err := utils.Request().SetHeaders(map[string]string{
		"Referer": constant.OnlineFixURL,
	}).SetCookies(cookies).Get(URL)
	if err != nil {
		return nil, err
	}
	body := utils.Windows1251ToUTF8(resp.Body())
	titleRegex := regexp.MustCompile(`(?i)<h1.*?>(.*?)</h1>`)
	titleRegexRes := titleRegex.FindAllStringSubmatch(string(body), -1)
	if len(titleRegexRes) == 0 {
		return nil, errors.New("failed to find title")
	}
	downloadRegex := regexp.MustCompile(`(?i)<a[^>]+\bhref="([^"]+)"[^>]+>(Скачать Torrent|Скачать торрент)</a>`)
	downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(body), -1)
	if len(downloadRegexRes) == 0 {
		return nil, errors.New("failed to find download button")
	}
	item, err := db.GetGameItemByUrl(URL)
	if err != nil {
		return nil, err
	}
	item.RawName = titleRegexRes[0][1]
	item.Name = OnlineFixFormatter(item.RawName)
	item.Url = URL
	item.Author = "OnlineFix"
	item.Size = "0"
	resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1])
	body = utils.Windows1251ToUTF8(resp.Body())
	if err != nil {
		return nil, err
	}
	if strings.Contains(downloadRegexRes[0][1], "uploads.online-fix.me") {
		magnetRegex := regexp.MustCompile(`(?i)"(.*?).torrent"`)
		magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(body), -1)
		if len(magnetRegexRes) == 0 {
			return nil, errors.New("failed to find magnet")
		}
		resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1] + strings.Trim(magnetRegexRes[0][0], "\""))
		if err != nil {
			return nil, err
		}
		item.Download, item.Size, err = utils.ConvertTorrentToMagnet(resp.Body())
		if err != nil {
			return nil, err
		}
	} else if strings.Contains(downloadRegexRes[0][1], "online-fix.me/ext") {
		if strings.Contains(string(body), "mega.nz") {
			if !config.Config.MegaAvaliable {
				return nil, errors.New("mega is not avaliable")
			}
			megaRegex := regexp.MustCompile(`(?i)location.href=\\'([^\\']*)\\'`)
			megaRegexRes := megaRegex.FindAllStringSubmatch(string(body), -1)
			if len(megaRegexRes) == 0 {
				return nil, errors.New("failed to find download link")
			}
			path, files, err := utils.MegaDownload(megaRegexRes[0][1], "torrent")
			if err != nil {
				return nil, err
			}
			torrent := ""
			for _, file := range files {
				if strings.HasSuffix(file, ".torrent") {
					torrent = file
					break
				}
			}
			dataBytes, err := os.ReadFile(torrent)
			if err != nil {
				return nil, err
			}
			item.Download, item.Size, err = utils.ConvertTorrentToMagnet(dataBytes)
			if err != nil {
				return nil, err
			}
			_ = os.RemoveAll(path)
		} else {
			return nil, errors.New("failed to find download link")
		}
	} else {
		return nil, errors.New("failed to find download link")
	}
	return item, nil
}

func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
	var res []*model.GameItem
	for _, page := range pages {
		items, err := c.Crawl(page)
		if err != nil {
			return nil, err
		}
		res = append(res, items...)
	}
	return res, nil
}

func (c *OnlineFixCrawler) CrawlAll() ([]*model.GameItem, error) {
	var res []*model.GameItem
	totalPageNum, err := c.GetTotalPageNum()
	if err != nil {
		return nil, err
	}
	for i := 1; i <= totalPageNum; i++ {
		items, err := c.Crawl(i)
		if err != nil {
			return nil, err
		}
		res = append(res, items...)
	}
	return res, nil
}

func (c *OnlineFixCrawler) GetTotalPageNum() (int, error) {
	resp, err := utils.Request().SetHeader("Referer", constant.OnlineFixURL).Get(constant.OnlineFixURL)
	if err != nil {
		return 0, err
	}
	pageRegex := regexp.MustCompile(`(?i)<a href="https://online-fix.me/page/(\d+)/">.*?</a>`)
	pageRegexRes := pageRegex.FindAllStringSubmatch(string(resp.Body()), -1)
	if len(pageRegexRes) == 0 {
		return 0, err
	}
	totalPageNum, err := strconv.Atoi(pageRegexRes[len(pageRegexRes)-2][1])
	if err != nil {
		return 0, err
	}
	return totalPageNum, nil
}

type csrf struct {
	Field string `json:"field"`
	Value string `json:"value"`
}

func (c *OnlineFixCrawler) getCookies() ([]*http.Cookie, error) {
	val, exists := cache.Get("onlinefix_cookies")
	if exists {
		var cookies []*http.Cookie
		if err := json.Unmarshal([]byte(val), &cookies); err != nil {
			return nil, err
		}
		return cookies, nil
	}

	resp, err := utils.Request().SetHeaders(map[string]string{
		"X-Requested-With": "XMLHttpRequest",
		"Referer":          constant.OnlineFixURL,
	}).Get(constant.OnlineFixCSRFURL)
	if err != nil {
		return nil, err
	}
	var csrf csrf
	if err = json.Unmarshal(resp.Body(), &csrf); err != nil {
		return nil, err
	}

	cookies := resp.Cookies()

	params := url.Values{}
	params.Add("login_name", config.Config.OnlineFix.User)
	params.Add("login_password", config.Config.OnlineFix.Password)
	params.Add(csrf.Field, csrf.Value)
	params.Add("login", "submit")
	resp, err = utils.Request().SetHeaders(map[string]string{
		"Origin":       constant.OnlineFixURL,
		"Content-Type": "application/x-www-form-urlencoded",
		"Referer":      constant.OnlineFixURL,
	}).SetCookies(cookies).SetBody(params.Encode()).Post(constant.OnlineFixURL)
	if err != nil {
		return nil, err
	}
	cookies = resp.Cookies()
	jsonBytes, _ := json.Marshal(cookies)
	_ = cache.SetWithExpire("onlinefix_cookies", string(jsonBytes), time.Hour)

	return cookies, nil
}

func OnlineFixFormatter(name string) string {
	name = strings.Replace(name, "по сети", "", -1)
	reg1 := regexp.MustCompile(`(?i)\(.*?\)`)
	name = reg1.ReplaceAllString(name, "")
	return strings.TrimSpace(name)
}