275 lines
7.5 KiB
Go
275 lines
7.5 KiB
Go
package crawler
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"pcgamedb/config"
|
|
"pcgamedb/constant"
|
|
"pcgamedb/db"
|
|
"pcgamedb/model"
|
|
"pcgamedb/utils"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type OnlineFixCrawler struct {
|
|
logger *zap.Logger
|
|
cookies []*http.Cookie
|
|
}
|
|
|
|
func NewOnlineFixCrawler(logger *zap.Logger) *OnlineFixCrawler {
|
|
return &OnlineFixCrawler{
|
|
logger: logger,
|
|
cookies: []*http.Cookie{},
|
|
}
|
|
}
|
|
|
|
func (c *OnlineFixCrawler) Name() string {
|
|
return "OnlineFixCrawler"
|
|
}
|
|
|
|
func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
|
if !config.Config.OnlineFixAvaliable {
|
|
c.logger.Error("Need Online Fix account")
|
|
return nil, errors.New("online Fix is not available")
|
|
}
|
|
cookies, err := c.getCookies()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
requestURL := fmt.Sprintf("%s/page/%d/", constant.OnlineFixURL, page)
|
|
resp, err := utils.Request().SetHeader("Referer", constant.OnlineFixURL).SetCookies(cookies).Get(requestURL)
|
|
if err != nil {
|
|
c.logger.Error("Failed to fetch", zap.Error(err))
|
|
return nil, err
|
|
}
|
|
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
|
if err != nil {
|
|
c.logger.Error("Failed to parse HTML", zap.Error(err))
|
|
return nil, err
|
|
}
|
|
var urls []string
|
|
var updateFlags []string //link+date
|
|
doc.Find("article.news").Each(func(i int, s *goquery.Selection) {
|
|
urls = append(urls, s.Find(".big-link").First().AttrOr("href", ""))
|
|
updateFlags = append(
|
|
updateFlags,
|
|
s.Find(".big-link").First().AttrOr("href", "")+
|
|
s.Find("time").Text(),
|
|
)
|
|
})
|
|
|
|
var res []*model.GameItem
|
|
for i, u := range urls {
|
|
if db.IsOnlineFixCrawled(updateFlags[i]) {
|
|
continue
|
|
}
|
|
c.logger.Info("Crawling", zap.String("URL", u))
|
|
item, err := c.CrawlByUrl(u)
|
|
if err != nil {
|
|
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
|
|
continue
|
|
}
|
|
item.UpdateFlag = updateFlags[i]
|
|
err = db.SaveGameItem(item)
|
|
if err != nil {
|
|
c.logger.Warn("Failed to save", zap.Error(err))
|
|
continue
|
|
}
|
|
res = append(res, item)
|
|
if err := OrganizeGameItem(item); err != nil {
|
|
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
|
continue
|
|
}
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
|
cookies, err := c.getCookies()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := utils.Request().SetHeaders(map[string]string{
|
|
"Referer": constant.OnlineFixURL,
|
|
}).SetCookies(cookies).Get(URL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
titleRegex := regexp.MustCompile(`(?i)<h1.*?>(.*?)</h1>`)
|
|
titleRegexRes := titleRegex.FindAllStringSubmatch(string(resp.Body()), -1)
|
|
if len(titleRegexRes) == 0 {
|
|
return nil, errors.New("failed to find title")
|
|
}
|
|
downloadRegex := regexp.MustCompile(`(?i)<a[^>]*\bhref="([^"]+)"[^>]*>(Скачать Torrent|Скачать торрент)</a>`)
|
|
downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(resp.Body()), -1)
|
|
if len(downloadRegexRes) == 0 {
|
|
return nil, errors.New("failed to find download button")
|
|
}
|
|
item, err := db.GetGameItemByUrl(URL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
item.RawName = titleRegexRes[0][1]
|
|
item.Name = OnlineFixFormatter(item.RawName)
|
|
item.Url = URL
|
|
item.Author = "OnlineFix"
|
|
item.Size = "0"
|
|
resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.Contains(downloadRegexRes[0][1], "uploads.online-fix.me") {
|
|
magnetRegex := regexp.MustCompile(`(?i)"(.*?).torrent"`)
|
|
magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(resp.Body()), -1)
|
|
if len(magnetRegexRes) == 0 {
|
|
return nil, errors.New("failed to find magnet")
|
|
}
|
|
resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1] + strings.Trim(magnetRegexRes[0][0], "\""))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(resp.Body())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
} else if strings.Contains(downloadRegexRes[0][1], "online-fix.me/ext") {
|
|
if strings.Contains(string(resp.Body()), "mega.nz") {
|
|
if !config.Config.MegaAvaliable {
|
|
return nil, errors.New("mega is not avaliable")
|
|
}
|
|
megaRegex := regexp.MustCompile(`(?i)location.href=\\'([^\\']*)\\'`)
|
|
megaRegexRes := megaRegex.FindAllStringSubmatch(string(resp.Body()), -1)
|
|
if len(megaRegexRes) == 0 {
|
|
return nil, errors.New("failed to find download link")
|
|
}
|
|
path, files, err := utils.MegaDownload(megaRegexRes[0][1], "torrent")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
torrent := ""
|
|
for _, file := range files {
|
|
if strings.HasSuffix(file, ".torrent") {
|
|
torrent = file
|
|
break
|
|
}
|
|
}
|
|
dataBytes, err := os.ReadFile(torrent)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(dataBytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_ = os.RemoveAll(path)
|
|
} else {
|
|
return nil, errors.New("failed to find download link")
|
|
}
|
|
} else {
|
|
return nil, errors.New("failed to find download link")
|
|
}
|
|
return item, nil
|
|
}
|
|
|
|
func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
|
|
var res []*model.GameItem
|
|
for _, page := range pages {
|
|
items, err := c.Crawl(page)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res = append(res, items...)
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
func (c *OnlineFixCrawler) CrawlAll() ([]*model.GameItem, error) {
|
|
var res []*model.GameItem
|
|
totalPageNum, err := c.GetTotalPageNum()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := 1; i <= totalPageNum; i++ {
|
|
items, err := c.Crawl(i)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res = append(res, items...)
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
func (c *OnlineFixCrawler) GetTotalPageNum() (int, error) {
|
|
resp, err := utils.Request().SetHeader("Referer", constant.OnlineFixURL).Get(constant.OnlineFixURL)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
pageRegex := regexp.MustCompile(`(?i)<a href="https://online-fix.me/page/(\d+)/">.*?</a>`)
|
|
pageRegexRes := pageRegex.FindAllStringSubmatch(string(resp.Body()), -1)
|
|
if len(pageRegexRes) == 0 {
|
|
return 0, err
|
|
}
|
|
totalPageNum, err := strconv.Atoi(pageRegexRes[len(pageRegexRes)-2][1])
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return totalPageNum, nil
|
|
}
|
|
|
|
type csrf struct {
|
|
Field string `json:"field"`
|
|
Value string `json:"value"`
|
|
}
|
|
|
|
func (c *OnlineFixCrawler) getCookies() ([]*http.Cookie, error) {
|
|
if c.cookies == nil {
|
|
resp, err := utils.Request().SetHeaders(map[string]string{
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
"Referer": constant.OnlineFixURL,
|
|
}).Get(constant.OnlineFixCSRFURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var csrf csrf
|
|
if err = json.Unmarshal(resp.Body(), &csrf); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
c.cookies = resp.Cookies()
|
|
|
|
params := url.Values{}
|
|
params.Add("login_name", config.Config.OnlineFix.User)
|
|
params.Add("login_password", config.Config.OnlineFix.Password)
|
|
params.Add(csrf.Field, csrf.Value)
|
|
params.Add("login", "submit")
|
|
resp, err = utils.Request().SetHeaders(map[string]string{
|
|
"Origin": constant.OnlineFixURL,
|
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
"Referer": constant.OnlineFixURL,
|
|
}).SetCookies(c.cookies).SetBody(params).Post(constant.OnlineFixURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
c.cookies = append(c.cookies, resp.Cookies()...)
|
|
}
|
|
return c.cookies, nil
|
|
}
|
|
|
|
func OnlineFixFormatter(name string) string {
|
|
name = strings.Replace(name, "по сети", "", -1)
|
|
reg1 := regexp.MustCompile(`(?i)\(.*?\)`)
|
|
name = reg1.ReplaceAllString(name, "")
|
|
return strings.TrimSpace(name)
|
|
}
|