nite/game-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

fix onlinefix

Browse Source
This commit is contained in:
Nite07 2024-12-10 21:37:16 +08:00
parent 8c3b59d622
commit cdf263b611
5 changed files with 118 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/freegog.go
View File

@ -47,7 +47,7 @@ func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) {
}
jsonBytes, err := json.Marshal(session)
if err == nil {
_ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour)
_ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 1*time.Hour)
}
}
return &session, nil

91
crawler/onlinefix.go
View File

@ -11,7 +11,9 @@ import (
"regexp"
"strconv"
"strings"
"time"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/constant"
"pcgamedb/db"
@ -23,14 +25,12 @@ import (
)
type OnlineFixCrawler struct {
logger *zap.Logger
cookies []*http.Cookie
logger *zap.Logger
}
func NewOnlineFixCrawler(logger *zap.Logger) *OnlineFixCrawler {
return &OnlineFixCrawler{
logger: logger,
cookies: []*http.Cookie{},
logger: logger,
}
}
@ -49,7 +49,8 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
c.logger.Error("Failed to fetch", zap.Error(err))
return nil, err
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
body := utils.Windows1251ToUTF8(resp.Body())
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
c.logger.Error("Failed to parse HTML", zap.Error(err))
return nil, err
@ -102,13 +103,14 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
if err != nil {
return nil, err
}
body := utils.Windows1251ToUTF8(resp.Body())
titleRegex := regexp.MustCompile(`(?i)<h1.*?>(.*?)</h1>`)
titleRegexRes := titleRegex.FindAllStringSubmatch(string(resp.Body()), -1)
titleRegexRes := titleRegex.FindAllStringSubmatch(string(body), -1)
if len(titleRegexRes) == 0 {
return nil, errors.New("failed to find title")
}
downloadRegex := regexp.MustCompile(`(?i)<a[^>]*\bhref="([^"]+)"[^>]*>(Скачать Torrent|Скачать торрент)</a>`)
downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(resp.Body()), -1)
downloadRegex := regexp.MustCompile(`(?i)<a[^>]+\bhref="([^"]+)"[^>]+>(Скачать Torrent|Скачать торрент)</a>`)
downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(body), -1)
if len(downloadRegexRes) == 0 {
return nil, errors.New("failed to find download button")
}
@ -122,12 +124,13 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.Author = "OnlineFix"
item.Size = "0"
resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1])
body = utils.Windows1251ToUTF8(resp.Body())
if err != nil {
return nil, err
}
if strings.Contains(downloadRegexRes[0][1], "uploads.online-fix.me") {
magnetRegex := regexp.MustCompile(`(?i)"(.*?).torrent"`)
magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(resp.Body()), -1)
magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(body), -1)
if len(magnetRegexRes) == 0 {
return nil, errors.New("failed to find magnet")
}
@ -140,12 +143,12 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
return nil, err
}
} else if strings.Contains(downloadRegexRes[0][1], "online-fix.me/ext") {
if strings.Contains(string(resp.Body()), "mega.nz") {
if strings.Contains(string(body), "mega.nz") {
if !config.Config.MegaAvaliable {
return nil, errors.New("mega is not avaliable")
}
megaRegex := regexp.MustCompile(`(?i)location.href=\\'([^\\']*)\\'`)
megaRegexRes := megaRegex.FindAllStringSubmatch(string(resp.Body()), -1)
megaRegexRes := megaRegex.FindAllStringSubmatch(string(body), -1)
if len(megaRegexRes) == 0 {
return nil, errors.New("failed to find download link")
}
@ -229,37 +232,47 @@ type csrf struct {
}
func (c *OnlineFixCrawler) getCookies() ([]*http.Cookie, error) {
if c.cookies == nil {
resp, err := utils.Request().SetHeaders(map[string]string{
"X-Requested-With": "XMLHttpRequest",
"Referer": constant.OnlineFixURL,
}).Get(constant.OnlineFixCSRFURL)
if err != nil {
val, exists := cache.Get("onlinefix_cookies")
if exists {
var cookies []*http.Cookie
if err := json.Unmarshal([]byte(val), &cookies); err != nil {
return nil, err
}
var csrf csrf
if err = json.Unmarshal(resp.Body(), &csrf); err != nil {
return nil, err
}
c.cookies = resp.Cookies()
params := url.Values{}
params.Add("login_name", config.Config.OnlineFix.User)
params.Add("login_password", config.Config.OnlineFix.Password)
params.Add(csrf.Field, csrf.Value)
params.Add("login", "submit")
resp, err = utils.Request().SetHeaders(map[string]string{
"Origin": constant.OnlineFixURL,
"Content-Type": "application/x-www-form-urlencoded",
"Referer": constant.OnlineFixURL,
}).SetCookies(c.cookies).SetBody(params).Post(constant.OnlineFixURL)
if err != nil {
return nil, err
}
c.cookies = append(c.cookies, resp.Cookies()...)
return cookies, nil
}
return c.cookies, nil
resp, err := utils.Request().SetHeaders(map[string]string{
"X-Requested-With": "XMLHttpRequest",
"Referer": constant.OnlineFixURL,
}).Get(constant.OnlineFixCSRFURL)
if err != nil {
return nil, err
}
var csrf csrf
if err = json.Unmarshal(resp.Body(), &csrf); err != nil {
return nil, err
}
cookies := resp.Cookies()
params := url.Values{}
params.Add("login_name", config.Config.OnlineFix.User)
params.Add("login_password", config.Config.OnlineFix.Password)
params.Add(csrf.Field, csrf.Value)
params.Add("login", "submit")
resp, err = utils.Request().SetHeaders(map[string]string{
"Origin": constant.OnlineFixURL,
"Content-Type": "application/x-www-form-urlencoded",
"Referer": constant.OnlineFixURL,
}).SetCookies(cookies).SetBody(params.Encode()).Post(constant.OnlineFixURL)
if err != nil {
return nil, err
}
cookies = resp.Cookies()
jsonBytes, _ := json.Marshal(cookies)
_ = cache.SetWithExpire("onlinefix_cookies", string(jsonBytes), time.Hour)
return cookies, nil
}
func OnlineFixFormatter(name string) string {

71
server/middleware/log.go
View File

@ -1,12 +1,10 @@
package middleware
import (
"strconv"
"pcgamedb/log"
"strings"
"time"
"pcgamedb/log"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
)
@ -14,34 +12,61 @@ import (
func Logger() gin.HandlerFunc {
return func(c *gin.Context) {
startTime := time.Now()
path := c.Request.URL.Path
raw := c.Request.URL.RawQuery
c.Next()
endTime := time.Now()
latencyTime := endTime.Sub(startTime).Milliseconds()
reqMethod := c.Request.Method
reqURI := c.Request.RequestURI
statusCode := c.Writer.Status()
clientIP := c.ClientIP()
if strings.HasPrefix(reqURI, "/swagger/") ||
strings.EqualFold(reqURI, "/favicon.ico") {
if shouldSkipLog(path) {
return
}
log.Logger.Info(
"request",
zap.Int("code", statusCode),
zap.String("method", reqMethod),
zap.String("uri", reqURI),
zap.String("ip", clientIP),
zap.String("latency", strconv.Itoa(int(latencyTime))+"ms"),
)
if raw != "" {
path = path + "?" + raw
}
fields := []zap.Field{
zap.Int("status", c.Writer.Status()),
zap.String("method", c.Request.Method),
zap.String("path", path),
zap.String("ip", getRealIP(c)),
zap.Duration("latency", time.Since(startTime)),
}
if len(c.Errors) > 0 {
for _, e := range c.Errors.Errors() {
log.Logger.Error(e)
}
fields = append(fields, zap.Strings("errors", c.Errors.Errors()))
}
log.Logger.Info("Request", fields...)
}
}
func getRealIP(c *gin.Context) string {
if ip := c.GetHeader("X-Real-IP"); ip != "" {
return ip
}
if ip := c.GetHeader("X-Forwarded-For"); ip != "" {
if index := strings.Index(ip, ","); index != -1 {
return strings.TrimSpace(ip[:index])
}
return ip
}
if ip := c.GetHeader("X-Originating-IP"); ip != "" {
return ip
}
return c.ClientIP()
}
func shouldSkipLog(path string) bool {
skipPaths := []string{
"/swagger/",
"/favicon.ico",
"/health",
}
for _, p := range skipPaths {
if strings.HasPrefix(path, p) {
return true
}
}
return false
}

16
utils/decoder.go Normal file
View File

@ -0,0 +1,16 @@
package utils
import (
"bytes"
"io"
"golang.org/x/text/encoding/htmlindex"
"golang.org/x/text/transform"
)
func Windows1251ToUTF8(b []byte) []byte {
decoder, _ := htmlindex.Get("windows-1251")
reader := transform.NewReader(bytes.NewReader(b), decoder.NewDecoder().Transformer)
body, _ := io.ReadAll(reader)
return body
}

2
utils/request.go
View File

@ -14,5 +14,5 @@ func init() {
}
func Request() *resty.Request {
return client.R().SetHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0")
return client.R().SetHeader("Accept-Charset", "utf-8").SetHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0")
}