fix onlinefix

crawler/freegog.go

@@ -47,7 +47,7 @@ func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) {
 		}
 		jsonBytes, err := json.Marshal(session)
 		if err == nil {
-			_ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour)
+			_ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 1*time.Hour)
 		}
 	}
 	return &session, nil

crawler/onlinefix.go

@@ -11,7 +11,9 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
+	"pcgamedb/cache"
 	"pcgamedb/config"
 	"pcgamedb/constant"
 	"pcgamedb/db"
@@ -24,13 +26,11 @@ import (
 
 type OnlineFixCrawler struct {
 	logger *zap.Logger
-	cookies []*http.Cookie
 }
 
 func NewOnlineFixCrawler(logger *zap.Logger) *OnlineFixCrawler {
 	return &OnlineFixCrawler{
 		logger: logger,
-		cookies: []*http.Cookie{},
 	}
 }
 
@@ -49,7 +49,8 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
 		c.logger.Error("Failed to fetch", zap.Error(err))
 		return nil, err
 	}
-	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
+	body := utils.Windows1251ToUTF8(resp.Body())
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
 	if err != nil {
 		c.logger.Error("Failed to parse HTML", zap.Error(err))
 		return nil, err
@@ -102,13 +103,14 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
 	if err != nil {
 		return nil, err
 	}
+	body := utils.Windows1251ToUTF8(resp.Body())
 	titleRegex := regexp.MustCompile(`(?i)<h1.*?>(.*?)</h1>`)
-	titleRegexRes := titleRegex.FindAllStringSubmatch(string(resp.Body()), -1)
+	titleRegexRes := titleRegex.FindAllStringSubmatch(string(body), -1)
 	if len(titleRegexRes) == 0 {
 		return nil, errors.New("failed to find title")
 	}
-	downloadRegex := regexp.MustCompile(`(?i)<a[^>]*\bhref="([^"]+)"[^>]*>(Скачать Torrent|Скачать торрент)</a>`)
+	downloadRegex := regexp.MustCompile(`(?i)<a[^>]+\bhref="([^"]+)"[^>]+>(Скачать Torrent|Скачать торрент)</a>`)
-	downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(resp.Body()), -1)
+	downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(body), -1)
 	if len(downloadRegexRes) == 0 {
 		return nil, errors.New("failed to find download button")
 	}
@@ -122,12 +124,13 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
 	item.Author = "OnlineFix"
 	item.Size = "0"
 	resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1])
+	body = utils.Windows1251ToUTF8(resp.Body())
 	if err != nil {
 		return nil, err
 	}
 	if strings.Contains(downloadRegexRes[0][1], "uploads.online-fix.me") {
 		magnetRegex := regexp.MustCompile(`(?i)"(.*?).torrent"`)
-		magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(resp.Body()), -1)
+		magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(body), -1)
 		if len(magnetRegexRes) == 0 {
 			return nil, errors.New("failed to find magnet")
 		}
@@ -140,12 +143,12 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
 			return nil, err
 		}
 	} else if strings.Contains(downloadRegexRes[0][1], "online-fix.me/ext") {
-		if strings.Contains(string(resp.Body()), "mega.nz") {
+		if strings.Contains(string(body), "mega.nz") {
 			if !config.Config.MegaAvaliable {
 				return nil, errors.New("mega is not avaliable")
 			}
 			megaRegex := regexp.MustCompile(`(?i)location.href=\\'([^\\']*)\\'`)
-			megaRegexRes := megaRegex.FindAllStringSubmatch(string(resp.Body()), -1)
+			megaRegexRes := megaRegex.FindAllStringSubmatch(string(body), -1)
 			if len(megaRegexRes) == 0 {
 				return nil, errors.New("failed to find download link")
 			}
@@ -229,7 +232,15 @@ type csrf struct {
 }
 
 func (c *OnlineFixCrawler) getCookies() ([]*http.Cookie, error) {
-	if c.cookies == nil {
+	val, exists := cache.Get("onlinefix_cookies")
+	if exists {
+		var cookies []*http.Cookie
+		if err := json.Unmarshal([]byte(val), &cookies); err != nil {
+			return nil, err
+		}
+		return cookies, nil
+	}
+
 	resp, err := utils.Request().SetHeaders(map[string]string{
 		"X-Requested-With": "XMLHttpRequest",
 		"Referer":          constant.OnlineFixURL,
@@ -242,7 +253,7 @@ func (c *OnlineFixCrawler) getCookies() ([]*http.Cookie, error) {
 		return nil, err
 	}
 
-		c.cookies = resp.Cookies()
+	cookies := resp.Cookies()
 
 	params := url.Values{}
 	params.Add("login_name", config.Config.OnlineFix.User)
@@ -253,13 +264,15 @@ func (c *OnlineFixCrawler) getCookies() ([]*http.Cookie, error) {
 		"Origin":       constant.OnlineFixURL,
 		"Content-Type": "application/x-www-form-urlencoded",
 		"Referer":      constant.OnlineFixURL,
-		}).SetCookies(c.cookies).SetBody(params).Post(constant.OnlineFixURL)
+	}).SetCookies(cookies).SetBody(params.Encode()).Post(constant.OnlineFixURL)
 	if err != nil {
 		return nil, err
 	}
-		c.cookies = append(c.cookies, resp.Cookies()...)
+	cookies = resp.Cookies()
-	}
+	jsonBytes, _ := json.Marshal(cookies)
-	return c.cookies, nil
+	_ = cache.SetWithExpire("onlinefix_cookies", string(jsonBytes), time.Hour)
+
+	return cookies, nil
 }
 
 func OnlineFixFormatter(name string) string {

server/middleware/log.go

@@ -1,12 +1,10 @@
 package middleware
 
 import (
-	"strconv"
+	"pcgamedb/log"
 	"strings"
 	"time"
 
-	"pcgamedb/log"
-
 	"github.com/gin-gonic/gin"
 	"go.uber.org/zap"
 )
@@ -14,34 +12,61 @@ import (
 func Logger() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		startTime := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery
 
 		c.Next()
 
-		endTime := time.Now()
+		if shouldSkipLog(path) {
-		latencyTime := endTime.Sub(startTime).Milliseconds()
-		reqMethod := c.Request.Method
-		reqURI := c.Request.RequestURI
-		statusCode := c.Writer.Status()
-		clientIP := c.ClientIP()
-
-		if strings.HasPrefix(reqURI, "/swagger/") ||
-			strings.EqualFold(reqURI, "/favicon.ico") {
 			return
 		}
 
-		log.Logger.Info(
+		if raw != "" {
-			"request",
+			path = path + "?" + raw
-			zap.Int("code", statusCode),
+		}
-			zap.String("method", reqMethod),
+
-			zap.String("uri", reqURI),
+		fields := []zap.Field{
-			zap.String("ip", clientIP),
+			zap.Int("status", c.Writer.Status()),
-			zap.String("latency", strconv.Itoa(int(latencyTime))+"ms"),
+			zap.String("method", c.Request.Method),
-		)
+			zap.String("path", path),
+			zap.String("ip", getRealIP(c)),
+			zap.Duration("latency", time.Since(startTime)),
+		}
 
 		if len(c.Errors) > 0 {
-			for _, e := range c.Errors.Errors() {
+			fields = append(fields, zap.Strings("errors", c.Errors.Errors()))
-				log.Logger.Error(e)
-			}
 		}
+		log.Logger.Info("Request", fields...)
 	}
 }
+
+func getRealIP(c *gin.Context) string {
+	if ip := c.GetHeader("X-Real-IP"); ip != "" {
+		return ip
+	}
+	if ip := c.GetHeader("X-Forwarded-For"); ip != "" {
+		if index := strings.Index(ip, ","); index != -1 {
+			return strings.TrimSpace(ip[:index])
+		}
+		return ip
+	}
+	if ip := c.GetHeader("X-Originating-IP"); ip != "" {
+		return ip
+	}
+	return c.ClientIP()
+}
+
+func shouldSkipLog(path string) bool {
+	skipPaths := []string{
+		"/swagger/",
+		"/favicon.ico",
+		"/health",
+	}
+
+	for _, p := range skipPaths {
+		if strings.HasPrefix(path, p) {
+			return true
+		}
+	}
+	return false
+}

utils/decoder.go

@@ -0,0 +1,16 @@
+package utils
+
+import (
+	"bytes"
+	"io"
+
+	"golang.org/x/text/encoding/htmlindex"
+	"golang.org/x/text/transform"
+)
+
+func Windows1251ToUTF8(b []byte) []byte {
+	decoder, _ := htmlindex.Get("windows-1251")
+	reader := transform.NewReader(bytes.NewReader(b), decoder.NewDecoder().Transformer)
+	body, _ := io.ReadAll(reader)
+	return body
+}

utils/request.go

@@ -14,5 +14,5 @@ func init() {
 }
 
 func Request() *resty.Request {
-	return client.R().SetHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0")
+	return client.R().SetHeader("Accept-Charset", "utf-8").SetHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0")
 }