From 6d212864c554a56996391670c5f59e906ad7097d Mon Sep 17 00:00:00 2001 From: nite07 Date: Thu, 5 Dec 2024 01:36:55 +0800 Subject: [PATCH] remove cache-related judgments and simplify the code --- cache/redis.go | 7 +- config/config.go | 20 +- crawler/freegog.go | 12 +- crawler/game.go | 10 +- crawler/igdb.go | 387 +++++------------------ crawler/onlinefix.go | 4 - crawler/steam.go | 79 ++--- crawler/steam250.go | 158 +-------- db/db.go | 4 - db/game.go | 55 ++-- go.mod | 2 +- go.sum | 4 + server/handler/get_popular_game_infos.go | 2 +- server/handler/healthcheck.go | 48 ++- server/handler/search_games.go | 2 +- server/route.go | 6 +- 16 files changed, 208 insertions(+), 592 deletions(-) diff --git a/cache/redis.go b/cache/redis.go index e5f9eb3..29b303a 100644 --- a/cache/redis.go +++ b/cache/redis.go @@ -16,9 +16,6 @@ var cache *redis.Client var mutx = &sync.RWMutex{} func connect() { - if !config.Config.RedisAvaliable { - return - } cache = redis.NewClient(&redis.Options{ Addr: fmt.Sprintf("%s:%d", config.Config.Redis.Host, config.Config.Redis.Port), Password: config.Config.Redis.Password, @@ -69,14 +66,14 @@ func Get(key string) (string, bool) { return value, true } -func Add(key string, value interface{}) error { +func Set(key string, value interface{}) error { CheckConnect() ctx := context.Background() cmd := cache.Set(ctx, key, value, 0) return cmd.Err() } -func AddWithExpire(key string, value interface{}, expire time.Duration) error { +func SetWithExpire(key string, value interface{}, expire time.Duration) error { CheckConnect() ctx := context.Background() cmd := cache.Set(ctx, key, value, expire) diff --git a/config/config.go b/config/config.go index 833e720..de1687a 100644 --- a/config/config.go +++ b/config/config.go @@ -20,10 +20,7 @@ type config struct { Twitch twitch `json:"twitch"` Webhooks webhooks `json:"webhooks"` CFClearanceScraper cfClearanceScraper `json:"cf_clearance_scraper"` - DatabaseAvaliable bool - OnlineFixAvaliable bool MegaAvaliable bool - RedisAvaliable bool } type cfClearanceScraper struct { @@ -97,11 +94,20 @@ func init() { } } loadEnvVariables(&Config) - Config.OnlineFixAvaliable = Config.OnlineFix.User != "" && Config.OnlineFix.Password != "" - Config.RedisAvaliable = Config.Redis.Host != "" - Config.DatabaseAvaliable = Config.Database.Database != "" && Config.Database.Host != "" - if Config.CFClearanceScraper.Url != "" && !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") { + if Config.OnlineFix.User == "" || Config.OnlineFix.Password == "" { + panic("Need OnlineFix User and Password") + } + if Config.Redis.Host == "" { + panic("Need Redis Host") + } + if Config.Database.Database == "" || Config.Database.Host == "" { + panic("Need Database Name and Host") + } + if Config.CFClearanceScraper.Url == "" { + panic("Need CF Clearance Scraper URL") + } + if !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") { Config.CFClearanceScraper.Url += "/cf-clearance-scraper" } } diff --git a/crawler/freegog.go b/crawler/freegog.go index b7b7b0c..51bde54 100644 --- a/crawler/freegog.go +++ b/crawler/freegog.go @@ -32,7 +32,7 @@ func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler { } } -func (c *FreeGOGCrawler) getWAFSession() (*ccs.Session, error) { +func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) { var session ccs.Session var err error if val, exist := cache.Get("freegog_waf_session"); exist { @@ -45,8 +45,10 @@ func (c *FreeGOGCrawler) getWAFSession() (*ccs.Session, error) { if err != nil { return nil, err } - jsonBytes, _ := json.Marshal(session) - _ = cache.AddWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour) + jsonBytes, err := json.Marshal(session) + if err == nil { + _ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour) + } } return &session, nil } @@ -57,7 +59,7 @@ func (c *FreeGOGCrawler) Name() string { func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) { count := 0 - session, err := c.getWAFSession() + session, err := c.getSession() if err != nil { c.logger.Error("Failed to create session", zap.Error(err)) return nil, err @@ -111,7 +113,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) { } func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { - session, err := c.getWAFSession() + session, err := c.getSession() if err != nil { return nil, err } diff --git a/crawler/game.go b/crawler/game.go index 93a1985..58e5639 100644 --- a/crawler/game.go +++ b/crawler/game.go @@ -38,7 +38,7 @@ func OrganizeGameItem(game *model.GameItem) error { if err == nil { if item.SteamID == 0 { // get steam id from igdb - steamID, err := GetSteamIDByIGDBIDCache(item.IGDBID) + steamID, err := GetSteamIDByIGDBID(item.IGDBID) if err == nil { item.SteamID = steamID } @@ -81,13 +81,13 @@ func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platfo return nil, err } if platform == "igdb" { - steamID, err := GetSteamIDByIGDBIDCache(platformID) + steamID, err := GetSteamIDByIGDBID(platformID) if err == nil { info.SteamID = steamID } } if platform == "steam" { - igdbID, err := GetIGDBIDBySteamAppIDCache(platformID) + igdbID, err := GetIGDBIDBySteamAppID(platformID) if err == nil { info.IGDBID = igdbID } @@ -116,7 +116,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error { for _, info := range infos { changed := false if info.IGDBID != 0 && info.SteamID == 0 { - steamID, err := GetSteamIDByIGDBIDCache(info.IGDBID) + steamID, err := GetSteamIDByIGDBID(info.IGDBID) time.Sleep(time.Millisecond * 100) if err != nil { continue @@ -125,7 +125,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error { changed = true } if info.SteamID != 0 && info.IGDBID == 0 { - igdbID, err := GetIGDBIDBySteamAppIDCache(info.SteamID) + igdbID, err := GetIGDBIDBySteamAppID(info.SteamID) time.Sleep(time.Millisecond * 100) if err != nil { continue diff --git a/crawler/igdb.go b/crawler/igdb.go index 518bd45..37bc629 100644 --- a/crawler/igdb.go +++ b/crawler/igdb.go @@ -6,11 +6,9 @@ import ( "errors" "fmt" "net/url" - "regexp" "runtime/debug" "strconv" "strings" - "sync" "time" "pcgamedb/cache" @@ -25,37 +23,23 @@ import ( ) type twitchToken struct { - Token string `json:"token"` - Expires time.Time `json:"expires"` - once sync.Once } var token = twitchToken{} func (t *twitchToken) getToken() (string, error) { - t.once.Do(func() { - if config.Config.RedisAvaliable { - if dataBytes, exist := cache.Get("twitch_token"); exist { - _ = json.Unmarshal([]byte(dataBytes), &token) - } - } - }) - if t.Token == "" || time.Now().After(t.Expires) { - token, expires, err := loginTwitch() - if err != nil { - return "", fmt.Errorf("failed to login twitch: %w", err) - } - t.Token = token - t.Expires = expires - j, err := json.Marshal(t) - if err == nil { - _ = cache.Add("twitch_token", j) - } + if val, exist := cache.Get("twitch_token"); exist { + return val, nil } - return t.Token, nil + token, expires, err := loginTwitch() + if err != nil { + return "", fmt.Errorf("failed to login twitch: %w", err) + } + _ = cache.SetWithExpire("twitch_token", token, expires) + return token, nil } -func loginTwitch() (string, time.Time, error) { +func loginTwitch() (string, time.Duration, error) { baseURL, _ := url.Parse(constant.TwitchAuthURL) params := url.Values{} params.Add("client_id", config.Config.Twitch.ClientID) @@ -64,7 +48,7 @@ func loginTwitch() (string, time.Time, error) { baseURL.RawQuery = params.Encode() resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String()) if err != nil { - return "", time.Time{}, err + return "", 0, err } data := struct { AccessToken string `json:"access_token"` @@ -73,9 +57,9 @@ func loginTwitch() (string, time.Time, error) { }{} err = json.Unmarshal(resp.Body(), &data) if err != nil { - return "", time.Time{}, err + return "", 0, err } - return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil + return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil } func igdbRequest(URL string, dataBody any) (*resty.Response, error) { @@ -113,7 +97,7 @@ func getIGDBID(name string) (int, error) { return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) } if len(data) == 1 { - return GetIGDBAppParentCache(data[0].Game) + return GetIGDBAppParent(data[0].Game) } maxSimilairty := 0.0 maxSimilairtyIndex := 0 @@ -127,7 +111,7 @@ func getIGDBID(name string) (int, error) { maxSimilairtyIndex = i } } - detail, err := GetIGDBAppDetailCache(item.Game) + detail, err := GetIGDBAppDetail(item.Game) if err != nil { return 0, err } @@ -141,7 +125,7 @@ func getIGDBID(name string) (int, error) { } } if maxSimilairty >= 0.8 { - return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game) + return GetIGDBAppParent(data[maxSimilairtyIndex].Game) } return 0, fmt.Errorf("IGDB ID not found: %s", name) } @@ -212,10 +196,10 @@ func getIGDBIDBySteamSearch(name string) (int, error) { } if maxSim != 0 { if maxSimItem.Type == "App" { - return GetIGDBIDBySteamAppIDCache(maxSimItem.ID) + return GetIGDBIDBySteamAppID(maxSimItem.ID) } if maxSimItem.Type == "Bundle" { - return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID) + return GetIGDBIDBySteamBundleID(maxSimItem.ID) } } return 0, fmt.Errorf("steam ID not found: %s", name) @@ -223,14 +207,23 @@ func getIGDBIDBySteamSearch(name string) (int, error) { // GetIGDBAppParent returns the parent of the game, if no parent return itself func GetIGDBAppParent(id int) (int, error) { - detail, err := GetIGDBAppDetailCache(id) + key := fmt.Sprintf("igdb_parent:%d", id) + val, exist := cache.Get(key) + if exist { + id, err := strconv.Atoi(val) + if err != nil { + return 0, err + } + return id, nil + } + detail, err := GetIGDBAppDetail(id) if err != nil { return 0, err } hasParent := false for detail.VersionParent != 0 { hasParent = true - detail, err = GetIGDBAppDetailCache(detail.VersionParent) + detail, err = GetIGDBAppDetail(detail.VersionParent) if err != nil { return 0, err } @@ -238,67 +231,19 @@ func GetIGDBAppParent(id int) (int, error) { if hasParent { return detail.ID, nil } + + _ = cache.Set(key, id) + return id, nil } -func GetIGDBAppParetns(ids []int) (map[int]int, error) { - var err error - idsStr := make([]string, len(ids)) - for i, id := range ids { - idsStr[i] = strconv.Itoa(id) - } - resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ","))) - - if err != nil { - return nil, err - } - var data []struct { - ID int `json:"id"` - VersionParent int `json:"version_parent"` - } - if err = json.Unmarshal(resp.Body(), &data); err != nil { - return nil, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) - } - parents := make(map[int]int) - for _, item := range data { - if item.VersionParent != 0 { - pid, err := GetIGDBAppParentCache(item.VersionParent) - if err != nil { - parents[item.ID] = item.ID - } else { - parents[item.ID] = pid - } - } else { - parents[item.ID] = item.ID - } - } - return parents, nil -} - -func GetIGDBAppParentCache(id int) (int, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("igdb_parent:%d", id) - val, exist := cache.Get(key) - if exist { - id, err := strconv.Atoi(val) - if err != nil { - return 0, err - } - return id, nil - } else { - id, err := GetIGDBAppParent(id) - if err != nil { - return 0, err - } - _ = cache.Add(key, id) - return id, nil - } - } - return GetIGDBAppParent(id) -} - // GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search func GetIGDBID(name string) (int, error) { + key := fmt.Sprintf("igdb_id:%s", name) + val, exist := cache.Get(key) + if exist { + return strconv.Atoi(val) + } name1 := name name2 := FormatName(name) names := []string{name1} @@ -308,42 +253,30 @@ func GetIGDBID(name string) (int, error) { for _, name := range names { id, err := getIGDBID(name) if err == nil { + _ = cache.Set(key, id) return id, nil } } for _, name := range names { id, err := getIGDBIDBySteamSearch(name) if err == nil { + _ = cache.Set(key, id) return id, nil } } return 0, errors.New("IGDB ID not found") } -func GetIGDBIDCache(name string) (int, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("igdb_id:%s", name) - val, exist := cache.Get(key) - if exist { - id, err := strconv.Atoi(val) - if err != nil { - return 0, err - } - return id, nil - } else { - id, err := GetIGDBID(name) - if err != nil { - return 0, err - } - _ = cache.Add(key, id) - return id, nil - } - } else { - return GetIGDBID(name) - } -} - func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { + key := fmt.Sprintf("igdb_game:%v", id) + val, exist := cache.Get(key) + if exist { + var data model.IGDBGameDetail + if err := json.Unmarshal([]byte(val), &data); err != nil { + return nil, err + } + return &data, nil + } var err error resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id)) @@ -360,37 +293,21 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { if data[0].Name == "" { return GetIGDBAppDetail(id) } + + jsonBytes, err := json.Marshal(data[0]) + if err == nil { + _ = cache.Set(key, string(jsonBytes)) + } + return data[0], nil } -func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("igdb_game:%v", id) - val, exist := cache.Get(key) - if exist { - var data model.IGDBGameDetail - if err := json.Unmarshal([]byte(val), &data); err != nil { - return nil, err - } - return &data, nil - } else { - data, err := GetIGDBAppDetail(id) - if err != nil { - return nil, err - } - dataBytes, err := json.Marshal(data) - if err != nil { - return nil, err - } - _ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour) - return data, nil - } - } else { - return GetIGDBAppDetail(id) - } -} - func GetIGDBCompany(id int) (string, error) { + key := fmt.Sprintf("igdb_companies:%v", id) + val, exist := cache.Get(key) + if exist { + return val, nil + } var err error resp, err := igdbRequest(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id)) if err != nil { @@ -406,31 +323,15 @@ func GetIGDBCompany(id int) (string, error) { if data[0].Name == "" { return GetIGDBCompany(id) } - return data[0].Name, nil -} -func GetIGDBCompanyCache(id int) (string, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("igdb_companies:%v", id) - val, exist := cache.Get(key) - if exist { - return val, nil - } else { - data, err := GetIGDBCompany(id) - if err != nil { - return "", err - } - _ = cache.Add(key, data) - return data, nil - } - } else { - return GetIGDBCompany(id) - } + _ = cache.Set(key, data[0].Name) + + return data[0].Name, nil } func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { item := &model.GameInfo{} - detail, err := GetIGDBAppDetailCache(id) + detail, err := GetIGDBAppDetail(id) if err != nil { return nil, err } @@ -460,7 +361,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { for _, company := range detail.InvolvedCompanies { if company.Developer || company.Publisher { - companyName, err := GetIGDBCompanyCache(company.Company) + companyName, err := GetIGDBCompany(company.Company) if err != nil { continue } @@ -478,7 +379,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { // OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) { - id, err := GetIGDBIDCache(game.Name) + id, err := GetIGDBID(game.Name) if err != nil { return nil, err } @@ -498,6 +399,11 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) { } func GetIGDBIDBySteamAppID(id int) (int, error) { + key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id) + val, exist := cache.Get(key) + if exist { + return strconv.Atoi(val) + } var err error resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id)) if err != nil { @@ -515,29 +421,19 @@ func GetIGDBIDBySteamAppID(id int) (int, error) { if data[0].Game == 0 { return GetIGDBIDBySteamAppID(id) } - return GetIGDBAppParentCache(data[0].Game) -} -func GetIGDBIDBySteamAppIDCache(id int) (int, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id) - val, exist := cache.Get(key) - if exist { - return strconv.Atoi(val) - } else { - data, err := GetIGDBIDBySteamAppID(id) - if err != nil { - return 0, err - } - _ = cache.Add(key, strconv.Itoa(data)) - return data, nil - } - } else { - return GetIGDBIDBySteamAppID(id) - } + _ = cache.Set(key, strconv.Itoa(data[0].Game)) + + return GetIGDBAppParent(data[0].Game) } func GetIGDBIDBySteamBundleID(id int) (int, error) { + key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id) + val, exist := cache.Get(key) + if exist { + return strconv.Atoi(val) + } + var err error resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id)) if err != nil { @@ -555,104 +451,10 @@ func GetIGDBIDBySteamBundleID(id int) (int, error) { if data[0].Game == 0 { return GetIGDBIDBySteamBundleID(id) } - return GetIGDBAppParentCache(data[0].Game) -} -func GetIGDBIDBySteamBundleIDCache(id int) (int, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id) - val, exist := cache.Get(key) - if exist { - return strconv.Atoi(val) - } else { - data, err := GetIGDBIDBySteamBundleID(id) - if err != nil { - return 0, err - } - _ = cache.Add(key, strconv.Itoa(data)) - return data, nil - } - } else { - return GetIGDBIDBySteamBundleID(id) - } -} + _ = cache.Set(key, strconv.Itoa(data[0].Game)) -func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) { - var err error - conditionBuilder := strings.Builder{} - for _, id := range ids { - conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id)) - conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id)) - } - condition := strings.TrimSuffix(conditionBuilder.String(), " | ") - respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition) - resp, err := igdbRequest(constant.IGDBWebsitesURL, respBody) - if err != nil { - return nil, err - } - var data []struct { - Game int `json:"game"` - Url string `json:"url"` - } - if err = json.Unmarshal(resp.Body(), &data); err != nil { - return nil, err - } - ret := make(map[int]int) - regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`) - for _, d := range data { - idStr := regex.FindStringSubmatch(d.Url) - if len(idStr) < 2 { - continue - } - id, err := strconv.Atoi(idStr[1]) - if err == nil { - pid, err := GetIGDBAppParentCache(d.Game) - if err == nil { - ret[id] = pid - } else { - ret[id] = 0 - } - } - } - for _, id := range ids { - if _, ok := ret[id]; !ok { - ret[id] = 0 - } - } - return ret, nil -} - -func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) { - res := make(map[int]int) - notExistIDs := make([]int, 0) - if config.Config.RedisAvaliable { - for _, steamID := range ids { - key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID) - val, exist := cache.Get(key) - if exist { - igdbID, _ := strconv.Atoi(val) - res[steamID] = igdbID - } else { - notExistIDs = append(notExistIDs, steamID) - } - } - if len(res) == len(ids) { - return res, nil - } - idMap, err := GetIGDBIDsBySteamIDs(notExistIDs) - if err != nil { - return nil, err - } - for steamID, igdbID := range idMap { - res[steamID] = igdbID - if igdbID != 0 { - _ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID) - } - } - return res, nil - } else { - return GetIGDBIDsBySteamIDs(ids) - } + return GetIGDBAppParent(data[0].Game) } // GetIGDBPopularGameIDs get IGDB popular game IDs @@ -676,7 +478,7 @@ func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, er } ret := make([]int, 0) for _, d := range data { - pid, err := GetIGDBAppParentCache(d.GameID) + pid, err := GetIGDBAppParent(d.GameID) if err != nil { ret = append(ret, d.GameID) continue @@ -685,30 +487,3 @@ func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, er } return ret, nil } - -func GetIGDBPopularGameIDsCache(popularityType int, offset int, limit int) ([]int, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("igdb_popular_game_ids:%v:%v:%v", popularityType, offset, limit) - val, exist := cache.Get(key) - if exist { - var data []int - if err := json.Unmarshal([]byte(val), &data); err != nil { - return nil, err - } - return data, nil - } else { - data, err := GetIGDBPopularGameIDs(popularityType, offset, limit) - if err != nil { - return nil, err - } - jsonBytes, err := json.Marshal(data) - if err != nil { - return nil, err - } - _ = cache.AddWithExpire(key, jsonBytes, 12*time.Hour) - return data, nil - } - } else { - return GetIGDBPopularGameIDs(popularityType, offset, limit) - } -} diff --git a/crawler/onlinefix.go b/crawler/onlinefix.go index e85b771..dd902bd 100644 --- a/crawler/onlinefix.go +++ b/crawler/onlinefix.go @@ -39,10 +39,6 @@ func (c *OnlineFixCrawler) Name() string { } func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) { - if !config.Config.OnlineFixAvaliable { - c.logger.Error("Need Online Fix account") - return nil, errors.New("online Fix is not available") - } cookies, err := c.getCookies() if err != nil { return nil, err diff --git a/crawler/steam.go b/crawler/steam.go index 389b7dc..32e9330 100644 --- a/crawler/steam.go +++ b/crawler/steam.go @@ -11,13 +11,22 @@ import ( "time" "pcgamedb/cache" - "pcgamedb/config" "pcgamedb/constant" "pcgamedb/model" "pcgamedb/utils" ) func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) { + key := fmt.Sprintf("steam_game:%d", id) + val, exist := cache.Get(key) + if exist { + var detail model.SteamAppDetail + if err := json.Unmarshal([]byte(val), &detail); err != nil { + return nil, err + } + return &detail, nil + } + baseURL, _ := url.Parse(constant.SteamAppDetailURL) params := url.Values{} params.Add("appids", strconv.Itoa(id)) @@ -39,39 +48,18 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) { if detail[strconv.Itoa(id)] == nil { return nil, fmt.Errorf("steam App not found: %d", id) } - return detail[strconv.Itoa(id)], nil -} -func GetSteamAppDetailCache(id int) (*model.SteamAppDetail, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("steam_game:%d", id) - val, exist := cache.Get(key) - if exist { - var detail model.SteamAppDetail - if err := json.Unmarshal([]byte(val), &detail); err != nil { - return nil, err - } - return &detail, nil - } else { - data, err := GetSteamAppDetail(id) - if err != nil { - return nil, err - } - dataBytes, err := json.Marshal(data) - if err != nil { - return nil, err - } - _ = cache.Add(key, dataBytes) - return data, nil - } - } else { - return GetSteamAppDetail(id) + jsonBytes, err := json.Marshal(detail[strconv.Itoa(id)]) + if err == nil { + _ = cache.Set(key, string(jsonBytes)) } + + return detail[strconv.Itoa(id)], nil } func GenerateSteamGameInfo(id int) (*model.GameInfo, error) { item := &model.GameInfo{} - detail, err := GetSteamAppDetailCache(id) + detail, err := GetSteamAppDetail(id) if err != nil { return nil, err } @@ -91,6 +79,16 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) { } func GetSteamIDByIGDBID(IGDBID int) (int, error) { + key := fmt.Sprintf("steam_game:%d", IGDBID) + val, exist := cache.Get(key) + if exist { + id, err := strconv.Atoi(val) + if err != nil { + return 0, err + } + return id, nil + } + var err error resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID)) if err != nil { @@ -117,32 +115,9 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) { if err != nil { return 0, err } + _ = cache.Set(key, strconv.Itoa(steamID)) return steamID, nil } } return 0, errors.New("not found") } - -func GetSteamIDByIGDBIDCache(IGDBID int) (int, error) { - if config.Config.RedisAvaliable { - key := fmt.Sprintf("steam_game:%d", IGDBID) - val, exist := cache.Get(key) - if exist { - id, err := strconv.Atoi(val) - if err != nil { - return 0, err - } - return id, nil - } else { - id, err := GetSteamIDByIGDBID(IGDBID) - if err != nil { - return 0, err - } - dataBytes := strconv.Itoa(id) - _ = cache.Add(key, dataBytes) - return id, nil - } - } else { - return GetSteamIDByIGDBID(IGDBID) - } -} diff --git a/crawler/steam250.go b/crawler/steam250.go index 73486fb..18134f1 100644 --- a/crawler/steam250.go +++ b/crawler/steam250.go @@ -4,8 +4,8 @@ import ( "bytes" "encoding/json" "fmt" + "net/url" "pcgamedb/cache" - "pcgamedb/config" "pcgamedb/db" "regexp" "strconv" @@ -19,6 +19,16 @@ import ( ) func GetSteam250(URL string) ([]*model.GameInfo, error) { + key := "steam250:" + url.QueryEscape(URL) + if val, ok := cache.Get(key); ok { + var infos []*model.GameInfo + err := json.Unmarshal([]byte(val), &infos) + if err != nil { + return nil, err + } + return infos, nil + } + resp, err := utils.Request().Get(URL) if err != nil { return nil, err @@ -48,6 +58,12 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) { if len(infos) > 10 { return infos[:10], nil } + + jsonBytes, err := json.Marshal(infos) + if err == nil { + _ = cache.SetWithExpire(key, string(jsonBytes), 12*time.Hour) + } + return infos, nil } @@ -55,158 +71,18 @@ func GetSteam250Top250() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250Top250URL) } -func GetSteam250Top250Cache() ([]*model.GameInfo, error) { - if config.Config.RedisAvaliable { - key := "steam250_top250" - val, exist := cache.Get(key) - if exist { - var infos []*model.GameInfo - err := json.Unmarshal([]byte(val), &infos) - if err != nil { - return nil, err - } - return infos, nil - } else { - infos, err := GetSteam250Top250() - if err != nil { - return nil, err - } - jsonBytes, err := json.Marshal(infos) - if err != nil { - return nil, err - } - _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) - return infos, nil - } - } else { - return GetSteam250Top250() - } -} - func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) { return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, time.Now().UTC().Year())) } -func GetSteam250BestOfTheYearCache() ([]*model.GameInfo, error) { - if config.Config.RedisAvaliable { - key := "steam250_best_of_the_year" - val, exist := cache.Get(key) - if exist { - var infos []*model.GameInfo - err := json.Unmarshal([]byte(val), &infos) - if err != nil { - return nil, err - } - return infos, nil - } else { - infos, err := GetSteam250BestOfTheYear() - if err != nil { - return nil, err - } - jsonBytes, err := json.Marshal(infos) - if err != nil { - return nil, err - } - _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) - return infos, nil - } - } else { - return GetSteam250BestOfTheYear() - } -} - func GetSteam250WeekTop50() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250WeekTop50URL) } -func GetSteam250WeekTop50Cache() ([]*model.GameInfo, error) { - if config.Config.RedisAvaliable { - key := "steam250_week_top50" - val, exist := cache.Get(key) - if exist { - var infos []*model.GameInfo - err := json.Unmarshal([]byte(val), &infos) - if err != nil { - return nil, err - } - return infos, nil - } else { - infos, err := GetSteam250WeekTop50() - if err != nil { - return nil, err - } - jsonBytes, err := json.Marshal(infos) - if err != nil { - return nil, err - } - _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) - return infos, nil - } - } else { - return GetSteam250WeekTop50() - } -} - func GetSteam250MonthTop50() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250MonthTop50URL) } -func GetSteam250MonthTop50Cache() ([]*model.GameInfo, error) { - if config.Config.RedisAvaliable { - key := "steam250_month_top50" - val, exist := cache.Get(key) - if exist { - var infos []*model.GameInfo - err := json.Unmarshal([]byte(val), &infos) - if err != nil { - return nil, err - } - return infos, nil - } else { - infos, err := GetSteam250MonthTop50() - if err != nil { - return nil, err - } - jsonBytes, err := json.Marshal(infos) - if err != nil { - return nil, err - } - _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) - return infos, nil - } - } else { - return GetSteam250MonthTop50() - } -} - func GetSteam250MostPlayed() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250MostPlayedURL) } - -func GetSteam250MostPlayedCache() ([]*model.GameInfo, error) { - if config.Config.RedisAvaliable { - key := "steam250_most_played" - val, exist := cache.Get(key) - if exist { - var infos []*model.GameInfo - err := json.Unmarshal([]byte(val), &infos) - if err != nil { - return nil, err - } - return infos, nil - } else { - infos, err := GetSteam250MostPlayed() - if err != nil { - return nil, err - } - jsonBytes, err := json.Marshal(infos) - if err != nil { - return nil, err - } - _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) - return infos, nil - } - } else { - return GetSteam250MostPlayed() - } -} diff --git a/db/db.go b/db/db.go index 617abd8..47896fe 100644 --- a/db/db.go +++ b/db/db.go @@ -32,10 +32,6 @@ var ( ) func connect() { - if !config.Config.DatabaseAvaliable { - log.Logger.Panic("Missing database configuration information") - return - } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() clientOptions := options.Client().ApplyURI(fmt.Sprintf( diff --git a/db/game.go b/db/game.go index 77e96ef..b651bec 100644 --- a/db/game.go +++ b/db/game.go @@ -11,7 +11,6 @@ import ( "time" "pcgamedb/cache" - "pcgamedb/config" "pcgamedb/model" "go.mongodb.org/mongo-driver/bson" @@ -299,6 +298,21 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in name = strings.TrimSpace(name) name = strings.Replace(name, " ", ".*", -1) name = fmt.Sprintf("%s.*", name) + + key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize) + val, exist := cache.Get(key) + if exist { + var data struct { + Items []*model.GameInfo + TotalPage int + } + err := json.Unmarshal([]byte(val), &data) + if err != nil { + return nil, 0, err + } + return data.Items, data.TotalPage, nil + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -334,40 +348,19 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in if err := cursor.Err(); err != nil { return nil, 0, err } - return items, int(totalPage), nil -} -func SearchGameInfosCache(name string, page int, pageSize int) ([]*model.GameInfo, int, error) { - type res struct { + jsonBytes, err := json.Marshal(struct { Items []*model.GameInfo TotalPage int + }{ + Items: items, + TotalPage: int(totalPage), + }) + if err == nil { + _ = cache.SetWithExpire(key, string(jsonBytes), time.Minute*5) } - name = strings.ToLower(name) - if config.Config.RedisAvaliable { - key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize) - val, exist := cache.Get(key) - if exist { - var data res - err := json.Unmarshal([]byte(val), &data) - if err != nil { - return nil, 0, err - } - return data.Items, data.TotalPage, nil - } else { - data, totalPage, err := SearchGameInfos(name, page, pageSize) - if err != nil { - return nil, 0, err - } - dataBytes, err := json.Marshal(res{Items: data, TotalPage: totalPage}) - if err != nil { - return nil, 0, err - } - _ = cache.AddWithExpire(key, string(dataBytes), 5*time.Minute) - return data, totalPage, nil - } - } else { - return SearchGameInfos(name, page, pageSize) - } + + return items, int(totalPage), nil } func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) { diff --git a/go.mod b/go.mod index 30bba1d..4a247ad 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( ) require ( - git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580 // indirect + git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760 // indirect github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 // indirect github.com/Danny-Dasilva/fhttp v0.0.0-20240217042913-eeeb0b347ce1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect diff --git a/go.sum b/go.sum index d1ec983..b119a8c 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,10 @@ git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff h1:f5OEMRc/zhMxSfHdTJ git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w= git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580 h1:B2ewPM44DgyrkycIrUfyTRLM7mXggA0JE8pNbvxjFKw= git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w= +git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c h1:UcIxgKmcQGZqjTJWsQf9MVDviUQFlU+ZK6HJjpW+nAU= +git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w= +git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760 h1:ZCMgQt2ILohQ3MSk6RVhnRY4fbQTJVLREbWZjLtVs+Y= +git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 h1:6fexoGmvzoXMSk14BZ0AirapVm5c3KUsEjE0jLlVKi8= github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26/go.mod h1:QFi/EVO7qqru3Ftxz1LR+96jIc91Tifv0DnskF/gWQ8= diff --git a/server/handler/get_popular_game_infos.go b/server/handler/get_popular_game_infos.go index 995e50f..1deeaaf 100644 --- a/server/handler/get_popular_game_infos.go +++ b/server/handler/get_popular_game_infos.go @@ -87,7 +87,7 @@ func GetPopularGameInfosHandler(c *gin.Context) { offset += 20 pids := make([]int, 20) for _, id := range ids { - pid, err := crawler.GetIGDBAppParentCache(id) + pid, err := crawler.GetIGDBAppParent(id) if err != nil { continue } diff --git a/server/handler/healthcheck.go b/server/handler/healthcheck.go index ef4d0eb..e62043b 100644 --- a/server/handler/healthcheck.go +++ b/server/handler/healthcheck.go @@ -14,19 +14,17 @@ import ( ) type HealthCheckResponse struct { - Version string `json:"version"` - Status string `json:"status"` - Date string `json:"date"` - Uptime string `json:"uptime"` - Alloc string `json:"alloc"` - AutoCrawl bool `json:"auto_crawl"` - AutoCrawlCron string `json:"auto_crawl_cron"` - GameItem int64 `json:"game_num"` - GameInfo int64 `json:"game_info_num"` - Unorganized int64 `json:"unorganized_game_num"` - RedisAvaliable bool `json:"redis_avaliable"` - OnlineFixAvaliable bool `json:"online_fix_avaliable"` - MegaAvaliable bool `json:"mega_avaliable"` + Version string `json:"version"` + Status string `json:"status"` + Date string `json:"date"` + Uptime string `json:"uptime"` + Alloc string `json:"alloc"` + AutoCrawl bool `json:"auto_crawl"` + AutoCrawlCron string `json:"auto_crawl_cron"` + GameItem int64 `json:"game_num"` + GameInfo int64 `json:"game_info_num"` + Unorganized int64 `json:"unorganized_game_num"` + MegaAvaliable bool `json:"mega_avaliable"` } // HealthCheckHandler performs a health check of the service. @@ -49,18 +47,16 @@ func HealthCheckHandler(c *gin.Context) { unorganizedCount = int64(len(unorganized)) } c.JSON(http.StatusOK, HealthCheckResponse{ - Status: "ok", - Version: constant.Version, - Date: time.Now().Format("2006-01-02 15:04:05"), - Uptime: time.Since(config.Runtime.ServerStartTime).String(), - AutoCrawl: config.Config.Server.AutoCrawl, - AutoCrawlCron: config.Config.Server.AutoCrawlCron, - Alloc: fmt.Sprintf("%.2f MB", float64(m.Alloc)/1024.0/1024.0), - GameItem: downloadCount, - GameInfo: infoCount, - Unorganized: unorganizedCount, - RedisAvaliable: config.Config.RedisAvaliable, - OnlineFixAvaliable: config.Config.OnlineFixAvaliable, - MegaAvaliable: config.Config.MegaAvaliable, + Status: "ok", + Version: constant.Version, + Date: time.Now().Format("2006-01-02 15:04:05"), + Uptime: time.Since(config.Runtime.ServerStartTime).String(), + AutoCrawl: config.Config.Server.AutoCrawl, + AutoCrawlCron: config.Config.Server.AutoCrawlCron, + Alloc: fmt.Sprintf("%.2f MB", float64(m.Alloc)/1024.0/1024.0), + GameItem: downloadCount, + GameInfo: infoCount, + Unorganized: unorganizedCount, + MegaAvaliable: config.Config.MegaAvaliable, }) } diff --git a/server/handler/search_games.go b/server/handler/search_games.go index 8825c22..63dad64 100644 --- a/server/handler/search_games.go +++ b/server/handler/search_games.go @@ -53,7 +53,7 @@ func SearchGamesHandler(c *gin.Context) { if req.PageSize > 10 { req.PageSize = 10 } - items, totalPage, err := db.SearchGameInfosCache(req.Keyword, req.Page, req.PageSize) + items, totalPage, err := db.SearchGameInfos(req.Keyword, req.Page, req.PageSize) if err != nil { c.JSON(http.StatusInternalServerError, SearchGamesResponse{ Status: "error", diff --git a/server/route.go b/server/route.go index 3c1dadf..f15105c 100644 --- a/server/route.go +++ b/server/route.go @@ -83,17 +83,17 @@ func initFrontend(app *gin.Engine) { // Load routes app.GET("/", func(ctx *gin.Context) { - monthTop, err := crawler.GetSteam250MonthTop50Cache() + monthTop, err := crawler.GetSteam250MonthTop50() if err != nil { ctx.HTML(500, "500.html", err) return } - mostPlayed, err := crawler.GetSteam250MostPlayedCache() + mostPlayed, err := crawler.GetSteam250MostPlayed() if err != nil { ctx.HTML(500, "500.html", err) return } - bestOfTheYear, err := crawler.GetSteam250BestOfTheYearCache() + bestOfTheYear, err := crawler.GetSteam250BestOfTheYear() if err != nil { ctx.HTML(500, "500.html", err) return