nite/game-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

remove cache-related judgments and simplify the code

Browse Source
This commit is contained in:
Nite07 2024-12-05 01:36:55 +08:00
parent 8ea32e61ba
commit 6d212864c5
16 changed files with 208 additions and 592 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cache/redis.go vendored
View File

@ -16,9 +16,6 @@ var cache *redis.Client
var mutx = &sync.RWMutex{}
func connect() {
if !config.Config.RedisAvaliable {
return
}
cache = redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%d", config.Config.Redis.Host, config.Config.Redis.Port),
Password: config.Config.Redis.Password,
@ -69,14 +66,14 @@ func Get(key string) (string, bool) {
return value, true
}
func Add(key string, value interface{}) error {
func Set(key string, value interface{}) error {
CheckConnect()
ctx := context.Background()
cmd := cache.Set(ctx, key, value, 0)
return cmd.Err()
}
func AddWithExpire(key string, value interface{}, expire time.Duration) error {
func SetWithExpire(key string, value interface{}, expire time.Duration) error {
CheckConnect()
ctx := context.Background()
cmd := cache.Set(ctx, key, value, expire)

20
config/config.go
View File

@ -20,10 +20,7 @@ type config struct {
Twitch twitch `json:"twitch"`
Webhooks webhooks `json:"webhooks"`
CFClearanceScraper cfClearanceScraper `json:"cf_clearance_scraper"`
DatabaseAvaliable bool
OnlineFixAvaliable bool
MegaAvaliable bool
RedisAvaliable bool
}
type cfClearanceScraper struct {
@ -97,11 +94,20 @@ func init() {
}
}
loadEnvVariables(&Config)
Config.OnlineFixAvaliable = Config.OnlineFix.User != "" && Config.OnlineFix.Password != ""
Config.RedisAvaliable = Config.Redis.Host != ""
Config.DatabaseAvaliable = Config.Database.Database != "" && Config.Database.Host != ""
if Config.CFClearanceScraper.Url != "" && !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") {
if Config.OnlineFix.User == "" || Config.OnlineFix.Password == "" {
panic("Need OnlineFix User and Password")
}
if Config.Redis.Host == "" {
panic("Need Redis Host")
}
if Config.Database.Database == "" || Config.Database.Host == "" {
panic("Need Database Name and Host")
}
if Config.CFClearanceScraper.Url == "" {
panic("Need CF Clearance Scraper URL")
}
if !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") {
Config.CFClearanceScraper.Url += "/cf-clearance-scraper"
}
}

12
crawler/freegog.go
View File

@ -32,7 +32,7 @@ func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
}
}
func (c *FreeGOGCrawler) getWAFSession() (*ccs.Session, error) {
func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) {
var session ccs.Session
var err error
if val, exist := cache.Get("freegog_waf_session"); exist {
@ -45,8 +45,10 @@ func (c *FreeGOGCrawler) getWAFSession() (*ccs.Session, error) {
if err != nil {
return nil, err
}
jsonBytes, _ := json.Marshal(session)
_ = cache.AddWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour)
jsonBytes, err := json.Marshal(session)
if err == nil {
_ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour)
}
}
return &session, nil
}
@ -57,7 +59,7 @@ func (c *FreeGOGCrawler) Name() string {
func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
count := 0
session, err := c.getWAFSession()
session, err := c.getSession()
if err != nil {
c.logger.Error("Failed to create session", zap.Error(err))
return nil, err
@ -111,7 +113,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
}
func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
session, err := c.getWAFSession()
session, err := c.getSession()
if err != nil {
return nil, err
}

10
crawler/game.go
View File

@ -38,7 +38,7 @@ func OrganizeGameItem(game *model.GameItem) error {
if err == nil {
if item.SteamID == 0 {
// get steam id from igdb
steamID, err := GetSteamIDByIGDBIDCache(item.IGDBID)
steamID, err := GetSteamIDByIGDBID(item.IGDBID)
if err == nil {
item.SteamID = steamID
}
@ -81,13 +81,13 @@ func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platfo
return nil, err
}
if platform == "igdb" {
steamID, err := GetSteamIDByIGDBIDCache(platformID)
steamID, err := GetSteamIDByIGDBID(platformID)
if err == nil {
info.SteamID = steamID
}
}
if platform == "steam" {
igdbID, err := GetIGDBIDBySteamAppIDCache(platformID)
igdbID, err := GetIGDBIDBySteamAppID(platformID)
if err == nil {
info.IGDBID = igdbID
}
@ -116,7 +116,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
for _, info := range infos {
changed := false
if info.IGDBID != 0 && info.SteamID == 0 {
steamID, err := GetSteamIDByIGDBIDCache(info.IGDBID)
steamID, err := GetSteamIDByIGDBID(info.IGDBID)
time.Sleep(time.Millisecond * 100)
if err != nil {
continue
@ -125,7 +125,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
changed = true
}
if info.SteamID != 0 && info.IGDBID == 0 {
igdbID, err := GetIGDBIDBySteamAppIDCache(info.SteamID)
igdbID, err := GetIGDBIDBySteamAppID(info.SteamID)
time.Sleep(time.Millisecond * 100)
if err != nil {
continue

387
crawler/igdb.go
View File

@ -6,11 +6,9 @@ import (
"errors"
"fmt"
"net/url"
"regexp"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
"pcgamedb/cache"
@ -25,37 +23,23 @@ import (
)
type twitchToken struct {
Token string `json:"token"`
Expires time.Time `json:"expires"`
once sync.Once
}
var token = twitchToken{}
func (t *twitchToken) getToken() (string, error) {
t.once.Do(func() {
if config.Config.RedisAvaliable {
if dataBytes, exist := cache.Get("twitch_token"); exist {
_ = json.Unmarshal([]byte(dataBytes), &token)
}
}
})
if t.Token == "" || time.Now().After(t.Expires) {
token, expires, err := loginTwitch()
if err != nil {
return "", fmt.Errorf("failed to login twitch: %w", err)
}
t.Token = token
t.Expires = expires
j, err := json.Marshal(t)
if err == nil {
_ = cache.Add("twitch_token", j)
}
if val, exist := cache.Get("twitch_token"); exist {
return val, nil
}
return t.Token, nil
token, expires, err := loginTwitch()
if err != nil {
return "", fmt.Errorf("failed to login twitch: %w", err)
}
_ = cache.SetWithExpire("twitch_token", token, expires)
return token, nil
}
func loginTwitch() (string, time.Time, error) {
func loginTwitch() (string, time.Duration, error) {
baseURL, _ := url.Parse(constant.TwitchAuthURL)
params := url.Values{}
params.Add("client_id", config.Config.Twitch.ClientID)
@ -64,7 +48,7 @@ func loginTwitch() (string, time.Time, error) {
baseURL.RawQuery = params.Encode()
resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String())
if err != nil {
return "", time.Time{}, err
return "", 0, err
}
data := struct {
AccessToken string `json:"access_token"`
@ -73,9 +57,9 @@ func loginTwitch() (string, time.Time, error) {
}{}
err = json.Unmarshal(resp.Body(), &data)
if err != nil {
return "", time.Time{}, err
return "", 0, err
}
return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil
return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil
}
func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
@ -113,7 +97,7 @@ func getIGDBID(name string) (int, error) {
return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
}
if len(data) == 1 {
return GetIGDBAppParentCache(data[0].Game)
return GetIGDBAppParent(data[0].Game)
}
maxSimilairty := 0.0
maxSimilairtyIndex := 0
@ -127,7 +111,7 @@ func getIGDBID(name string) (int, error) {
maxSimilairtyIndex = i
}
}
detail, err := GetIGDBAppDetailCache(item.Game)
detail, err := GetIGDBAppDetail(item.Game)
if err != nil {
return 0, err
}
@ -141,7 +125,7 @@ func getIGDBID(name string) (int, error) {
}
}
if maxSimilairty >= 0.8 {
return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game)
return GetIGDBAppParent(data[maxSimilairtyIndex].Game)
}
return 0, fmt.Errorf("IGDB ID not found: %s", name)
}
@ -212,10 +196,10 @@ func getIGDBIDBySteamSearch(name string) (int, error) {
}
if maxSim != 0 {
if maxSimItem.Type == "App" {
return GetIGDBIDBySteamAppIDCache(maxSimItem.ID)
return GetIGDBIDBySteamAppID(maxSimItem.ID)
}
if maxSimItem.Type == "Bundle" {
return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID)
return GetIGDBIDBySteamBundleID(maxSimItem.ID)
}
}
return 0, fmt.Errorf("steam ID not found: %s", name)
@ -223,14 +207,23 @@ func getIGDBIDBySteamSearch(name string) (int, error) {
// GetIGDBAppParent returns the parent of the game, if no parent return itself
func GetIGDBAppParent(id int) (int, error) {
detail, err := GetIGDBAppDetailCache(id)
key := fmt.Sprintf("igdb_parent:%d", id)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
}
detail, err := GetIGDBAppDetail(id)
if err != nil {
return 0, err
}
hasParent := false
for detail.VersionParent != 0 {
hasParent = true
detail, err = GetIGDBAppDetailCache(detail.VersionParent)
detail, err = GetIGDBAppDetail(detail.VersionParent)
if err != nil {
return 0, err
}
@ -238,67 +231,19 @@ func GetIGDBAppParent(id int) (int, error) {
if hasParent {
return detail.ID, nil
}
_ = cache.Set(key, id)
return id, nil
}
func GetIGDBAppParetns(ids []int) (map[int]int, error) {
var err error
idsStr := make([]string, len(ids))
for i, id := range ids {
idsStr[i] = strconv.Itoa(id)
}
resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")))
if err != nil {
return nil, err
}
var data []struct {
ID int `json:"id"`
VersionParent int `json:"version_parent"`
}
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return nil, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
}
parents := make(map[int]int)
for _, item := range data {
if item.VersionParent != 0 {
pid, err := GetIGDBAppParentCache(item.VersionParent)
if err != nil {
parents[item.ID] = item.ID
} else {
parents[item.ID] = pid
}
} else {
parents[item.ID] = item.ID
}
}
return parents, nil
}
func GetIGDBAppParentCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_parent:%d", id)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
} else {
id, err := GetIGDBAppParent(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, id)
return id, nil
}
}
return GetIGDBAppParent(id)
}
// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
func GetIGDBID(name string) (int, error) {
key := fmt.Sprintf("igdb_id:%s", name)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
}
name1 := name
name2 := FormatName(name)
names := []string{name1}
@ -308,42 +253,30 @@ func GetIGDBID(name string) (int, error) {
for _, name := range names {
id, err := getIGDBID(name)
if err == nil {
_ = cache.Set(key, id)
return id, nil
}
}
for _, name := range names {
id, err := getIGDBIDBySteamSearch(name)
if err == nil {
_ = cache.Set(key, id)
return id, nil
}
}
return 0, errors.New("IGDB ID not found")
}
func GetIGDBIDCache(name string) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id:%s", name)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
} else {
id, err := GetIGDBID(name)
if err != nil {
return 0, err
}
_ = cache.Add(key, id)
return id, nil
}
} else {
return GetIGDBID(name)
}
}
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
key := fmt.Sprintf("igdb_game:%v", id)
val, exist := cache.Get(key)
if exist {
var data model.IGDBGameDetail
if err := json.Unmarshal([]byte(val), &data); err != nil {
return nil, err
}
return &data, nil
}
var err error
resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id))
@ -360,37 +293,21 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
if data[0].Name == "" {
return GetIGDBAppDetail(id)
}
jsonBytes, err := json.Marshal(data[0])
if err == nil {
_ = cache.Set(key, string(jsonBytes))
}
return data[0], nil
}
func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_game:%v", id)
val, exist := cache.Get(key)
if exist {
var data model.IGDBGameDetail
if err := json.Unmarshal([]byte(val), &data); err != nil {
return nil, err
}
return &data, nil
} else {
data, err := GetIGDBAppDetail(id)
if err != nil {
return nil, err
}
dataBytes, err := json.Marshal(data)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour)
return data, nil
}
} else {
return GetIGDBAppDetail(id)
}
}
func GetIGDBCompany(id int) (string, error) {
key := fmt.Sprintf("igdb_companies:%v", id)
val, exist := cache.Get(key)
if exist {
return val, nil
}
var err error
resp, err := igdbRequest(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
if err != nil {
@ -406,31 +323,15 @@ func GetIGDBCompany(id int) (string, error) {
if data[0].Name == "" {
return GetIGDBCompany(id)
}
return data[0].Name, nil
}
func GetIGDBCompanyCache(id int) (string, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_companies:%v", id)
val, exist := cache.Get(key)
if exist {
return val, nil
} else {
data, err := GetIGDBCompany(id)
if err != nil {
return "", err
}
_ = cache.Add(key, data)
return data, nil
}
} else {
return GetIGDBCompany(id)
}
_ = cache.Set(key, data[0].Name)
return data[0].Name, nil
}
func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
item := &model.GameInfo{}
detail, err := GetIGDBAppDetailCache(id)
detail, err := GetIGDBAppDetail(id)
if err != nil {
return nil, err
}
@ -460,7 +361,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
for _, company := range detail.InvolvedCompanies {
if company.Developer || company.Publisher {
companyName, err := GetIGDBCompanyCache(company.Company)
companyName, err := GetIGDBCompany(company.Company)
if err != nil {
continue
}
@ -478,7 +379,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
id, err := GetIGDBIDCache(game.Name)
id, err := GetIGDBID(game.Name)
if err != nil {
return nil, err
}
@ -498,6 +399,11 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
}
func GetIGDBIDBySteamAppID(id int) (int, error) {
key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
}
var err error
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
@ -515,29 +421,19 @@ func GetIGDBIDBySteamAppID(id int) (int, error) {
if data[0].Game == 0 {
return GetIGDBIDBySteamAppID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamAppIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamAppID(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, strconv.Itoa(data))
return data, nil
}
} else {
return GetIGDBIDBySteamAppID(id)
}
_ = cache.Set(key, strconv.Itoa(data[0].Game))
return GetIGDBAppParent(data[0].Game)
}
func GetIGDBIDBySteamBundleID(id int) (int, error) {
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
}
var err error
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
@ -555,104 +451,10 @@ func GetIGDBIDBySteamBundleID(id int) (int, error) {
if data[0].Game == 0 {
return GetIGDBIDBySteamBundleID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamBundleIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamBundleID(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, strconv.Itoa(data))
return data, nil
}
} else {
return GetIGDBIDBySteamBundleID(id)
}
}
_ = cache.Set(key, strconv.Itoa(data[0].Game))
func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
var err error
conditionBuilder := strings.Builder{}
for _, id := range ids {
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id))
}
condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
resp, err := igdbRequest(constant.IGDBWebsitesURL, respBody)
if err != nil {
return nil, err
}
var data []struct {
Game int `json:"game"`
Url string `json:"url"`
}
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return nil, err
}
ret := make(map[int]int)
regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
for _, d := range data {
idStr := regex.FindStringSubmatch(d.Url)
if len(idStr) < 2 {
continue
}
id, err := strconv.Atoi(idStr[1])
if err == nil {
pid, err := GetIGDBAppParentCache(d.Game)
if err == nil {
ret[id] = pid
} else {
ret[id] = 0
}
}
}
for _, id := range ids {
if _, ok := ret[id]; !ok {
ret[id] = 0
}
}
return ret, nil
}
func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
res := make(map[int]int)
notExistIDs := make([]int, 0)
if config.Config.RedisAvaliable {
for _, steamID := range ids {
key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID)
val, exist := cache.Get(key)
if exist {
igdbID, _ := strconv.Atoi(val)
res[steamID] = igdbID
} else {
notExistIDs = append(notExistIDs, steamID)
}
}
if len(res) == len(ids) {
return res, nil
}
idMap, err := GetIGDBIDsBySteamIDs(notExistIDs)
if err != nil {
return nil, err
}
for steamID, igdbID := range idMap {
res[steamID] = igdbID
if igdbID != 0 {
_ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID)
}
}
return res, nil
} else {
return GetIGDBIDsBySteamIDs(ids)
}
return GetIGDBAppParent(data[0].Game)
}
// GetIGDBPopularGameIDs get IGDB popular game IDs
@ -676,7 +478,7 @@ func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, er
}
ret := make([]int, 0)
for _, d := range data {
pid, err := GetIGDBAppParentCache(d.GameID)
pid, err := GetIGDBAppParent(d.GameID)
if err != nil {
ret = append(ret, d.GameID)
continue
@ -685,30 +487,3 @@ func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, er
}
return ret, nil
}
func GetIGDBPopularGameIDsCache(popularityType int, offset int, limit int) ([]int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_popular_game_ids:%v:%v:%v", popularityType, offset, limit)
val, exist := cache.Get(key)
if exist {
var data []int
if err := json.Unmarshal([]byte(val), &data); err != nil {
return nil, err
}
return data, nil
} else {
data, err := GetIGDBPopularGameIDs(popularityType, offset, limit)
if err != nil {
return nil, err
}
jsonBytes, err := json.Marshal(data)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, jsonBytes, 12*time.Hour)
return data, nil
}
} else {
return GetIGDBPopularGameIDs(popularityType, offset, limit)
}
}

4
crawler/onlinefix.go
View File

@ -39,10 +39,6 @@ func (c *OnlineFixCrawler) Name() string {
}
func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
if !config.Config.OnlineFixAvaliable {
c.logger.Error("Need Online Fix account")
return nil, errors.New("online Fix is not available")
}
cookies, err := c.getCookies()
if err != nil {
return nil, err

79
crawler/steam.go
View File

@ -11,13 +11,22 @@ import (
"time"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/constant"
"pcgamedb/model"
"pcgamedb/utils"
)
func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
key := fmt.Sprintf("steam_game:%d", id)
val, exist := cache.Get(key)
if exist {
var detail model.SteamAppDetail
if err := json.Unmarshal([]byte(val), &detail); err != nil {
return nil, err
}
return &detail, nil
}
baseURL, _ := url.Parse(constant.SteamAppDetailURL)
params := url.Values{}
params.Add("appids", strconv.Itoa(id))
@ -39,39 +48,18 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
if detail[strconv.Itoa(id)] == nil {
return nil, fmt.Errorf("steam App not found: %d", id)
}
return detail[strconv.Itoa(id)], nil
}
func GetSteamAppDetailCache(id int) (*model.SteamAppDetail, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("steam_game:%d", id)
val, exist := cache.Get(key)
if exist {
var detail model.SteamAppDetail
if err := json.Unmarshal([]byte(val), &detail); err != nil {
return nil, err
}
return &detail, nil
} else {
data, err := GetSteamAppDetail(id)
if err != nil {
return nil, err
}
dataBytes, err := json.Marshal(data)
if err != nil {
return nil, err
}
_ = cache.Add(key, dataBytes)
return data, nil
}
} else {
return GetSteamAppDetail(id)
jsonBytes, err := json.Marshal(detail[strconv.Itoa(id)])
if err == nil {
_ = cache.Set(key, string(jsonBytes))
}
return detail[strconv.Itoa(id)], nil
}
func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
item := &model.GameInfo{}
detail, err := GetSteamAppDetailCache(id)
detail, err := GetSteamAppDetail(id)
if err != nil {
return nil, err
}
@ -91,6 +79,16 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
}
func GetSteamIDByIGDBID(IGDBID int) (int, error) {
key := fmt.Sprintf("steam_game:%d", IGDBID)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
}
var err error
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))
if err != nil {
@ -117,32 +115,9 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) {
if err != nil {
return 0, err
}
_ = cache.Set(key, strconv.Itoa(steamID))
return steamID, nil
}
}
return 0, errors.New("not found")
}
func GetSteamIDByIGDBIDCache(IGDBID int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("steam_game:%d", IGDBID)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
} else {
id, err := GetSteamIDByIGDBID(IGDBID)
if err != nil {
return 0, err
}
dataBytes := strconv.Itoa(id)
_ = cache.Add(key, dataBytes)
return id, nil
}
} else {
return GetSteamIDByIGDBID(IGDBID)
}
}

158
crawler/steam250.go
View File

@ -4,8 +4,8 @@ import (
"bytes"
"encoding/json"
"fmt"
"net/url"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/db"
"regexp"
"strconv"
@ -19,6 +19,16 @@ import (
)
func GetSteam250(URL string) ([]*model.GameInfo, error) {
key := "steam250:" + url.QueryEscape(URL)
if val, ok := cache.Get(key); ok {
var infos []*model.GameInfo
err := json.Unmarshal([]byte(val), &infos)
if err != nil {
return nil, err
}
return infos, nil
}
resp, err := utils.Request().Get(URL)
if err != nil {
return nil, err
@ -48,6 +58,12 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
if len(infos) > 10 {
return infos[:10], nil
}
jsonBytes, err := json.Marshal(infos)
if err == nil {
_ = cache.SetWithExpire(key, string(jsonBytes), 12*time.Hour)
}
return infos, nil
}
@ -55,158 +71,18 @@ func GetSteam250Top250() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250Top250URL)
}
func GetSteam250Top250Cache() ([]*model.GameInfo, error) {
if config.Config.RedisAvaliable {
key := "steam250_top250"
val, exist := cache.Get(key)
if exist {
var infos []*model.GameInfo
err := json.Unmarshal([]byte(val), &infos)
if err != nil {
return nil, err
}
return infos, nil
} else {
infos, err := GetSteam250Top250()
if err != nil {
return nil, err
}
jsonBytes, err := json.Marshal(infos)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
return infos, nil
}
} else {
return GetSteam250Top250()
}
}
func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) {
return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, time.Now().UTC().Year()))
}
func GetSteam250BestOfTheYearCache() ([]*model.GameInfo, error) {
if config.Config.RedisAvaliable {
key := "steam250_best_of_the_year"
val, exist := cache.Get(key)
if exist {
var infos []*model.GameInfo
err := json.Unmarshal([]byte(val), &infos)
if err != nil {
return nil, err
}
return infos, nil
} else {
infos, err := GetSteam250BestOfTheYear()
if err != nil {
return nil, err
}
jsonBytes, err := json.Marshal(infos)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
return infos, nil
}
} else {
return GetSteam250BestOfTheYear()
}
}
func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250WeekTop50URL)
}
func GetSteam250WeekTop50Cache() ([]*model.GameInfo, error) {
if config.Config.RedisAvaliable {
key := "steam250_week_top50"
val, exist := cache.Get(key)
if exist {
var infos []*model.GameInfo
err := json.Unmarshal([]byte(val), &infos)
if err != nil {
return nil, err
}
return infos, nil
} else {
infos, err := GetSteam250WeekTop50()
if err != nil {
return nil, err
}
jsonBytes, err := json.Marshal(infos)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
return infos, nil
}
} else {
return GetSteam250WeekTop50()
}
}
func GetSteam250MonthTop50() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250MonthTop50URL)
}
func GetSteam250MonthTop50Cache() ([]*model.GameInfo, error) {
if config.Config.RedisAvaliable {
key := "steam250_month_top50"
val, exist := cache.Get(key)
if exist {
var infos []*model.GameInfo
err := json.Unmarshal([]byte(val), &infos)
if err != nil {
return nil, err
}
return infos, nil
} else {
infos, err := GetSteam250MonthTop50()
if err != nil {
return nil, err
}
jsonBytes, err := json.Marshal(infos)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
return infos, nil
}
} else {
return GetSteam250MonthTop50()
}
}
func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250MostPlayedURL)
}
func GetSteam250MostPlayedCache() ([]*model.GameInfo, error) {
if config.Config.RedisAvaliable {
key := "steam250_most_played"
val, exist := cache.Get(key)
if exist {
var infos []*model.GameInfo
err := json.Unmarshal([]byte(val), &infos)
if err != nil {
return nil, err
}
return infos, nil
} else {
infos, err := GetSteam250MostPlayed()
if err != nil {
return nil, err
}
jsonBytes, err := json.Marshal(infos)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
return infos, nil
}
} else {
return GetSteam250MostPlayed()
}
}

4
db/db.go
View File

@ -32,10 +32,6 @@ var (
)
func connect() {
if !config.Config.DatabaseAvaliable {
log.Logger.Panic("Missing database configuration information")
return
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
clientOptions := options.Client().ApplyURI(fmt.Sprintf(

55
db/game.go
View File

@ -11,7 +11,6 @@ import (
"time"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/model"
"go.mongodb.org/mongo-driver/bson"
@ -299,6 +298,21 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
name = strings.TrimSpace(name)
name = strings.Replace(name, " ", ".*", -1)
name = fmt.Sprintf("%s.*", name)
key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize)
val, exist := cache.Get(key)
if exist {
var data struct {
Items []*model.GameInfo
TotalPage int
}
err := json.Unmarshal([]byte(val), &data)
if err != nil {
return nil, 0, err
}
return data.Items, data.TotalPage, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@ -334,40 +348,19 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
if err := cursor.Err(); err != nil {
return nil, 0, err
}
return items, int(totalPage), nil
}
func SearchGameInfosCache(name string, page int, pageSize int) ([]*model.GameInfo, int, error) {
type res struct {
jsonBytes, err := json.Marshal(struct {
Items []*model.GameInfo
TotalPage int
}{
Items: items,
TotalPage: int(totalPage),
})
if err == nil {
_ = cache.SetWithExpire(key, string(jsonBytes), time.Minute*5)
}
name = strings.ToLower(name)
if config.Config.RedisAvaliable {
key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize)
val, exist := cache.Get(key)
if exist {
var data res
err := json.Unmarshal([]byte(val), &data)
if err != nil {
return nil, 0, err
}
return data.Items, data.TotalPage, nil
} else {
data, totalPage, err := SearchGameInfos(name, page, pageSize)
if err != nil {
return nil, 0, err
}
dataBytes, err := json.Marshal(res{Items: data, TotalPage: totalPage})
if err != nil {
return nil, 0, err
}
_ = cache.AddWithExpire(key, string(dataBytes), 5*time.Minute)
return data, totalPage, nil
}
} else {
return SearchGameInfos(name, page, pageSize)
}
return items, int(totalPage), nil
}
func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) {

2
go.mod
View File

@ -25,7 +25,7 @@ require (
)
require (
git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580 // indirect
git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760 // indirect
github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 // indirect
github.com/Danny-Dasilva/fhttp v0.0.0-20240217042913-eeeb0b347ce1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect

4
go.sum
View File

@ -21,6 +21,10 @@ git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff h1:f5OEMRc/zhMxSfHdTJ
git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580 h1:B2ewPM44DgyrkycIrUfyTRLM7mXggA0JE8pNbvxjFKw=
git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c h1:UcIxgKmcQGZqjTJWsQf9MVDviUQFlU+ZK6HJjpW+nAU=
git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760 h1:ZCMgQt2ILohQ3MSk6RVhnRY4fbQTJVLREbWZjLtVs+Y=
git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 h1:6fexoGmvzoXMSk14BZ0AirapVm5c3KUsEjE0jLlVKi8=
github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26/go.mod h1:QFi/EVO7qqru3Ftxz1LR+96jIc91Tifv0DnskF/gWQ8=

2
server/handler/get_popular_game_infos.go
View File

@ -87,7 +87,7 @@ func GetPopularGameInfosHandler(c *gin.Context) {
offset += 20
pids := make([]int, 20)
for _, id := range ids {
pid, err := crawler.GetIGDBAppParentCache(id)
pid, err := crawler.GetIGDBAppParent(id)
if err != nil {
continue
}

48
server/handler/healthcheck.go
View File

@ -14,19 +14,17 @@ import (
)
type HealthCheckResponse struct {
Version string `json:"version"`
Status string `json:"status"`
Date string `json:"date"`
Uptime string `json:"uptime"`
Alloc string `json:"alloc"`
AutoCrawl bool `json:"auto_crawl"`
AutoCrawlCron string `json:"auto_crawl_cron"`
GameItem int64 `json:"game_num"`
GameInfo int64 `json:"game_info_num"`
Unorganized int64 `json:"unorganized_game_num"`
RedisAvaliable bool `json:"redis_avaliable"`
OnlineFixAvaliable bool `json:"online_fix_avaliable"`
MegaAvaliable bool `json:"mega_avaliable"`
Version string `json:"version"`
Status string `json:"status"`
Date string `json:"date"`
Uptime string `json:"uptime"`
Alloc string `json:"alloc"`
AutoCrawl bool `json:"auto_crawl"`
AutoCrawlCron string `json:"auto_crawl_cron"`
GameItem int64 `json:"game_num"`
GameInfo int64 `json:"game_info_num"`
Unorganized int64 `json:"unorganized_game_num"`
MegaAvaliable bool `json:"mega_avaliable"`
}
// HealthCheckHandler performs a health check of the service.
@ -49,18 +47,16 @@ func HealthCheckHandler(c *gin.Context) {
unorganizedCount = int64(len(unorganized))
}
c.JSON(http.StatusOK, HealthCheckResponse{
Status: "ok",
Version: constant.Version,
Date: time.Now().Format("2006-01-02 15:04:05"),
Uptime: time.Since(config.Runtime.ServerStartTime).String(),
AutoCrawl: config.Config.Server.AutoCrawl,
AutoCrawlCron: config.Config.Server.AutoCrawlCron,
Alloc: fmt.Sprintf("%.2f MB", float64(m.Alloc)/1024.0/1024.0),
GameItem: downloadCount,
GameInfo: infoCount,
Unorganized: unorganizedCount,
RedisAvaliable: config.Config.RedisAvaliable,
OnlineFixAvaliable: config.Config.OnlineFixAvaliable,
MegaAvaliable: config.Config.MegaAvaliable,
Status: "ok",
Version: constant.Version,
Date: time.Now().Format("2006-01-02 15:04:05"),
Uptime: time.Since(config.Runtime.ServerStartTime).String(),
AutoCrawl: config.Config.Server.AutoCrawl,
AutoCrawlCron: config.Config.Server.AutoCrawlCron,
Alloc: fmt.Sprintf("%.2f MB", float64(m.Alloc)/1024.0/1024.0),
GameItem: downloadCount,
GameInfo: infoCount,
Unorganized: unorganizedCount,
MegaAvaliable: config.Config.MegaAvaliable,
})
}

2
server/handler/search_games.go
View File

@ -53,7 +53,7 @@ func SearchGamesHandler(c *gin.Context) {
if req.PageSize > 10 {
req.PageSize = 10
}
items, totalPage, err := db.SearchGameInfosCache(req.Keyword, req.Page, req.PageSize)
items, totalPage, err := db.SearchGameInfos(req.Keyword, req.Page, req.PageSize)
if err != nil {
c.JSON(http.StatusInternalServerError, SearchGamesResponse{
Status: "error",

6
server/route.go
View File

@ -83,17 +83,17 @@ func initFrontend(app *gin.Engine) {
// Load routes
app.GET("/", func(ctx *gin.Context) {
monthTop, err := crawler.GetSteam250MonthTop50Cache()
monthTop, err := crawler.GetSteam250MonthTop50()
if err != nil {
ctx.HTML(500, "500.html", err)
return
}
mostPlayed, err := crawler.GetSteam250MostPlayedCache()
mostPlayed, err := crawler.GetSteam250MostPlayed()
if err != nil {
ctx.HTML(500, "500.html", err)
return
}
bestOfTheYear, err := crawler.GetSteam250BestOfTheYearCache()
bestOfTheYear, err := crawler.GetSteam250BestOfTheYear()
if err != nil {
ctx.HTML(500, "500.html", err)
return