package crawler import ( "bytes" "encoding/json" "fmt" "pcgamedb/cache" "pcgamedb/config" "pcgamedb/db" "regexp" "strconv" "time" "pcgamedb/constant" "pcgamedb/model" "pcgamedb/utils" "github.com/PuerkitoBio/goquery" ) func GetSteam250(url string) ([]*model.GameInfo, error) { resp, err := utils.Fetch(utils.FetchConfig{ Url: url, }) if err != nil { return nil, err } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data)) if err != nil { return nil, err } var rank []model.Steam250Item var item model.Steam250Item steamIDs := make([]int, 0) doc.Find(".appline").Each(func(i int, s *goquery.Selection) { item.Name = s.Find(".title>a").First().Text() idStr := s.Find(".store").AttrOr("href", "") idSlice := regexp.MustCompile(`app/(\d+)/`).FindStringSubmatch(idStr) if len(idSlice) < 2 { return } item.SteamID, _ = strconv.Atoi(idSlice[1]) rank = append(rank, item) steamIDs = append(steamIDs, item.SteamID) }) infos, err := db.GetGameInfosByPlatformIDs("steam", steamIDs) if err != nil { return nil, err } if len(infos) > 10 { return infos[:10], nil } return infos, nil } func GetSteam250Top250() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250Top250URL) } func GetSteam250Top250Cache() ([]*model.GameInfo, error) { if config.Config.RedisAvaliable { key := "steam250_top250" val, exist := cache.Get(key) if exist { var infos []*model.GameInfo err := json.Unmarshal([]byte(val), &infos) if err != nil { return nil, err } return infos, nil } else { infos, err := GetSteam250Top250() if err != nil { return nil, err } jsonBytes, err := json.Marshal(infos) if err != nil { return nil, err } _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) return infos, nil } } else { return GetSteam250Top250() } } func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) { return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, time.Now().UTC().Year())) } func GetSteam250BestOfTheYearCache() ([]*model.GameInfo, error) { if config.Config.RedisAvaliable { key := "steam250_best_of_the_year" val, exist := cache.Get(key) if exist { var infos []*model.GameInfo err := json.Unmarshal([]byte(val), &infos) if err != nil { return nil, err } return infos, nil } else { infos, err := GetSteam250BestOfTheYear() if err != nil { return nil, err } jsonBytes, err := json.Marshal(infos) if err != nil { return nil, err } _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) return infos, nil } } else { return GetSteam250BestOfTheYear() } } func GetSteam250WeekTop50() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250WeekTop50URL) } func GetSteam250WeekTop50Cache() ([]*model.GameInfo, error) { if config.Config.RedisAvaliable { key := "steam250_week_top50" val, exist := cache.Get(key) if exist { var infos []*model.GameInfo err := json.Unmarshal([]byte(val), &infos) if err != nil { return nil, err } return infos, nil } else { infos, err := GetSteam250WeekTop50() if err != nil { return nil, err } jsonBytes, err := json.Marshal(infos) if err != nil { return nil, err } _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) return infos, nil } } else { return GetSteam250WeekTop50() } } func GetSteam250MonthTop50() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250MonthTop50URL) } func GetSteam250MonthTop50Cache() ([]*model.GameInfo, error) { if config.Config.RedisAvaliable { key := "steam250_month_top50" val, exist := cache.Get(key) if exist { var infos []*model.GameInfo err := json.Unmarshal([]byte(val), &infos) if err != nil { return nil, err } return infos, nil } else { infos, err := GetSteam250MonthTop50() if err != nil { return nil, err } jsonBytes, err := json.Marshal(infos) if err != nil { return nil, err } _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) return infos, nil } } else { return GetSteam250MonthTop50() } } func GetSteam250MostPlayed() ([]*model.GameInfo, error) { return GetSteam250(constant.Steam250MostPlayedURL) } func GetSteam250MostPlayedCache() ([]*model.GameInfo, error) { if config.Config.RedisAvaliable { key := "steam250_most_played" val, exist := cache.Get(key) if exist { var infos []*model.GameInfo err := json.Unmarshal([]byte(val), &infos) if err != nil { return nil, err } return infos, nil } else { infos, err := GetSteam250MostPlayed() if err != nil { return nil, err } jsonBytes, err := json.Marshal(infos) if err != nil { return nil, err } _ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour) return infos, nil } } else { return GetSteam250MostPlayed() } }