package service import ( "bufio" "designs/config" "designs/global" "designs/model" "designs/utils" "fmt" "github.com/pkg/errors" "math" "os" "path/filepath" "strconv" "strings" "time" ) func RemainDataBydDay(types int, pf string, gid string, startTime string, endTime string) (map[string]map[string]interface{}, error) { var err error start := time.Now() //对于传过来的endTime ,做一个处理 t, _ := time.Parse("2006-01-02", endTime) endTimeData := t.AddDate(0, 0, 1).Format("2006-01-02") UsersBydDay := utils.GetTimeDayDate(startTime, endTime) //用户分别是在哪天注册的 UserLoginBydDay := utils.GetTimeDayDate(startTime, endTime) //用户分别是在哪天注册的 var UsersId []int userIdMap := make(map[int]bool) if types == 1 { //先计算出这个时间内 ,每天的注册用户数量 var users []model.User err = global.App.DB.Table("user"). Where("pf", pf).Where("gid", gid). Where("createdAt", ">=", startTime). Where("createdAt", "<=", endTimeData). Select("userId", "createdAt"). Scan(&users).Error if err != nil { return nil, err } fmt.Printf("步骤2耗时: %v\n", time.Since(start)) for _, user := range users { userIdMap[user.UserId] = true UsersBydDay[user.CreatedAt.Format("2006-01-02")] = append(UsersBydDay[user.CreatedAt.Format("2006-01-02")], user.UserId) } for user := range userIdMap { UsersId = append(UsersId, user) } users = nil //用完后清空内存 } else if types == 2 { var users []model.UserLogin err = global.App.DB.Table("user_login"). Where("pf", pf).Where("gid", gid). Where("loginTime", ">=", startTime). Where("loginTime", "<=", endTimeData). Group("userId"). Scan(&users).Error if err != nil { return nil, err } for _, user := range users { UsersId = append(UsersId, user.UserId) UsersBydDay[user.LoginTime.Format("2006-01-02")] = append(UsersBydDay[user.LoginTime.Format("2006-01-02")], user.UserId) } } else { return nil, errors.New("type 参数错误") } //把每天的注册用户进行集合,查出后面所有天数的活跃情况 var UserLogin []model.UserOnline //err = global.App.DB.Table("user_online_copy1 as user_online"). // Where("pf", pf).Where("gid", gid). // Raw("USE INDEX (DATE)"). // WhereIn("userId", UsersId). // Where("logTime", ">=", startTime). // Select("logTime", "user_online.userId"). // Group("date,user_online.userId"). // Scan(&UserLogin).Error //if err != nil { // return nil, err //} // 使用 Raw 方法执行带有 FORCE INDEX 的 SQL 查询 query := ` SELECT logTime, user_online.userId FROM user_online AS user_online USE INDEX (date) INNER JOIN ( SELECT userId FROM user WHERE pf = ? AND gid = ? AND createdAt >= ? AND createdAt <= ? ) AS t ON t.userId = user_online.userId WHERE user_online.pf = ? AND user_online.gid = ? AND DATE(logTime) >= ? GROUP BY date, user_online.userId; ` err = global.App.DB.Raw(query, pf, gid, startTime, endTime, pf, gid, startTime).Scan(&UserLogin).Error if err != nil { return nil, err } fmt.Printf("步骤3耗时: %v\n", time.Since(start)) //对这些数据进行整理 for _, v := range UserLogin { //根据天进行分组,得出总共有多少 times := v.LogTime.Format("2006-01-02") UserLoginBydDay[times] = append(UserLoginBydDay[times], v.UserId) } res := make(map[string]map[string]interface{}) //逐天比较注册的数据和留存的数据,得出每天的活跃比例和人数 for day, v := range UsersBydDay { remain := make(map[string]interface{}) remain["count"] = len(v) remain["date"] = day //分别比较每一天的数据 for dayDate, loginDay := range UserLoginBydDay { ok, remainDays := utils.CompareDates(dayDate, day) if !ok { continue } remain[remainDays] = math.Round(utils.IntersectionRate(v, loginDay)*100*100) / 100 } res[day] = remain } fmt.Printf("步骤4耗时: %v\n", time.Since(start)) return res, nil } func RemainDataBydDayNew(types int, pf string, gid string, startTime string, endTime string) (map[string]map[string]interface{}, error) { //目前只有types = 1 //对于传过来的endTime ,做一个处理 t, _ := time.Parse("2006-01-02", endTime) endTimeData := t.AddDate(0, 0, 1).Format("2006-01-02") UsersBydDay := utils.GetTimeDayDate(startTime, endTime) //用户分别是在哪天注册的 //先计算出这个时间内 ,每天的注册用户数量 var users []model.User err := global.App.DB.Table("user"). Where("pf", pf).Where("gid", gid). Where("createdAt", ">=", startTime). Where("createdAt", "<=", endTimeData). Select("userId", "createdAt"). Scan(&users).Error if err != nil { return nil, err } var UsersId []int userIdMap := make(map[int]bool) for _, user := range users { userIdMap[user.UserId] = true UsersBydDay[user.CreatedAt.Format("2006-01-02")] = append(UsersBydDay[user.CreatedAt.Format("2006-01-02")], user.UserId) } for user := range userIdMap { UsersId = append(UsersId, user) } UserLoginBydDay := utils.GetTimeDayDate(startTime, endTime) //用户分别是在哪天注册的 for k, _ := range UserLoginBydDay { //根据pf,gid,startTime,endTime 取出数据 UserLoginBydDay[k] = GetLocalActiveLog(gid, pf, k) } res := make(map[string]map[string]interface{}) for day, v := range UsersBydDay { remain := make(map[string]interface{}) remain["count"] = len(v) remain["date"] = day //分别比较每一天的数据 for dayDate, loginDay := range UserLoginBydDay { ok, remainDays := utils.CompareDates(dayDate, day) if !ok { continue } remain[remainDays] = math.Round(utils.IntersectionRate(v, loginDay)*100*100) / 100 } res[day] = remain } return res, nil } func OnlineToFile() { //分批读取数据 now := time.Now() for i := 1; i <= 45; i++ { var onlineData []model.UserOnline date := now.AddDate(0, 0, -i).Format("20060102") global.App.DB.Table("user_online_old").Where("date", date).Scan(&onlineData) //根据gid和pf,分文件存储 for _, v := range onlineData { SetLocalActiveLog(v.Gid, v.Pf, "", v.UserId, v.Type, v.LogTime) } fmt.Println(date, "数据归档完成,耗时:", time.Since(now)) } //var onlineData []model.UserOnline // //date := "20250423" // //global.App.DB.Table("user_online").Where("date", date).Scan(&onlineData) // ////根据gid和pf,分文件存储 //for _, v := range onlineData { // SetLocalActiveLog(v.Gid, v.Pf, "", v.UserId, v.Type, v.LogTime) //} } func SetLocalActiveLog(gid string, pf string, openId string, userId int, types int, now time.Time) error { // 生成日期目录路径(示例:./data/2024-06-15) dateDir := now.Format("2006-01-02") dirPath := filepath.Join("storage", dateDir) // 创建日期目录(若不存在) if err := os.MkdirAll(dirPath, 0755); err != nil { return err } // 生成文件名(示例:g123_wechat.txt) fileName := fmt.Sprintf("%s_%s.txt", gid, pf) filePath := filepath.Join(dirPath, fileName) // 追加写入文件 file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return fmt.Errorf("failed to open file: %v", err) } defer file.Close() // 写入数据(示例:1584,oXyZ123456\n) if _, err := fmt.Fprintf(file, "%d,%d,%s\n", userId, types, now.Format("2006-01-02 15:04:05")); err != nil { return fmt.Errorf("failed to write file: %v", err) } return nil } func GetLocalActiveLog(gid string, pf string, date string) []int { var dir string if config.Get("app.local") == "local" { //url = "mongodb://localhost:27017" dir = "storage" } else { dir = "/www/wwwroot/chunhao_receive/storage" } dirPath := filepath.Join(dir, date) fileName := fmt.Sprintf("%s_%s.txt", gid, pf) filePath := filepath.Join(dirPath, fileName) file, _ := os.Open(filePath) defer file.Close() // 确保函数结束时关闭文件 // 创建 Scanner 对象 scanner := bufio.NewScanner(file) var res []int // 逐行读取文件内容 lineNumber := 1 for scanner.Scan() { line := scanner.Text() // 获取当前行内容 lineNumber++ userId, _ := strconv.Atoi(strings.Split(line, ",")[0]) if !utils.InArray(userId, res) { res = append(res, userId) } } return res }