增加clickhouse相关功能，修改部分接口逻辑

.gitignore

@@ -1 +1,2 @@
 storage
+/uploads

bootstrap/clickhouse.go

@@ -0,0 +1,41 @@
+package bootstrap
+
+import (
+	"designs/config"
+	"designs/global"
+	"fmt"
+	"go.uber.org/zap"
+	"gorm.io/driver/clickhouse"
+	"gorm.io/gorm"
+	"time"
+)
+
+func InitClickhouseGorm() *gorm.DB {
+
+	UserName := config.Get("clickhouse.userName")
+	Password := config.Get("clickhouse.password")
+	Host := config.Get("clickhouse.host")
+	Port := config.Get("clickhouse.port")
+	Database := config.Get("clickhouse.database")
+
+	dsn := fmt.Sprintf("tcp://%s:%s?database=%s&username=%s&password=%s", Host, Port, Database, UserName, Password)
+	clickConfig := clickhouse.Config{
+		DSN: dsn,
+	}
+
+	if db, err := gorm.Open(clickhouse.New(clickConfig), &gorm.Config{
+		Logger: getGormLogger(), // 使用自定义 Logger
+	}); err != nil {
+		global.App.Log.Error("clickhouse connect failed, err:", zap.Any("err", err))
+		return nil
+	} else {
+		sqlDB, _ := db.DB()
+
+		sqlDB.SetMaxIdleConns(5)                   // 空闲连接数 ≈ 平均QPS * 平均查询时间(秒)
+		sqlDB.SetMaxOpenConns(20)                  // 最大连接数 ≈ 2 * CPU核心数
+		sqlDB.SetConnMaxLifetime(30 * time.Minute) // 连接最大存活时间
+		sqlDB.SetConnMaxIdleTime(5 * time.Minute)  // 连接最大空闲时间
+		//initMySqlTables(db)
+		return db
+	}
+}

bootstrap/cron.go

@@ -19,6 +19,7 @@ func InitializeCron() {
 		global.App.Cron.AddFunc("0 0 1 * * *", crons.ActiveDelete)         //清理用户在线数据文件夹
 		global.App.Cron.AddFunc("0 0 2 * * *", crons.OnlineDatabaseDelete) //清理用户在线数据数据库
 		global.App.Cron.AddFunc("0 0 1 * * *", crons.AdsDataSummary)       //汇总昨日的广告数据
+		global.App.Cron.AddFunc("0 0 3 * * *", crons.Behavior)             //汇总用户的在线时长数据
 		global.App.Cron.Start()
 		defer global.App.Cron.Stop()
 		select {}

bootstrap/db.go

@@ -53,6 +53,7 @@ func initMySqlGorm() *gorm.DB {
 		DontSupportRenameColumn:   true,  // 用 `change` 重命名列，MySQL 8 之前的数据库和 MariaDB 不支持重命名列
 		SkipInitializeWithVersion: false, // 根据版本自动配置
 	}
+
 	if db, err := gorm.Open(mysql.New(mysqlConfig), &gorm.Config{
 		DisableForeignKeyConstraintWhenMigrating: true,            // 禁用自动创建外键约束
 		Logger:                                   getGormLogger(), // 使用自定义 Logger

config/app.go

@@ -35,9 +35,7 @@ func App() *ConfigNode {
 		"black_list_table": env("BLACK_LIST_TABLE", "black_list_table:"), //黑名单表
 		"option_key":       env("OPTION_KEY", "option_key:"),             //用户配置表    option_key:openid
 
-		"admin_user":   env("ADMIN_USER", "chunhao"),    //默认管理员
-		"admin_secret": env("ADMIN_SECRET", "123456"),   //默认管理密码
-		"user_total":   env("USER_TOTAL", "user_total"), //所有用户的集合
+		"user_total": env("USER_TOTAL", "user_total"), //所有用户的集合
 	}
 }
 

config/clickhouse.go

@@ -0,0 +1,14 @@
+package config
+
+func Clickhouse() *ConfigNode {
+	return &ConfigNode{
+		"host":         env("CLICKHOUSE_HOST", "101.42.6.84"),
+		"port":         env("CLICKHOUSE_PORT", "9000"),
+		"database":     env("CLICKHOUSE_DATABASE", "chunhao"),
+		"userName":     env("CLICKHOUSE_USERNAME", "default"),
+		"password":     env("CLICKHOUSE_PASSWORD", "chunhao2024"),
+		"charset":      "utf8mb4",
+		"maxIdleConns": "10",
+		"maxOpenConns": "50",
+	}
+}

config/config.go

@@ -6,11 +6,12 @@ const (
 )
 
 var RootConfig = ConfigRoot{
-	"app":      App,
-	"log":      Log,
-	"jwt":      Jwt,
-	"redis":    Redis,
-	"temp":     Temp,
-	"download": Download,
-	"database": Database,
+	"app":        App,
+	"log":        Log,
+	"jwt":        Jwt,
+	"redis":      Redis,
+	"temp":       Temp,
+	"download":   Download,
+	"database":   Database,
+	"clickhouse": Clickhouse,
 }

controller/v1/gameAction.go

@@ -17,10 +17,10 @@ import (
 
 func UserActionList(c *gin.Context) {
 	form := request.Check(c, &struct {
-		Gid string `form:"gid" binding:"required"`
-		//Pf        []string `form:"pf" binding:""`
-		StartTime string `form:"startTime" binding:"required"`
-		EndTime   string `form:"endTime" binding:"required"`
+		Gid       string   `form:"gid" binding:"required"`
+		Pf        []string `form:"pf" binding:""`
+		StartTime string   `form:"startTime" binding:"required"`
+		EndTime   string   `form:"endTime" binding:"required"`
 	}{})
 
 	form.EndTime = form.EndTime + " 23:59:59"
@@ -67,6 +67,7 @@ func UserActionList(c *gin.Context) {
 	//}
 	err = query2.
 		Where("gid", form.Gid).
+		WhereIn("pf", form.Pf).
 		Where("createdAt", ">=", form.StartTime).
 		Where("createdAt", "<=", form.EndTime).
 		Select("id", "actionId", "userId", "createdAt").

controller/v1/gameConfig.go

@@ -7,13 +7,13 @@ import (
 	"designs/config"
 	"designs/global"
 	"designs/model"
+	"designs/service"
 	"designs/utils"
 	"encoding/json"
 	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/go-playground/validator/v10"
 	"strconv"
-	"strings"
 	"time"
 )
 
@@ -215,7 +215,7 @@ func GidList(c *gin.Context) {
 
 	if form.Active == true {
 		//获取近七天有数据的gid
-		ActiveGid := GetActiveGid()
+		ActiveGid := service.GetActiveGid()
 		query = query.WhereIn("gid", ActiveGid)
 	}
 
@@ -232,73 +232,6 @@ func GidList(c *gin.Context) {
 
 }
 
-func GetActiveGid() []string {
-	var activeGidList []string
-
-	key := "activeGidList"
-	activeGid, _ := global.App.Redis.Get(context.Background(), key).Result()
-
-	activeGidList = strings.Split(activeGid, ",")
-
-	if len(activeGidList) <= 2 {
-		//if true {
-		var gidList []string
-		//
-		////重新读取数据
-		//var dir string
-		//if config.Get("app.local") == "local" {
-		//	//url = "mongodb://localhost:27017"
-		//	dir = "storage"
-		//} else {
-		//	dir = "/www/wwwroot/chunhao_receive/storage"
-		//}
-		//
-		//now := time.Now()
-		//
-		//for i := 0; i <= 7; i++ {
-		//	date := now.AddDate(0, 0, -i).Format("2006-01-02")
-		//	//读取对应的文件夹
-		//	dirPath := filepath.Join(dir, date)
-		//	dateDir, _ := os.ReadDir(dirPath)
-		//	//fmt.Println(dirPath, dateDir)
-		//
-		//	for _, v := range dateDir {
-		//
-		//		fileName := v.Name()
-		//
-		//		fileNameSplit := strings.Split(fileName, "_")
-		//
-		//		if len(fileNameSplit) < 2 {
-		//			continue
-		//		}
-		//		last := fileNameSplit[len(fileNameSplit)-1]
-		//		gid := strings.TrimRight(fileName, "_"+last)
-		//
-		//		if !utils.InArray(gid, gidList) {
-		//			gidList = append(gidList, gid)
-		//		}
-		//	}
-		//}
-		//
-
-		global.App.DB.Table("user").
-			Where("createdAt", ">=", time.Now().AddDate(0, 0, -3)).
-			Distinct("gid").Pluck("gid", &gidList)
-
-		if len(gidList) > 0 {
-
-			var gidString string
-			for _, gid := range gidList {
-				gidString = gid + ","
-			}
-			global.App.Redis.Set(context.Background(), key, gidString, time.Second*3600)
-			activeGidList = gidList
-		}
-	}
-
-	return activeGidList
-}
-
 type ActionOption struct {
 	OptionName   string `json:"optionName" form:"optionName" binding:"required"`
 	OptionId     string `json:"optionId" form:"optionId" binding:"required"`
@@ -352,7 +285,7 @@ func SetGameAction(c *gin.Context) {
 				OptionId:   option.OptionId,
 				ActionId:   gameAction.ID,
 				OptionType: option.OptionType,
-				Status:     option.OptionStatus,
+				Status:     1,
 				CreatedAt:  model.XTime{Time: now},
 				UpdatedAt:  model.XTime{Time: now},
 			}

controller/v1/test.go

@@ -3,6 +3,7 @@ package v1
 import (
 	"context"
 	"designs/config"
+	"designs/crons"
 	"designs/global"
 	"designs/model"
 	"designs/response"
@@ -245,13 +246,29 @@ func ActionDataSummary(c *gin.Context) {
 	response.Success(c, gin.H{})
 }
 
+func BehaviorTimeSummary(c *gin.Context) {
+	//先查出所有gid
+	ActiveGid := service.GetActiveGid()
+
+	//现在直接汇总所有的数据（直到前一天的）
+	now := time.Now()
+	end := now.AddDate(0, 0, -1)
+	start, _ := time.Parse("2006-01-02 15:04:05", "2025-05-17 15:04:05")
+
+	crons.BehaviorSummary("wx", ActiveGid, start, end)
+
+	global.App.Log.Info("重新汇总用户在线时长数据完成")
+
+	response.Success(c, gin.H{})
+}
+
 func BehaviorSummary(c *gin.Context) {
 
 	//先查出所有gid
-	ActiveGid := GetActiveGid()
+	ActiveGid := service.GetActiveGid()
 
-	now := time.Now()
-	start := now.AddDate(0, 0, -40)
+	now := time.Now().AddDate(0, 0, -1)
+	start := now.AddDate(0, 0, -45)
 
 	pf := "wx"
 	//循环一下，查出userId 跟相关的在线数据
@@ -368,3 +385,13 @@ func BehaviorSummary(c *gin.Context) {
 	}
 
 }
+
+func TestClickHouse(c *gin.Context) {
+
+	sql := "CREATE TABLE user_see_ads\n(\n    `pf` LowCardinality(String) COMMENT '登录路径',\n    `gid` LowCardinality(String) COMMENT '游戏ID',\n    `userId` UInt32 COMMENT '用户ID',\n    `date` Date COMMENT '日期',\n    `createdAt` DateTime DEFAULT now() COMMENT '时间',\n    `adsId` Nullable(String) COMMENT '广告ID',\n    \n    -- 修复字段声明顺序：Nullable 应包裹 LowCardinality\n    `adsType` LowCardinality(Nullable(String)) COMMENT '类型',\n    `adsScene` LowCardinality(Nullable(String)) COMMENT '广告场景',\n    \n    `adsState` Nullable(Int8) COMMENT '广告状态: 0失败 1成功 2完整观看',\n    `startTime` Nullable(DateTime) COMMENT '广告开始时间',\n    `cost` UInt32 DEFAULT 0 COMMENT '广告收益'\n)\nENGINE = MergeTree()\nPARTITION BY (toYear(date), toMonth(date))\nORDER BY (date, pf, gid, userId)\nPRIMARY KEY (date, pf, gid)\nSETTINGS \n    index_granularity = 8192,\n    index_granularity_bytes = 10485760;\n"
+	err := global.App.Clickhouse.Exec(sql).Error
+
+	response.Success(c, gin.H{
+		"res": err,
+	})
+}

controller/v1/userAds.go

@@ -5,6 +5,7 @@ import (
 	"designs/app/common/response"
 	"designs/global"
 	"designs/model"
+	"designs/service"
 	"github.com/gin-gonic/gin"
 	"math"
 	"strconv"
@@ -157,6 +158,16 @@ func UserAdsDaily(c *gin.Context) {
 		resToday[newDate] = v.TodayCount
 	}
 
+	//查询今日数据
+	today := time.Now().Format("2006-01-02")
+	if today == form.EndTime {
+		//查询出今天的数据
+		count, todayCount, _ := service.GetTodayAdsSummary(form.Gid, form.Pf)
+
+		res[today] = int(count)
+		resToday[today] = int(todayCount)
+	}
+
 	response.Success(c, gin.H{
 		"data": map[string]interface{}{
 			"userAdsDaily":      res,

controller/v1/userBehavior.go

@@ -818,7 +818,7 @@ func AdRelatedList(c *gin.Context) {
 		Where("gid", form.Gid).Where("pf", form.Pf)
 
 	if form.OpenId != "" {
-		query = query.Where("open_id", form.OpenId)
+		query = query.Where("openId", form.OpenId)
 	}
 
 	if form.Pid != "" {
@@ -826,11 +826,12 @@ func AdRelatedList(c *gin.Context) {
 	}
 
 	if form.Aid != "" {
-		query = query.Where("pid", form.Aid)
+		query = query.Where("aid", form.Aid)
 	}
 
 	if form.CreateTime != nil {
-		query = BuildBehaviorQuery(query, form.CreateTime, "createdAt")
+		//时间戳转化为YMD
+		query = BuildBehaviorQuery(query, form.CreateTime, "UNIX_TIMESTAMP(createdAt)")
 	}
 	if form.StartNum != nil {
 		query = BuildBehaviorQuery(query, form.StartNum, "startNum")
@@ -1053,251 +1054,116 @@ func BehaviorListCake(c *gin.Context) {
 		Gid string `form:"gid" json:"gid" binding:"required"`
 		Pf  string `form:"pf" json:"pf" binding:"required"`
 
-		ActiveStatus       string `form:"activeStatus" json:"activeStatus" binding:""`         //all true false
-		ConversionStatus   string `form:"conversionStatus" json:"conversionStatus" binding:""` //all true false
 		TotalDuration      string `form:"totalDuration" json:"totalDuration" binding:""`
 		TotalAdReqCount    string `form:"totalAdReqCount" json:"totalAdReqCount" binding:""`
 		TotalAdEposedCount string `form:"totalAdEposedCount" json:"totalAdEposedCount" binding:""`
-		CreateTime         string `json:"createTime" bson:"createTime"`
-
-		//AdFromCount string `form:"adFromCount" json:"adFromCount" binding:""`
-		//StartNum string `form:"startNum" json:"startNum" binding:""`
+		CreateTime         string `form:"createTime" json:"createTime"`
 	}{})
-	collection := global.App.MongoDB.Database("chunhao").Collection("userBehavior")
-
-	ctx := context.Background()
-	filter := bson.M{"gid": form.Gid}
 
-	if form.Pf != "" {
-		filter["pf"] = form.Pf
-	}
+	//根据过滤条件分组统计
+	if form.TotalDuration != "" {
+		group := BehaviorCakeQuery(form.Gid, form.Pf, form.CreateTime, form.TotalDuration, "duration", "用户时长")
 
-	if form.ActiveStatus == "true" {
-		filter["activeStatus"] = true
-	} else if form.ActiveStatus == "false" {
-		filter["activeStatus"] = false
+		response.Success(c, gin.H{
+			"data": group,
+		})
+		return
 	}
+	if form.TotalAdReqCount != "" {
+		group := BehaviorCakeQuery(form.Gid, form.Pf, form.CreateTime, form.TotalAdReqCount, "adCount", "广告次数")
 
-	if form.ActiveStatus == "true" {
-		filter["activeStatus"] = true
-	} else if form.ActiveStatus == "false" {
-		filter["activeStatus"] = false
+		response.Success(c, gin.H{
+			"data": group,
+		})
+		return
 	}
 
-	if form.CreateTime != "" {
-		createTime := strings.Split(strings.Replace(form.CreateTime, "，", ",", -1), ",")
-		filters := bson.M{}
-		filters["$gt"], _ = strconv.Atoi(createTime[0])
-		filters["$lte"], _ = strconv.Atoi(createTime[1])
-		filter["createTime"] = filters
-	}
+	if form.TotalAdEposedCount != "" {
+		group := BehaviorCakeQuery(form.Gid, form.Pf, form.CreateTime, form.TotalAdEposedCount, "adExpCount", "广告完播次数")
 
-	type resData struct {
-		Count int    `json:"count"`
-		Name  string `json:"name"`
+		response.Success(c, gin.H{
+			"data": group,
+		})
+		return
 	}
 
-	filterListTotalDuration := make(map[string]bson.M)
+	response.Success(c, gin.H{})
+}
 
-	filterList := make(map[string]bson.M)
+type cakeRes struct {
+	Name  string `json:"name"`
+	Count int64  `json:"count"`
+}
 
-	var data []resData
-	if form.TotalDuration != "" {
-		totalDuration := strings.Split(strings.Replace(form.TotalDuration, "，", ",", -1), ",")
-		for k, _ := range totalDuration {
-			var gt, lte int
-			if k == 0 {
-				gt = 0
-				lte, _ = strconv.Atoi(totalDuration[k])
-			} else {
-				gt, _ = strconv.Atoi(totalDuration[k-1])
-				lte, _ = strconv.Atoi(totalDuration[k])
-			}
+func BehaviorCakeQuery(gid, pf, createTime, filter, filterRow, filterName string) []cakeRes {
 
-			filters := bson.M{}
-			filters["$gt"] = gt
-			filters["$lte"] = lte
+	var group []cakeRes
 
-			filter1 := utils.DeepCopyMap(filter)
+	totalDuration := CakeFilter(filter)
 
-			filter1["totalDuration"] = filters
+	for k, v := range totalDuration {
+		var count int64
+		if k == 0 {
+			query := SetCakeQuery(gid, pf, createTime)
+			query.Where(filterRow, "<=", v).Count(&count)
 
-			name := "在线时长:" + strconv.Itoa(gt) + "-" + strconv.Itoa(lte)
+			group = append(group, cakeRes{
+				Name:  fmt.Sprintf("%s<=%v", filterName, v),
+				Count: count,
+			})
 
-			filterListTotalDuration[name] = filter1
 		}
+		if k == len(totalDuration)-1 {
+			query := SetCakeQuery(gid, pf, createTime)
+			query.Where(filterRow, ">=", v).Count(&count)
 
-		filters := bson.M{}
-		filters["$gt"], _ = strconv.Atoi(totalDuration[len(totalDuration)-1])
-		filter["totalDuration"] = filters
-
-		name := "在线时长:" + totalDuration[len(totalDuration)-1] + "-" + "∞"
-
-		filterListTotalDuration[name] = filter
-	}
-
-	if form.TotalAdReqCount != "" && form.TotalAdEposedCount == "" {
-		totalDuration := strings.Split(strings.Replace(form.TotalAdReqCount, "，", ",", -1), ",")
-
-		for k, _ := range totalDuration {
-			var gt, lte int
-			if k == 0 {
-				gt = 0
-				lte, _ = strconv.Atoi(totalDuration[k])
-			} else {
-				gt, _ = strconv.Atoi(totalDuration[k-1])
-				lte, _ = strconv.Atoi(totalDuration[k])
-			}
-
-			filters := bson.M{}
-			filters["$gt"] = gt
-			filters["$lte"] = lte
-
-			if len(filterListTotalDuration) != 0 {
-				for nameD, filterD := range filterListTotalDuration {
-					filterE := utils.DeepCopyMap(filterD)
-
-					filterE["totalAdReqCount"] = filters
-
-					name := nameD + "&&" + "广告观看次数:" + strconv.Itoa(gt) + "-" + strconv.Itoa(lte)
-
-					filterList[name] = filterE
-				}
-
-			} else {
-				filter1 := utils.DeepCopyMap(filter)
-
-				filter1["totalAdReqCount"] = filters
-				name := "广告观看次数:" + strconv.Itoa(gt) + "-" + strconv.Itoa(lte)
-
-				filterList[name] = filter1
-			}
+			group = append(group, cakeRes{
+				Name:  fmt.Sprintf("%s>=%v", filterName, v),
+				Count: count,
+			})
 		}
 
-		filters := bson.M{}
-		filters["$gt"], _ = strconv.Atoi(totalDuration[len(totalDuration)-1])
-		filter["totalAdReqCount"] = filters
-
-		if len(filterListTotalDuration) != 0 {
-			for nameD, filterD := range filterListTotalDuration {
-
-				filterD["totalAdReqCount"] = filters
-
-				name := nameD + "&&" + "广告观看次数:" + totalDuration[len(totalDuration)-1] + "-" + "∞"
+		if k-1 >= 0 {
+			query := SetCakeQuery(gid, pf, createTime)
+			query.Where(filterRow, ">=", totalDuration[k-1]).Where(filterRow, "<", totalDuration[k]).Count(&count)
 
-				filterList[name] = filterD
-			}
-		} else {
-			filter1 := utils.DeepCopyMap(filter)
-			filter["totalAdReqCount"] = filters
-			name := "广告观看次数:" + totalDuration[len(totalDuration)-1] + "-" + "∞"
-
-			filterList[name] = filter1
+			group = append(group, cakeRes{
+				Name:  fmt.Sprintf("%s>=%v,且<%v", filterName, totalDuration[k-1], v),
+				Count: count,
+			})
 		}
 
-		//count, err := collection.CountDocuments(ctx, filter)
-		//if err != nil {
-		//	response.Fail(c, 1001, err.Error())
-		//	return
-		//}
-		//data = append(data, resData{
-		//	Count: int(count),
-		//	Name:  totalDuration[len(totalDuration)-1] + "-" + "∞",
-		//})
 	}
 
-	if form.TotalAdEposedCount != "" && form.TotalAdReqCount == "" {
-		totalDuration := strings.Split(strings.Replace(form.TotalAdEposedCount, "，", ",", -1), ",")
-
-		for k, _ := range totalDuration {
-			var gt, lte int
-			if k == 0 {
-				gt = 0
-				lte, _ = strconv.Atoi(totalDuration[k])
-			} else {
-				gt, _ = strconv.Atoi(totalDuration[k-1])
-				lte, _ = strconv.Atoi(totalDuration[k])
-			}
-
-			filters := bson.M{}
-			filters["$gt"] = gt
-			filters["$lte"] = lte
-
-			if len(filterListTotalDuration) != 0 {
-				for nameD, filterD := range filterListTotalDuration {
-					filterE := utils.DeepCopyMap(filterD)
-
-					filterE["TotalAdEposedCount"] = filters
-
-					name := nameD + "&&" + "广告看完次数:" + strconv.Itoa(gt) + "-" + strconv.Itoa(lte)
-
-					filterList[name] = filterE
-				}
-
-			} else {
-				filter1 := utils.DeepCopyMap(filter)
-
-				filter1["TotalAdEposedCount"] = filters
-				name := "广告看完次数:" + strconv.Itoa(gt) + "-" + strconv.Itoa(lte)
-
-				filterList[name] = filter1
-			}
-		}
-
-		filters := bson.M{}
-		filters["$gt"], _ = strconv.Atoi(totalDuration[len(totalDuration)-1])
-		filter["TotalAdEposedCount"] = filters
-
-		if len(filterListTotalDuration) != 0 {
-			for nameD, filterD := range filterListTotalDuration {
-
-				filterD["TotalAdEposedCount"] = filters
-
-				name := nameD + "&&" + "广告看完次数:" + totalDuration[len(totalDuration)-1] + "-" + "∞"
-
-				filterList[name] = filterD
-			}
-		} else {
-			filter1 := utils.DeepCopyMap(filter)
-			filter["TotalAdEposedCount"] = filters
-			name := "广告看完次数:" + totalDuration[len(totalDuration)-1] + "-" + "∞"
+	return group
+}
+func SetCakeQuery(Gid, Pf, CreateTime string) *utils.WtDB {
+	query := global.App.DB.Table("user").
+		LeftJoin("user_behavior", "user.id = user_behavior.id").Where("gid", Gid).Where("pf", Pf)
 
-			filterList[name] = filter1
-		}
+	if CreateTime != "" {
+		timeSlice := strings.Split(CreateTime, ",")
 
-		//count, err := collection.CountDocuments(ctx, filter)
-		//if err != nil {
-		//	response.Fail(c, 1001, err.Error())
-		//	return
-		//}
-		//data = append(data, resData{
-		//	Count: int(count),
-		//	Name:  totalDuration[len(totalDuration)-1] + "-" + "∞",
-		//})
-	}
+		start, _ := strconv.Atoi(timeSlice[0])
+		end, _ := strconv.Atoi(timeSlice[1])
+		startTime := time.Unix(int64(start), 0)
+		endTime := time.Unix(int64(end), 0)
 
-	if form.TotalAdEposedCount != "" && form.TotalAdReqCount != "" {
-		response.Fail(c, 1002, "筛选条件无效")
-		return
-	}
+		query = query.WhereRaw("createdAt >= ?", startTime).WhereRaw("createdAt <= ?", endTime)
 
-	if len(filterList) == 0 {
-		filterList = filterListTotalDuration
 	}
 
-	for k, filterC := range filterList {
-		count, _ := collection.CountDocuments(ctx, filterC)
+	return query
+}
+func CakeFilter(filter string) []int {
+	totalDuration := strings.Split(strings.Replace(filter, "，", ",", -1), ",")
 
-		data = append(data, resData{
-			Name:  k,
-			Count: int(count),
-		})
+	var result []int
+	for _, v := range totalDuration {
+		ints, _ := strconv.Atoi(v)
+		result = append(result, ints)
 	}
-
-	response.Success(c, gin.H{
-		"data": data,
-	})
-
+	return result
 }
 
 func SplitOnlineData(c *gin.Context) {

crons/userRelated.go

@@ -0,0 +1,80 @@
+package crons
+
+import (
+	"designs/global"
+	"designs/model"
+	"designs/service"
+	"fmt"
+	"gorm.io/gorm/clause"
+	"time"
+)
+
+func Behavior() {
+
+	//先查出所有gid
+	ActiveGid := service.GetActiveGid()
+
+	//现在直接汇总所有的数据（直到前一天的）
+	now := time.Now()
+	end := now.AddDate(0, 0, -1)
+	start, _ := time.Parse("2006-01-02 15:04:05", "2025-05-17 15:04:05")
+
+	BehaviorSummary("wx", ActiveGid, start, end)
+	BehaviorSummary("tt", ActiveGid, start, end)
+
+	global.App.Log.Info("重新汇总用户在线时长数据完成")
+}
+
+func BehaviorSummary(pf string, ActiveGid []string, start time.Time, end time.Time) {
+	//循环一下，查出userId 跟相关的在线数据
+	for _, gid := range ActiveGid {
+
+		start1 := time.Now()
+		fmt.Println("处理数据开始,gid :", gid)
+
+		//查询 30天内的在线数据
+
+		SummaryData, err := service.UserOnlineSummary(gid, pf, "", start.Format("2006-01-02 15:04:05"), end.Format("2006-01-02 15:04:05"))
+		if err != nil {
+			global.App.Log.Error("更新用户在线时长错误(查询汇总信息错误)", err.Error())
+			return
+		}
+
+		//格式化，存入数据库
+		var userIdList []model.User
+		err = global.App.DB.Table("user").Where("gid", gid).
+			Select("id", "userId").
+			Where("pf", pf).Scan(&userIdList).Error
+		if err != nil {
+			global.App.Log.Error("更新用户在线时长错误(查询userID 错误)", err.Error())
+			return
+		}
+
+		//fmt.Println(len(userIdList))
+
+		userBehavior := make(map[int]model.UserBehavior)
+		for _, users := range userIdList {
+			userBehavior[users.UserId] = model.UserBehavior{
+				ID:       users.ID,
+				Duration: int(SummaryData[users.UserId]),
+			}
+		}
+
+		var userBehaviorList []model.UserBehavior
+		for _, Behavior := range userBehavior {
+			userBehaviorList = append(userBehaviorList, Behavior)
+		}
+
+		err = global.App.DB.Table("user_behavior").Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "id"}},                  // 冲突列（主键）
+			DoUpdates: clause.AssignmentColumns([]string{"duration"}), // 需要更新的列
+		}).CreateInBatches(&userBehaviorList, 1000).Error
+		if err != nil {
+			global.App.Log.Error("更新用户在线时长错误", err.Error())
+			return
+		}
+
+		fmt.Println("存入汇总完成：", time.Since(start1))
+
+	}
+}

global/app.go

@@ -6,6 +6,7 @@ import (
 	"github.com/robfig/cron/v3"
 	"go.mongodb.org/mongo-driver/v2/mongo"
 	"go.uber.org/zap"
+	"gorm.io/gorm"
 	"io"
 )
 
@@ -21,6 +22,8 @@ type Application struct {
 
 	//数据库
 	DB *utils.WtDB
+	//clickhouse
+	Clickhouse *gorm.DB
 
 	Redis *redis.Client
 	Cron  *cron.Cron

go.mod

@@ -1,6 +1,8 @@
 module designs
 
-go 1.22.5
+go 1.23.0
+
+toolchain go1.23.3
 
 require (
 	github.com/gin-gonic/gin v1.10.0
@@ -18,17 +20,31 @@ require (
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.65
 	go.mongodb.org/mongo-driver/v2 v2.0.0-beta2
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.27.0
+	golang.org/x/crypto v0.39.0
+	gorm.io/driver/clickhouse v0.7.0
 	gorm.io/driver/mysql v1.5.7
-	gorm.io/gorm v1.25.11
+	gorm.io/gorm v1.30.0
 )
 
 require (
+	github.com/ClickHouse/ch-go v0.66.1 // indirect
+	github.com/ClickHouse/clickhouse-go/v2 v2.37.2 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/clbanning/mxj v1.8.4 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mozillazg/go-httpheader v0.4.0 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 )
 
 require (
@@ -49,7 +65,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lestrrat-go/strftime v1.0.6 // indirect
@@ -64,12 +80,12 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/sync v0.15.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"designs/bootstrap"
 	"designs/global"
 	"designs/middleware"
@@ -58,14 +57,24 @@ func main() {
 		}
 	}()
 
-	//初始化mongodb
-	global.App.MongoDB = bootstrap.InitializeMongo()
+	//初始化clickhouse
+	global.App.Clickhouse = bootstrap.InitClickhouseGorm()
+	//释放连接
 	defer func() {
-		if global.App.MongoDB != nil {
-			global.App.MongoDB.Disconnect(context.Background())
+		if global.App.Clickhouse != nil {
+			db, _ := global.App.Clickhouse.DB()
+			db.Close()
 		}
 	}()
 
+	////初始化mongodb
+	//global.App.MongoDB = bootstrap.InitializeMongo()
+	//defer func() {
+	//	if global.App.MongoDB != nil {
+	//		global.App.MongoDB.Disconnect(context.Background())
+	//	}
+	//}()
+
 	//初始化计划任务
 	bootstrap.InitializeCron()
 

route/api.go

@@ -112,6 +112,7 @@ func SetApiGroupRoutes(router *gin.RouterGroup) {
 	router.POST("/SplitOnlineData", v1.SplitOnlineData)             //迁移数据到online拆分表
 	router.POST("/InitGidConfig", v1.InitGidConfig)
 	//router.POST("/test", v1.ActionDataSummary)
-	router.POST("/test", v1.BehaviorSummary)
+	router.POST("/test", v1.BehaviorTimeSummary)
+	router.POST("/TestClickHouse", v1.TestClickHouse)
 
 }

service/actionService.go

@@ -2,14 +2,84 @@ package service
 
 import (
 	"bytes"
+	"context"
+	"designs/global"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 )
 
+func GetActiveGid() []string {
+	var activeGidList []string
+
+	key := "activeGidList"
+	activeGid, _ := global.App.Redis.Get(context.Background(), key).Result()
+
+	activeGidList = strings.Split(activeGid, ",")
+
+	if len(activeGidList) <= 2 {
+		//if true {
+		var gidList []string
+		//
+		////重新读取数据
+		//var dir string
+		//if config.Get("app.local") == "local" {
+		//	//url = "mongodb://localhost:27017"
+		//	dir = "storage"
+		//} else {
+		//	dir = "/www/wwwroot/chunhao_receive/storage"
+		//}
+		//
+		//now := time.Now()
+		//
+		//for i := 0; i <= 7; i++ {
+		//	date := now.AddDate(0, 0, -i).Format("2006-01-02")
+		//	//读取对应的文件夹
+		//	dirPath := filepath.Join(dir, date)
+		//	dateDir, _ := os.ReadDir(dirPath)
+		//	//fmt.Println(dirPath, dateDir)
+		//
+		//	for _, v := range dateDir {
+		//
+		//		fileName := v.Name()
+		//
+		//		fileNameSplit := strings.Split(fileName, "_")
+		//
+		//		if len(fileNameSplit) < 2 {
+		//			continue
+		//		}
+		//		last := fileNameSplit[len(fileNameSplit)-1]
+		//		gid := strings.TrimRight(fileName, "_"+last)
+		//
+		//		if !utils.InArray(gid, gidList) {
+		//			gidList = append(gidList, gid)
+		//		}
+		//	}
+		//}
+		//
+
+		global.App.DB.Table("user").
+			Where("createdAt", ">=", time.Now().AddDate(0, 0, -3)).
+			Distinct("gid").Pluck("gid", &gidList)
+
+		if len(gidList) > 0 {
+
+			var gidString string
+			for _, gid := range gidList {
+				gidString = gid + ","
+			}
+			global.App.Redis.Set(context.Background(), key, gidString, time.Second*3600)
+			activeGidList = gidList
+		}
+	}
+
+	return activeGidList
+}
+
 func CurPost(requestUrl string, requestBody interface{}, headerMap map[string]string) (string, error) {
 	//转换json
 	jsonBytes, err := json.Marshal(requestBody)

service/remainData.go

@@ -208,20 +208,22 @@ func RemainDataBydDayNew(types int, pf string, gid string, startTime string, end
 func OnlineToFile() {
 
 	//分批读取数据
-	now := time.Now()
-	for i := 1; i <= 45; i++ {
+	now, _ := time.Parse("20060102", "20250401")
+	for i := 0; i <= 31; i++ {
+		now1 := time.Now()
 		var onlineData []model.UserOnline
 
-		date := now.AddDate(0, 0, -i).Format("20060102")
+		date := now.AddDate(0, 0, i).Format("20060102")
 
-		global.App.DB.Table("user_online_old").Where("date", date).Scan(&onlineData)
+		global.App.DB.Table("user_online").Where("date", date).Scan(&onlineData)
+		fmt.Println(date, "数据查询完成，耗时：", time.Since(now1), date)
 
 		//根据gid和pf，分文件存储
 		for _, v := range onlineData {
 			SetLocalActiveLog(v.Gid, v.Pf, "", v.UserId, v.Type, v.LogTime)
 		}
 
-		fmt.Println(date, "数据归档完成，耗时：", time.Since(now))
+		fmt.Println(date, "数据归档完成，耗时：", time.Since(now1), date)
 	}
 
 	//var onlineData []model.UserOnline

service/userSeeAds.go

@@ -7,17 +7,19 @@ import (
 	"time"
 )
 
+type adsSummary struct {
+	Count    int    `json:"count" gorm:"column:count"`
+	Pf       string `json:"pf" gorm:"column:pf"`
+	Gid      string `json:"gid" gorm:"column:gid"`
+	SumType1 int    `json:"sumType1" gorm:"column:sumType1"`
+	SumType2 int    `json:"sumType2" gorm:"column:sumType2"`
+	SumType0 int    `json:"sumType0" gorm:"column:sumType0"`
+}
+
 // 汇总每日的广告数据
 func SeeAdsSummary(lastDay string) {
 	//计算出上一日的广告数据汇总
-	var adsSummary []struct {
-		Count    int    `json:"count" gorm:"column:count"`
-		Pf       string `json:"pf" gorm:"column:pf"`
-		Gid      string `json:"gid" gorm:"column:gid"`
-		SumType1 int    `json:"sumType1" gorm:"column:sumType1"`
-		SumType2 int    `json:"sumType2" gorm:"column:sumType2"`
-		SumType0 int    `json:"sumType0" gorm:"column:sumType0"`
-	}
+	var adsSummarys []adsSummary
 
 	err := global.App.DB.Table("user_see_ads").
 		Where("date", lastDay).
@@ -26,7 +28,7 @@ func SeeAdsSummary(lastDay string) {
 			"SUM(adsState = 1) AS sumType1",
 			"SUM(adsState = 2) AS sumType2",
 			"SUM(adsState = 0) AS sumType0").
-		Scan(&adsSummary).Error
+		Scan(&adsSummarys).Error
 
 	if err != nil {
 		global.App.Log.Error("查询广告汇总数据报错：", err)
@@ -63,7 +65,7 @@ func SeeAdsSummary(lastDay string) {
 	now := model.XTime{
 		Time: time.Now(),
 	}
-	for _, summary := range adsSummary {
+	for _, summary := range adsSummarys {
 		ds := model.SummaryUserSeeAds{
 			Date:      lastDay,
 			Count:     summary.Count,
@@ -97,3 +99,26 @@ func SeeAdsSummary(lastDay string) {
 		return
 	}
 }
+
+func GetTodayAdsSummary(gid string, pf []string) (count, todayCount int64, err error) {
+
+	today := time.Now().Format("20060102")
+	err = global.App.DB.Table("user_see_ads").
+		Where("date", today).
+		Where("gid", gid).WhereIn("pf", pf).
+		Count(&count).Error
+	if err != nil {
+		return 0, 0, err
+	}
+
+	query := global.App.DB.Table("user").Where("gid", gid).WhereIn("pf", pf).WhereRaw("DATE_FORMAT(createdAt, '%Y%m%d') = ?", today).Select("id").SubQuery()
+	err = global.App.DB.Table("user_see_ads").
+		Where("date", today).
+		LeftJoin("user", "user.pf = user_see_ads.pf and user.gid = user_see_ads.gid and user.userId = user_see_ads.userId").
+		WhereRaw("user.id in (?)", query).Count(&todayCount).Error
+	if err != nil {
+		return 0, 0, err
+	}
+
+	return count, todayCount, nil
+}