diff --git a/utils/golock/redis.go b/utils/golock/redis.go index 4f6acfb3..c3bf3bea 100644 --- a/utils/golock/redis.go +++ b/utils/golock/redis.go @@ -19,12 +19,12 @@ func NewLockRedis(client *dorm.RedisClient) *LockRedis { // key 锁名 // val 锁内容 // ttl 锁过期时间 -func (r *LockRedis) Lock(key string, val string, ttl time.Duration) (resp string, err error) { +func (r *LockRedis) Lock(ctx context.Context, key string, val string, ttl time.Duration) (resp string, err error) { if ttl <= 0 { return resp, errors.New("长期请使用 LockForever 方法") } // 获取 - get, err := r.client.Get(context.Background(), key).Result() + get, err := r.client.Get(ctx, key).Result() if err != nil { return resp, errors.New("获取异常") } @@ -32,7 +32,7 @@ func (r *LockRedis) Lock(key string, val string, ttl time.Duration) (resp string return resp, errors.New("上锁失败，已存在") } // 设置 - err = r.client.Set(context.Background(), key, val, ttl).Err() + err = r.client.Set(ctx, key, val, ttl).Err() if err != nil { return resp, errors.New("上锁失败") } @@ -41,17 +41,17 @@ func (r *LockRedis) Lock(key string, val string, ttl time.Duration) (resp string // Unlock 解锁 // key 锁名 -func (r *LockRedis) Unlock(key string) error { - _, err := r.client.Del(context.Background(), key).Result() +func (r *LockRedis) Unlock(ctx context.Context, key string) error { + _, err := r.client.Del(ctx, key).Result() return err } // LockForever 永远上锁 // key 锁名 // val 锁内容 -func (r *LockRedis) LockForever(key string, val string) (resp string, err error) { +func (r *LockRedis) LockForever(ctx context.Context, key string, val string) (resp string, err error) { // 获取 - get, err := r.client.Get(context.Background(), key).Result() + get, err := r.client.Get(ctx, key).Result() if err != nil { return resp, errors.New("获取异常") } @@ -59,7 +59,7 @@ func (r *LockRedis) LockForever(key string, val string) (resp string, err error) return resp, errors.New("上锁失败，已存在") } // 设置 - err = r.client.Set(context.Background(), key, val, 0).Err() + err = r.client.Set(ctx, key, val, 0).Err() if err != nil { return resp, errors.New("上锁失败") } diff --git a/utils/golog/api.go b/utils/golog/api.go index d0716919..23da01ab 100644 --- a/utils/golog/api.go +++ b/utils/golog/api.go @@ -5,91 +5,152 @@ import ( "errors" "github.com/dtapps/go-library/utils/dorm" "github.com/dtapps/go-library/utils/goip" - "gorm.io/gorm" - "os" - "runtime" + "github.com/dtapps/go-library/utils/gorequest" ) +// ApiClientFun *ApiClient 驱动 +type ApiClientFun func() *ApiClient + // ApiClient 接口 type ApiClient struct { - gormClient *gorm.DB // 数据库驱动 + gormClient *dorm.GormClient // 数据库驱动 mongoClient *dorm.MongoClient // 数据库驱动 + zapLog *ZapLog // 日志服务 config struct { - logType string // 日志类型 - tableName string // 表名 + systemHostname string // 主机名 + systemOs string // 系统类型 + systemVersion string // 系统版本 + systemKernel string // 系统内核 + systemKernelVersion string // 系统内核版本 + systemBootTime uint64 // 系统开机时间 + cpuCores int // CPU核数 + cpuModelName string // CPU型号名称 + cpuMhz float64 // CPU兆赫 + systemInsideIp string // 内网ip + systemOutsideIp string // 外网ip + goVersion string // go版本 + sdkVersion string // sdk版本 + mongoVersion string // mongo版本 + mongoSdkVersion string // mongo sdk版本 + } + gormConfig struct { + stats bool // 状态 + tableName string // 表名 + } + mongoConfig struct { + stats bool // 状态 databaseName string // 库名 collectionName string // 表名 - insideIp string // 内网ip - hostname string // 主机名 - goVersion string // go版本 - } // 配置 + } +} + +// ApiClientConfig 接口实例配置 +type ApiClientConfig struct { + GormClientFun dorm.GormClientTableFun // 日志配置 + MongoClientFun dorm.MongoClientCollectionFun // 日志配置 + ZapLog *ZapLog // 日志服务 + CurrentIp string // 当前ip } // NewApiClient 创建接口实例化 -// WithGormClient && WithTableName -// WithMongoCollectionClient && WithDatabaseName && WithCollectionName -func NewApiClient(attrs ...*OperationAttr) (*ApiClient, error) { +func NewApiClient(config *ApiClientConfig) (*ApiClient, error) { + + var ctx = context.Background() c := &ApiClient{} - for _, attr := range attrs { - if attr.gormClient != nil { - c.gormClient = attr.gormClient - c.config.logType = attr.logType - } - if attr.mongoClient != nil { - c.mongoClient = attr.mongoClient - c.config.logType = attr.logType - } - if attr.tableName != "" { - c.config.tableName = attr.tableName - } - if attr.databaseName != "" { - c.config.databaseName = attr.databaseName - } - if attr.collectionName != "" { - c.config.collectionName = attr.collectionName - } + + c.zapLog = config.ZapLog + + if config.CurrentIp != "" && config.CurrentIp != "0.0.0.0" { + c.config.systemOutsideIp = config.CurrentIp + } + c.config.systemOutsideIp = goip.IsIp(c.config.systemOutsideIp) + if c.config.systemOutsideIp == "" { + return nil, currentIpNoConfig } - switch c.config.logType { - case logTypeGorm: + // 配置信息 + c.setConfig(ctx) - if c.gormClient == nil { - return nil, errors.New("驱动不能为空") - } + gormClient, gormTableName := config.GormClientFun() + mongoClient, mongoDatabaseName, mongoCollectionName := config.MongoClientFun() - if c.config.tableName == "" { - return nil, errors.New("表名不能为空") - } + if (gormClient == nil || gormClient.GetDb() == nil) || (mongoClient == nil || mongoClient.GetDb() == nil) { + return nil, dbClientFunNoConfig + } - err := c.gormClient.Table(c.config.tableName).AutoMigrate(&apiPostgresqlLog{}) - if err != nil { - return nil, errors.New("创建表失败：" + err.Error()) - } + // 配置关系数据库 + if gormClient != nil || gormClient.GetDb() != nil { - case logTypeMongo: + c.gormClient = gormClient - if c.mongoClient.Db == nil { - return nil, errors.New("没有设置驱动") + if gormTableName == "" { + return nil, errors.New("没有设置表名") + } else { + c.gormConfig.tableName = gormTableName } - if c.config.databaseName == "" { + // 创建模型 + c.gormAutoMigrate(ctx) + + c.gormConfig.stats = true + } + + // 配置非关系数据库 + if mongoClient != nil || mongoClient.GetDb() != nil { + + c.mongoClient = mongoClient + + if mongoDatabaseName == "" { return nil, errors.New("没有设置库名") + } else { + c.mongoConfig.databaseName = mongoDatabaseName } - if c.config.collectionName == "" { + if mongoCollectionName == "" { return nil, errors.New("没有设置表名") + } else { + c.mongoConfig.collectionName = mongoCollectionName } - default: - return nil, errors.New("驱动为空") - } + // 创建时间序列集合 + c.mongoCreateCollection(ctx) - hostname, _ := os.Hostname() + // 创建索引 + c.mongoCreateIndexes(ctx) - c.config.hostname = hostname - c.config.insideIp = goip.GetInsideIp(context.Background()) - c.config.goVersion = runtime.Version() + c.mongoConfig.stats = true + } return c, nil } + +// Middleware 中间件 +func (c *ApiClient) Middleware(ctx context.Context, request gorequest.Response, sdkVersion string) { + if c.gormConfig.stats { + c.gormMiddleware(ctx, request, sdkVersion) + } + if c.mongoConfig.stats { + c.mongoMiddleware(ctx, request, sdkVersion) + } +} + +// MiddlewareXml 中间件 +func (c *ApiClient) MiddlewareXml(ctx context.Context, request gorequest.Response, sdkVersion string) { + if c.gormConfig.stats { + c.gormMiddlewareXml(ctx, request, sdkVersion) + } + if c.mongoConfig.stats { + c.mongoMiddlewareXml(ctx, request, sdkVersion) + } +} + +// MiddlewareCustom 中间件 +func (c *ApiClient) MiddlewareCustom(ctx context.Context, api string, request gorequest.Response, sdkVersion string) { + if c.gormConfig.stats { + c.gormMiddlewareCustom(ctx, api, request, sdkVersion) + } + if c.mongoConfig.stats { + c.mongoMiddlewareCustom(ctx, api, request, sdkVersion) + } +} diff --git a/utils/golog/api_cofing.go b/utils/golog/api_cofing.go new file mode 100644 index 00000000..bdca84ac --- /dev/null +++ b/utils/golog/api_cofing.go @@ -0,0 +1,31 @@ +package golog + +import ( + "context" + "github.com/dtapps/go-library" + "github.com/dtapps/go-library/utils/goip" + "go.mongodb.org/mongo-driver/version" + "runtime" +) + +func (c *ApiClient) setConfig(ctx context.Context) { + + info := getSystem() + + c.config.systemHostname = info.SystemHostname + c.config.systemOs = info.SystemOs + c.config.systemVersion = info.SystemVersion + c.config.systemKernel = info.SystemKernel + c.config.systemKernelVersion = info.SystemKernelVersion + c.config.systemBootTime = info.SystemBootTime + c.config.cpuCores = info.CpuCores + c.config.cpuModelName = info.CpuModelName + c.config.cpuMhz = info.CpuMhz + + c.config.systemInsideIp = goip.GetInsideIp(ctx) + + c.config.sdkVersion = go_library.Version() + c.config.goVersion = runtime.Version() + + c.config.mongoSdkVersion = version.Driver +} diff --git a/utils/golog/api_gorm.go b/utils/golog/api_gorm.go index 4483c7aa..a39c9421 100644 --- a/utils/golog/api_gorm.go +++ b/utils/golog/api_gorm.go @@ -3,127 +3,149 @@ package golog import ( "context" "github.com/dtapps/go-library/utils/dorm" - "github.com/dtapps/go-library/utils/gojson" "github.com/dtapps/go-library/utils/gorequest" + "github.com/dtapps/go-library/utils/gotime" "github.com/dtapps/go-library/utils/gotrace_id" "github.com/dtapps/go-library/utils/gourl" - "gorm.io/datatypes" - "gorm.io/gorm" - "log" "time" "unicode/utf8" ) -// 模型结构体 +// 模型 type apiPostgresqlLog struct { - LogId uint `gorm:"primaryKey;comment:【记录】编号" json:"log_id,omitempty"` //【记录】编号 - TraceId string `gorm:"index;comment:【系统】跟踪编号" json:"trace_id,omitempty"` //【系统】跟踪编号 - RequestTime time.Time `gorm:"index;comment:【请求】时间" json:"request_time,omitempty"` //【请求】时间 - RequestUri string `gorm:"comment:【请求】链接" json:"request_uri,omitempty"` //【请求】链接 - RequestUrl string `gorm:"comment:【请求】链接" json:"request_url,omitempty"` //【请求】链接 - RequestApi string `gorm:"index;comment:【请求】接口" json:"request_api,omitempty"` //【请求】接口 - RequestMethod string `gorm:"index;comment:【请求】方式" json:"request_method,omitempty"` //【请求】方式 - RequestParams datatypes.JSON `gorm:"type:jsonb;comment:【请求】参数" json:"request_params,omitempty"` //【请求】参数 - RequestHeader datatypes.JSON `gorm:"type:jsonb;comment:【请求】头部" json:"request_header,omitempty"` //【请求】头部 - ResponseHeader datatypes.JSON `gorm:"type:jsonb;comment:【返回】头部" json:"response_header,omitempty"` //【返回】头部 - ResponseStatusCode int `gorm:"index;comment:【返回】状态码" json:"response_status_code,omitempty"` //【返回】状态码 - ResponseBody datatypes.JSON `gorm:"type:jsonb;comment:【返回】内容" json:"response_body,omitempty"` //【返回】内容 - ResponseContentLength int64 `gorm:"comment:【返回】大小" json:"response_content_length,omitempty"` //【返回】大小 - ResponseTime time.Time `gorm:"index;comment:【返回】时间" json:"response_time,omitempty"` //【返回】时间 - SystemHostName string `gorm:"index;comment:【系统】主机名" json:"system_host_name,omitempty"` //【系统】主机名 - SystemInsideIp string `gorm:"index;comment:【系统】内网ip" json:"system_inside_ip,omitempty"` //【系统】内网ip - GoVersion string `gorm:"index;comment:【程序】Go版本" json:"go_version,omitempty"` //【程序】Go版本 - SdkVersion string `gorm:"index;comment:【程序】Sdk版本" json:"sdk_version,omitempty"` //【程序】Sdk版本 + LogId uint `gorm:"primaryKey;comment:【记录】编号" json:"log_id,omitempty"` //【记录】编号 + TraceId string `gorm:"index;comment:【系统】跟踪编号" json:"trace_id,omitempty"` //【系统】跟踪编号 + RequestTime time.Time `gorm:"index;comment:【请求】时间" json:"request_time,omitempty"` //【请求】时间 + RequestUri string `gorm:"comment:【请求】链接" json:"request_uri,omitempty"` //【请求】链接 + RequestUrl string `gorm:"comment:【请求】链接" json:"request_url,omitempty"` //【请求】链接 + RequestApi string `gorm:"index;comment:【请求】接口" json:"request_api,omitempty"` //【请求】接口 + RequestMethod string `gorm:"index;comment:【请求】方式" json:"request_method,omitempty"` //【请求】方式 + RequestParams string `gorm:"comment:【请求】参数" json:"request_params,omitempty"` //【请求】参数 + RequestHeader string `gorm:"comment:【请求】头部" json:"request_header,omitempty"` //【请求】头部 + RequestIp string `gorm:"default:0.0.0.0;index;comment:【请求】请求Ip" json:"request_ip,omitempty"` //【请求】请求Ip + ResponseHeader string `gorm:"comment:【返回】头部" json:"response_header,omitempty"` //【返回】头部 + ResponseStatusCode int `gorm:"index;comment:【返回】状态码" json:"response_status_code,omitempty"` //【返回】状态码 + ResponseBody string `gorm:"comment:【返回】数据" json:"response_content,omitempty"` //【返回】数据 + ResponseContentLength int64 `gorm:"comment:【返回】大小" json:"response_content_length,omitempty"` //【返回】大小 + ResponseTime time.Time `gorm:"index;comment:【返回】时间" json:"response_time,omitempty"` //【返回】时间 + SystemHostName string `gorm:"index;comment:【系统】主机名" json:"system_host_name,omitempty"` //【系统】主机名 + SystemInsideIp string `gorm:"default:0.0.0.0;comment:【系统】内网ip" json:"system_inside_ip,omitempty"` //【系统】内网ip + SystemOs string `gorm:"index;comment:【系统】系统类型" json:"system_os,omitempty"` //【系统】系统类型 + SystemArch string `gorm:"index;comment:【系统】系统架构" json:"system_arch,omitempty"` //【系统】系统架构 + GoVersion string `gorm:"comment:【程序】Go版本" json:"go_version,omitempty"` //【程序】Go版本 + SdkVersion string `gorm:"comment:【程序】Sdk版本" json:"sdk_version,omitempty"` //【程序】Sdk版本 } -// 记录日志 -func (c *ApiClient) gormRecord(ctx context.Context, postgresqlLog apiPostgresqlLog) error { - - if utf8.ValidString(string(postgresqlLog.ResponseBody)) == false { - postgresqlLog.ResponseBody = datatypes.JSON("") +// 创建模型 +func (c *ApiClient) gormAutoMigrate(ctx context.Context) { + err := c.gormClient.GetDb().Table(c.gormConfig.tableName).AutoMigrate(&apiPostgresqlLog{}) + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("创建模型：%s", err) } +} - postgresqlLog.SystemHostName = c.config.hostname - if postgresqlLog.SystemInsideIp == "" { - postgresqlLog.SystemInsideIp = c.config.insideIp +// 记录日志 +func (c *ApiClient) gormRecord(ctx context.Context, data apiPostgresqlLog) { + + if utf8.ValidString(data.ResponseBody) == false { + data.ResponseBody = "" } - postgresqlLog.GoVersion = c.config.goVersion - postgresqlLog.TraceId = gotrace_id.GetTraceIdContext(ctx) + data.SystemHostName = c.config.systemHostname //【系统】主机名 + data.SystemInsideIp = c.config.systemInsideIp //【系统】内网ip + data.GoVersion = c.config.goVersion //【程序】Go版本 + data.TraceId = gotrace_id.GetTraceIdContext(ctx) //【记录】跟踪编号 + data.RequestIp = c.config.systemOutsideIp //【请求】请求Ip + data.SystemOs = c.config.systemOs //【系统】系统类型 + data.SystemArch = c.config.systemKernel //【系统】系统架构 - return c.gormClient.Table(c.config.tableName).Create(&postgresqlLog).Error + err := c.gormClient.GetDb().Table(c.gormConfig.tableName).Create(&data).Error + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("记录接口日志错误：%s", err) + c.zapLog.WithTraceId(ctx).Sugar().Errorf("记录接口日志数据：%+v", data) + } } -// GormQuery 查询 -func (c *ApiClient) GormQuery() *gorm.DB { - return c.gormClient.Table(c.config.tableName) +// GormDelete 删除 +func (c *ApiClient) GormDelete(ctx context.Context, hour int64) error { + err := c.gormClient.GetDb().Table(c.gormConfig.tableName).Where("request_time < ?", gotime.Current().BeforeHour(hour).Format()).Delete(&apiPostgresqlLog{}).Error + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("删除失败：%s", err) + } + return err } -// GormMiddleware 中间件 -func (c *ApiClient) GormMiddleware(ctx context.Context, request gorequest.Response, sdkVersion string) { - if request.ResponseHeader.Get("Content-Type") == "image/jpeg" || request.ResponseHeader.Get("Content-Type") == "image/png" { - return +// 中间件 +func (c *ApiClient) gormMiddleware(ctx context.Context, request gorequest.Response, sdkVersion string) { + data := apiPostgresqlLog{ + RequestTime: request.RequestTime, //【请求】时间 + RequestUri: request.RequestUri, //【请求】链接 + RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 + RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 + RequestMethod: request.RequestMethod, //【请求】方式 + RequestParams: dorm.JsonEncodeNoError(request.RequestParams), //【请求】参数 + RequestHeader: dorm.JsonEncodeNoError(request.RequestHeader), //【请求】头部 + ResponseHeader: dorm.JsonEncodeNoError(request.ResponseHeader), //【返回】头部 + ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 + ResponseContentLength: request.ResponseContentLength, //【返回】大小 + ResponseTime: request.ResponseTime, //【返回】时间 + SdkVersion: sdkVersion, //【程序】Sdk版本 } - err := c.gormRecord(ctx, apiPostgresqlLog{ - RequestTime: request.RequestTime, //【请求】时间 - RequestUri: request.RequestUri, //【请求】链接 - RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 - RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 - RequestMethod: request.RequestMethod, //【请求】方式 - RequestParams: datatypes.JSON(gojson.JsonEncodeNoError(request.RequestParams)), //【请求】参数 - RequestHeader: datatypes.JSON(gojson.JsonEncodeNoError(request.RequestHeader)), //【请求】头部 - ResponseHeader: datatypes.JSON(gojson.JsonEncodeNoError(request.ResponseHeader)), //【返回】头部 - ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 - ResponseBody: request.ResponseBody, //【返回】内容 - ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: request.ResponseTime, //【返回】时间 - SdkVersion: sdkVersion, //【程序】Sdk版本 - }) - if err != nil { - log.Println("log.GormMiddleware：", err.Error()) + if !request.HeaderIsImg() { + if len(request.ResponseBody) > 0 { + data.ResponseBody = dorm.JsonEncodeNoError(dorm.JsonDecodeNoError(request.ResponseBody)) //【返回】数据 + } } + + c.gormRecord(ctx, data) } -// GormMiddlewareXml 中间件 -func (c *ApiClient) GormMiddlewareXml(ctx context.Context, request gorequest.Response, sdkVersion string) { - err := c.gormRecord(ctx, apiPostgresqlLog{ - RequestTime: request.RequestTime, //【请求】时间 - RequestUri: request.RequestUri, //【请求】链接 - RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 - RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 - RequestMethod: request.RequestMethod, //【请求】方式 - RequestParams: datatypes.JSON(gojson.JsonEncodeNoError(request.RequestParams)), //【请求】参数 - RequestHeader: datatypes.JSON(gojson.JsonEncodeNoError(request.RequestHeader)), //【请求】头部 - ResponseHeader: datatypes.JSON(gojson.JsonEncodeNoError(request.ResponseHeader)), //【返回】头部 - ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 - ResponseBody: datatypes.JSON(gojson.JsonEncodeNoError(dorm.XmlDecodeNoError(request.ResponseBody))), //【返回】内容 - ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: request.ResponseTime, //【返回】时间 - SdkVersion: sdkVersion, //【程序】Sdk版本 - }) - if err != nil { - log.Println("log.GormMiddlewareXml：", err.Error()) +// 中间件 +func (c *ApiClient) gormMiddlewareXml(ctx context.Context, request gorequest.Response, sdkVersion string) { + data := apiPostgresqlLog{ + RequestTime: request.RequestTime, //【请求】时间 + RequestUri: request.RequestUri, //【请求】链接 + RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 + RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 + RequestMethod: request.RequestMethod, //【请求】方式 + RequestParams: dorm.JsonEncodeNoError(request.RequestParams), //【请求】参数 + RequestHeader: dorm.JsonEncodeNoError(request.RequestHeader), //【请求】头部 + ResponseHeader: dorm.JsonEncodeNoError(request.ResponseHeader), //【返回】头部 + ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 + ResponseContentLength: request.ResponseContentLength, //【返回】大小 + ResponseTime: request.ResponseTime, //【返回】时间 + SdkVersion: sdkVersion, //【程序】Sdk版本 + } + if !request.HeaderIsImg() { + if len(request.ResponseBody) > 0 { + data.ResponseBody = dorm.JsonEncodeNoError(request.ResponseBody) //【返回】内容 + } } + + c.gormRecord(ctx, data) } -// GormMiddlewareCustom 中间件 -func (c *ApiClient) GormMiddlewareCustom(ctx context.Context, api string, request gorequest.Response, sdkVersion string) { - err := c.gormRecord(ctx, apiPostgresqlLog{ - RequestTime: request.RequestTime, //【请求】时间 - RequestUri: request.RequestUri, //【请求】链接 - RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 - RequestApi: api, //【请求】接口 - RequestMethod: request.RequestMethod, //【请求】方式 - RequestParams: datatypes.JSON(gojson.JsonEncodeNoError(request.RequestParams)), //【请求】参数 - RequestHeader: datatypes.JSON(gojson.JsonEncodeNoError(request.RequestHeader)), //【请求】头部 - ResponseHeader: datatypes.JSON(gojson.JsonEncodeNoError(request.ResponseHeader)), //【返回】头部 - ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 - ResponseBody: request.ResponseBody, //【返回】内容 - ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: request.ResponseTime, //【返回】时间 - SdkVersion: sdkVersion, //【程序】Sdk版本 - }) - if err != nil { - log.Println("log.GormMiddlewareCustom：", err.Error()) +// 中间件 +func (c *ApiClient) gormMiddlewareCustom(ctx context.Context, api string, request gorequest.Response, sdkVersion string) { + data := apiPostgresqlLog{ + RequestTime: request.RequestTime, //【请求】时间 + RequestUri: request.RequestUri, //【请求】链接 + RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 + RequestApi: api, //【请求】接口 + RequestMethod: request.RequestMethod, //【请求】方式 + RequestParams: dorm.JsonEncodeNoError(request.RequestParams), //【请求】参数 + RequestHeader: dorm.JsonEncodeNoError(request.RequestHeader), //【请求】头部 + ResponseHeader: dorm.JsonEncodeNoError(request.ResponseHeader), //【返回】头部 + ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 + ResponseContentLength: request.ResponseContentLength, //【返回】大小 + ResponseTime: request.ResponseTime, //【返回】时间 + SdkVersion: sdkVersion, //【程序】Sdk版本 } + if !request.HeaderIsImg() { + if len(request.ResponseBody) > 0 { + data.ResponseBody = dorm.JsonEncodeNoError(dorm.JsonDecodeNoError(request.ResponseBody)) //【返回】数据 + } + } + + c.gormRecord(ctx, data) } diff --git a/utils/golog/api_mongo.go b/utils/golog/api_mongo.go index ff096244..88eaefe1 100644 --- a/utils/golog/api_mongo.go +++ b/utils/golog/api_mongo.go @@ -7,107 +7,175 @@ import ( "github.com/dtapps/go-library/utils/gotime" "github.com/dtapps/go-library/utils/gotrace_id" "github.com/dtapps/go-library/utils/gourl" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" ) // 模型结构体 -type apiMongoLog struct { - LogId primitive.ObjectID `json:"log_id,omitempty" bson:"_id,omitempty"` //【记录】编号 - TraceId string `json:"trace_id,omitempty" bson:"trace_id,omitempty"` //【系统】跟踪编号 - RequestTime int64 `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 - RequestUri string `json:"request_uri,omitempty" bson:"request_uri,omitempty"` //【请求】链接 - RequestUrl string `json:"request_url,omitempty" bson:"request_url,omitempty"` //【请求】链接 - RequestApi string `json:"request_api,omitempty" bson:"request_api,omitempty"` //【请求】接口 - RequestMethod string `json:"request_method,omitempty" bson:"request_method,omitempty"` //【请求】方式 - RequestParams map[string]interface{} `json:"request_params,omitempty" bson:"request_params,omitempty"` //【请求】参数 - RequestHeader map[string]string `json:"request_header,omitempty" bson:"request_header,omitempty"` //【请求】头部 - ResponseHeader map[string][]string `json:"response_header,omitempty" bson:"response_header,omitempty"` //【返回】头部 - ResponseStatusCode int `json:"response_status_code,omitempty" bson:"response_status_code,omitempty"` //【返回】状态码 - ResponseBody interface{} `json:"response_body,omitempty" bson:"response_body,omitempty"` //【返回】内容 - ResponseContentLength int64 `json:"response_content_length,omitempty" bson:"response_content_length,omitempty"` //【返回】大小 - ResponseTime int64 `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 - SystemHostName string `json:"system_host_name,omitempty" bson:"system_host_name,omitempty"` //【系统】主机名 - SystemInsideIp string `json:"system_inside_ip,omitempty" bson:"system_inside_ip,omitempty"` //【系统】内网ip - GoVersion string `json:"go_version,omitempty" bson:"go_version,omitempty"` //【程序】Go版本 - SdkVersion string `json:"sdk_version,omitempty" bson:"sdk_version,omitempty"` //【程序】Sdk版本 +type apiMongolLog struct { + LogId primitive.ObjectID `json:"log_id,omitempty" bson:"_id,omitempty"` //【记录】编号 + LogTime primitive.DateTime `json:"log_time,omitempty" bson:"log_time"` //【记录】时间 + TraceId string `json:"trace_id,omitempty" bson:"trace_id,omitempty"` //【记录】跟踪编号 + RequestTime string `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 + RequestUri string `json:"request_uri,omitempty" bson:"request_uri,omitempty"` //【请求】链接 + RequestUrl string `json:"request_url,omitempty" bson:"request_url,omitempty"` //【请求】链接 + RequestApi string `json:"request_api,omitempty" bson:"request_api,omitempty"` //【请求】接口 + RequestMethod string `json:"request_method,omitempty" bson:"request_method,omitempty"` //【请求】方式 + RequestParams interface{} `json:"request_params,omitempty" bson:"request_params,omitempty"` //【请求】参数 + RequestHeader interface{} `json:"request_header,omitempty" bson:"request_header,omitempty"` //【请求】头部 + ResponseHeader interface{} `json:"response_header,omitempty" bson:"response_header,omitempty"` //【返回】头部 + ResponseStatusCode int `json:"response_status_code,omitempty" bson:"response_status_code,omitempty"` //【返回】状态码 + ResponseBody interface{} `json:"response_body,omitempty" bson:"response_body,omitempty"` //【返回】内容 + ResponseContentLength int64 `json:"response_content_length,omitempty" bson:"response_content_length,omitempty"` //【返回】大小 + ResponseTime string `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 + System struct { + Hostname string `json:"hostname" bson:"hostname"` //【系统】主机名 + Os string `json:"os" bson:"os"` //【系统】系统类型 + Version string `json:"version" bson:"version"` //【系统】系统版本 + Kernel string `json:"kernel" bson:"kernel"` //【系统】系统内核 + KernelVersion string `json:"kernel_version" bson:"kernel_version"` //【系统】系统内核版本 + BootTime string `json:"boot_time" bson:"boot_time"` //【系统】系统开机时间 + CpuCores int `json:"cpu_cores,omitempty" bson:"cpu_cores,omitempty"` //【系统】CPU核数 + CpuModelName string `json:"cpu_model_name,omitempty" bson:"cpu_model_name,omitempty"` //【系统】CPU型号名称 + CpuMhz float64 `json:"cpu_mhz,omitempty" bson:"cpu_mhz,omitempty"` //【系统】CPU兆赫 + InsideIp string `json:"inside_ip" bson:"inside_ip"` //【系统】内网ip + OutsideIp string `json:"outside_ip" bson:"outside_ip"` //【系统】外网ip + GoVersion string `json:"go_version,omitempty" bson:"go_version,omitempty"` //【系统】go版本 + SdkVersion string `json:"sdk_version,omitempty" bson:"sdk_version,omitempty"` //【系统】sdk版本 + MongoVersion string `json:"mongo_version,omitempty" bson:"mongo_version,omitempty"` //【系统】mongo版本 + MongoSdkVersion string `json:"mongo_sdk_version,omitempty" bson:"mongo_sdk_version,omitempty"` //【系统】mongo sdk版本 + } `json:"system,omitempty" bson:"system,omitempty"` //【系统】信息 } -// 记录日志 -func (c *ApiClient) mongoRecord(ctx context.Context, mongoLog apiMongoLog) error { +// 创建时间序列集合 +func (c *ApiClient) mongoCreateCollection(ctx context.Context) { + err := c.mongoClient.Database(c.mongoConfig.databaseName).CreateCollection(ctx, c.mongoConfig.collectionName, options.CreateCollection().SetTimeSeriesOptions(options.TimeSeries().SetTimeField("log_time"))) + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("创建时间序列集合：%s", err) + } +} - mongoLog.SystemHostName = c.config.hostname - if mongoLog.SystemInsideIp == "" { - mongoLog.SystemInsideIp = c.config.insideIp +// 创建索引 +func (c *ApiClient) mongoCreateIndexes(ctx context.Context) { + _, err := c.mongoClient.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).CreateManyIndexes(ctx, []mongo.IndexModel{ + { + Keys: bson.D{{ + Key: "log_time", + Value: -1, + }}, + }}) + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("创建索引：%s", err) } - mongoLog.GoVersion = c.config.goVersion +} - mongoLog.TraceId = gotrace_id.GetTraceIdContext(ctx) +// MongoDelete 删除 +func (c *ApiClient) MongoDelete(ctx context.Context, hour int64) (*mongo.DeleteResult, error) { + filter := bson.D{{"log_time", bson.D{{"$lt", primitive.NewDateTimeFromTime(gotime.Current().BeforeHour(hour).Time)}}}} + return c.mongoClient.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).DeleteMany(ctx, filter) +} - mongoLog.LogId = primitive.NewObjectID() +// 记录日志 +func (c *ApiClient) mongoRecord(ctx context.Context, data apiMongolLog, sdkVersion string) { - _, err := c.mongoClient.Database(c.config.databaseName).Collection(c.config.collectionName).InsertOne(mongoLog) + data.LogId = primitive.NewObjectID() //【记录】编号 + data.TraceId = gotrace_id.GetTraceIdContext(ctx) //【记录】跟踪编号 + data.System.Hostname = c.config.systemHostname //【系统】主机名 + data.System.Os = c.config.systemOs //【系统】系统类型 + data.System.Version = c.config.systemVersion //【系统】系统版本 + data.System.Kernel = c.config.systemKernel //【系统】系统内核 + data.System.KernelVersion = c.config.systemKernelVersion //【系统】系统内核版本 + data.System.BootTime = gotime.SetCurrent(gotime.SetCurrentUnix(int64(c.config.systemBootTime)).Time).Format() //【系统】系统开机时间 + data.System.CpuCores = c.config.cpuCores //【系统】CPU核数 + data.System.CpuModelName = c.config.cpuModelName //【系统】CPU型号名称 + data.System.CpuMhz = c.config.cpuMhz //【程序】CPU兆赫 + data.System.InsideIp = c.config.systemInsideIp //【系统】内网ip + data.System.OutsideIp = c.config.systemOutsideIp //【系统】外网ip + data.System.GoVersion = c.config.goVersion //【系统】Go版本 + data.System.SdkVersion = sdkVersion //【系统】Sdk版本 + data.System.MongoVersion = c.config.mongoVersion //【系统】mongo版本 + data.System.MongoSdkVersion = c.config.mongoSdkVersion //【系统】mongo sdk版本 - return err + _, err := c.mongoClient.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).InsertOne(ctx, data) + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("保存接口日志错误：%s", err) + c.zapLog.WithTraceId(ctx).Sugar().Errorf("保存接口日志数据：%+v", data) + } } -// MongoQuery 查询 -func (c *ApiClient) MongoQuery() *dorm.MongoClient { - return c.mongoClient.Database(c.config.databaseName).Collection(c.config.collectionName) -} +// 中间件 +func (c *ApiClient) mongoMiddleware(ctx context.Context, request gorequest.Response, sdkVersion string) { + data := apiMongolLog{ + LogTime: primitive.NewDateTimeFromTime(request.RequestTime), //【记录】时间 + RequestTime: gotime.SetCurrent(request.RequestTime).Format(), //【请求】时间 + RequestUri: request.RequestUri, //【请求】链接 + RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 + RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 + RequestMethod: request.RequestMethod, //【请求】方式 + RequestParams: request.RequestParams, //【请求】参数 + RequestHeader: request.RequestHeader, //【请求】头部 + ResponseHeader: request.ResponseHeader, //【返回】头部 + ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 + ResponseContentLength: request.ResponseContentLength, //【返回】大小 + ResponseTime: gotime.SetCurrent(request.ResponseTime).Format(), //【返回】时间 + } + if !request.HeaderIsImg() { + if len(request.ResponseBody) > 0 { + data.ResponseBody = dorm.JsonDecodeNoError(request.ResponseBody) //【返回】内容 + } + } -// MongoMiddleware 中间件 -func (c *ApiClient) MongoMiddleware(ctx context.Context, request gorequest.Response, sdkVersion string) { - c.mongoRecord(ctx, apiMongoLog{ - RequestTime: gotime.SetCurrent(request.RequestTime).Timestamp(), //【请求】时间 - RequestUri: request.RequestUri, //【请求】链接 - RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 - RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 - RequestMethod: request.RequestMethod, //【请求】方式 - RequestParams: request.RequestParams, //【请求】参数 - RequestHeader: request.RequestHeader, //【请求】头部 - ResponseHeader: request.ResponseHeader, //【返回】头部 - ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 - ResponseBody: request.ResponseBody, //【返回】内容 - ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: gotime.SetCurrent(request.ResponseTime).Timestamp(), //【返回】时间 - SdkVersion: sdkVersion, //【程序】Sdk版本 - }) + c.mongoRecord(ctx, data, sdkVersion) } -// MongoMiddlewareXml 中间件 -func (c *ApiClient) MongoMiddlewareXml(ctx context.Context, request gorequest.Response, sdkVersion string) { - c.mongoRecord(ctx, apiMongoLog{ - RequestTime: gotime.SetCurrent(request.RequestTime).Timestamp(), //【请求】时间 - RequestUri: request.RequestUri, //【请求】链接 - RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 - RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 - RequestMethod: request.RequestMethod, //【请求】方式 - RequestParams: request.RequestParams, //【请求】参数 - RequestHeader: request.RequestHeader, //【请求】头部 - ResponseHeader: request.ResponseHeader, //【返回】头部 - ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 - ResponseBody: dorm.XmlDecodeNoError(request.ResponseBody), //【返回】内容 - ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: gotime.SetCurrent(request.ResponseTime).Timestamp(), //【返回】时间 - SdkVersion: sdkVersion, //【程序】Sdk版本 - }) +// 中间件 +func (c *ApiClient) mongoMiddlewareXml(ctx context.Context, request gorequest.Response, sdkVersion string) { + data := apiMongolLog{ + LogTime: primitive.NewDateTimeFromTime(request.RequestTime), //【记录】时间 + RequestTime: gotime.SetCurrent(request.RequestTime).Format(), //【请求】时间 + RequestUri: request.RequestUri, //【请求】链接 + RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 + RequestApi: gourl.UriParse(request.RequestUri).Path, //【请求】接口 + RequestMethod: request.RequestMethod, //【请求】方式 + RequestParams: request.RequestParams, //【请求】参数 + RequestHeader: request.RequestHeader, //【请求】头部 + ResponseHeader: request.ResponseHeader, //【返回】头部 + ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 + ResponseContentLength: request.ResponseContentLength, //【返回】大小 + ResponseTime: gotime.SetCurrent(request.ResponseTime).Format(), //【返回】时间 + } + if !request.HeaderIsImg() { + if len(request.ResponseBody) > 0 { + data.ResponseBody = dorm.XmlDecodeNoError(request.ResponseBody) //【返回】内容 + } + } + + c.mongoRecord(ctx, data, sdkVersion) } -// MongoMiddlewareCustom 中间件 -func (c *ApiClient) MongoMiddlewareCustom(ctx context.Context, api string, request gorequest.Response, sdkVersion string) { - c.mongoRecord(ctx, apiMongoLog{ - RequestTime: gotime.SetCurrent(request.RequestTime).Timestamp(), //【请求】时间 - RequestUri: request.RequestUri, //【请求】链接 - RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 - RequestApi: api, //【请求】接口 - RequestMethod: request.RequestMethod, //【请求】方式 - RequestParams: request.RequestParams, //【请求】参数 - RequestHeader: request.RequestHeader, //【请求】头部 - ResponseHeader: request.ResponseHeader, //【返回】头部 - ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 - ResponseBody: request.ResponseBody, //【返回】内容 - ResponseContentLength: request.ResponseContentLength, //【返回】大小 - ResponseTime: gotime.SetCurrent(request.ResponseTime).Timestamp(), //【返回】时间 - SdkVersion: sdkVersion, //【程序】Sdk版本 - }) +// 中间件 +func (c *ApiClient) mongoMiddlewareCustom(ctx context.Context, api string, request gorequest.Response, sdkVersion string) { + data := apiMongolLog{ + LogTime: primitive.NewDateTimeFromTime(request.RequestTime), //【记录】时间 + RequestTime: gotime.SetCurrent(request.RequestTime).Format(), //【请求】时间 + RequestUri: request.RequestUri, //【请求】链接 + RequestUrl: gourl.UriParse(request.RequestUri).Url, //【请求】链接 + RequestApi: api, //【请求】接口 + RequestMethod: request.RequestMethod, //【请求】方式 + RequestParams: request.RequestParams, //【请求】参数 + RequestHeader: request.RequestHeader, //【请求】头部 + ResponseHeader: request.ResponseHeader, //【返回】头部 + ResponseStatusCode: request.ResponseStatusCode, //【返回】状态码 + ResponseContentLength: request.ResponseContentLength, //【返回】大小 + ResponseTime: gotime.SetCurrent(request.ResponseTime).Format(), //【返回】时间 + } + if !request.HeaderIsImg() { + if len(request.ResponseBody) > 0 { + data.ResponseBody = dorm.JsonDecodeNoError(request.ResponseBody) //【返回】内容 + } + } + + c.mongoRecord(ctx, data, sdkVersion) } diff --git a/utils/golog/config.go b/utils/golog/config.go new file mode 100644 index 00000000..15c69a31 --- /dev/null +++ b/utils/golog/config.go @@ -0,0 +1,56 @@ +package golog + +import ( + "github.com/shirou/gopsutil/cpu" + "github.com/shirou/gopsutil/host" + "log" +) + +type systemResult struct { + SystemHostname string // 主机名 + SystemOs string // 系统类型 + SystemVersion string // 系统版本 + SystemKernel string // 系统内核 + SystemKernelVersion string // 系统内核版本 + SystemUpTime uint64 // 系统运行时间 + SystemBootTime uint64 // 系统开机时间 + CpuCores int // CPU核数 + CpuModelName string // CPU型号名称 + CpuMhz float64 // CPU兆赫 +} + +func getSystem() (result systemResult) { + + hInfo, err := host.Info() + if err != nil { + log.Printf("getSystem.host.Info：%s\n", err) + } + + result.SystemHostname = hInfo.Hostname + result.SystemOs = hInfo.OS + result.SystemVersion = hInfo.PlatformVersion + result.SystemKernel = hInfo.KernelArch + result.SystemKernelVersion = hInfo.KernelVersion + result.SystemUpTime = hInfo.Uptime + if hInfo.BootTime != 0 { + result.SystemBootTime = hInfo.BootTime + } + + hCpu, err := cpu.Times(true) + if err != nil { + log.Printf("getSystem.cpu.Times：%s\n", err) + } + + result.CpuCores = len(hCpu) + + cInfo, err := cpu.Info() + if err != nil { + log.Printf("getSystem.cpu.Info：%s\n", err) + } + if len(cInfo) > 0 { + result.CpuModelName = cInfo[0].ModelName + result.CpuMhz = cInfo[0].Mhz + } + + return result +} diff --git a/utils/golog/error.go b/utils/golog/error.go new file mode 100644 index 00000000..85f4fdae --- /dev/null +++ b/utils/golog/error.go @@ -0,0 +1,8 @@ +package golog + +import "errors" + +var ( + currentIpNoConfig = errors.New("请配置 CurrentIp") + dbClientFunNoConfig = errors.New("请配置 GormClientFun 或 MongoClientFun") +) diff --git a/utils/golog/gin.go b/utils/golog/gin.go index 70c7b935..792dc2fc 100644 --- a/utils/golog/gin.go +++ b/utils/golog/gin.go @@ -7,96 +7,129 @@ import ( "errors" "github.com/dtapps/go-library/utils/dorm" "github.com/dtapps/go-library/utils/goip" + "github.com/dtapps/go-library/utils/gorequest" + "github.com/dtapps/go-library/utils/gotime" + "github.com/dtapps/go-library/utils/gotrace_id" "github.com/gin-gonic/gin" - "gorm.io/gorm" - "os" - "runtime" + "io/ioutil" ) +// GinClientFun *GinClient 驱动 +type GinClientFun func() *GinClient + // GinClient 框架 type GinClient struct { - gormClient *gorm.DB // 数据库驱动 + gormClient *dorm.GormClient // 数据库驱动 mongoClient *dorm.MongoClient // 数据库驱动 ipService *goip.Client // ip服务 + zapLog *ZapLog // 日志服务 config struct { - logType string // 日志类型 - tableName string // 表名 + systemHostname string // 主机名 + systemOs string // 系统类型 + systemVersion string // 系统版本 + systemKernel string // 系统内核 + systemKernelVersion string // 系统内核版本 + systemBootTime uint64 // 系统开机时间 + cpuCores int // CPU核数 + cpuModelName string // CPU型号名称 + cpuMhz float64 // CPU兆赫 + systemInsideIp string // 内网ip + systemOutsideIp string // 外网ip + goVersion string // go版本 + sdkVersion string // sdk版本 + mongoVersion string // mongo版本 + mongoSdkVersion string // mongo sdk版本 + } + gormConfig struct { + stats bool // 状态 + tableName string // 表名 + } + mongoConfig struct { + stats bool // 状态 databaseName string // 库名 collectionName string // 表名 - insideIp string // 内网ip - hostname string // 主机名 - goVersion string // go版本 - } // 配置 + } +} + +// GinClientConfig 框架实例配置 +type GinClientConfig struct { + IpService *goip.Client // ip服务 + GormClientFun dorm.GormClientTableFun // 日志配置 + MongoClientFun dorm.MongoClientCollectionFun // 日志配置 + ZapLog *ZapLog // 日志服务 + CurrentIp string // 当前ip } // NewGinClient 创建框架实例化 -// WithGormClient && WithTableName -// WithMongoCollectionClient && WithDatabaseName && WithCollectionName -func NewGinClient(attrs ...*OperationAttr) (*GinClient, error) { +func NewGinClient(config *GinClientConfig) (*GinClient, error) { + + var ctx = context.Background() c := &GinClient{} - for _, attr := range attrs { - if attr.gormClient != nil { - c.gormClient = attr.gormClient - c.config.logType = attr.logType - } - if attr.mongoClient != nil { - c.mongoClient = attr.mongoClient - c.config.logType = attr.logType - } - if attr.tableName != "" { - c.config.tableName = attr.tableName - } - if attr.databaseName != "" { - c.config.databaseName = attr.databaseName - } - if attr.collectionName != "" { - c.config.collectionName = attr.collectionName - } - if attr.ipService != nil { - c.ipService = attr.ipService - } + + c.zapLog = config.ZapLog + + if config.CurrentIp != "" && config.CurrentIp != "0.0.0.0" { + c.config.systemOutsideIp = config.CurrentIp + } + c.config.systemOutsideIp = goip.IsIp(c.config.systemOutsideIp) + if c.config.systemOutsideIp == "" { + return nil, currentIpNoConfig } - switch c.config.logType { - case logTypeGorm: + c.ipService = config.IpService - if c.gormClient == nil { - return nil, errors.New("没有设置驱动") - } + // 配置信息 + c.setConfig(ctx) + + gormClient, gormTableName := config.GormClientFun() + mongoClient, mongoDatabaseName, mongoCollectionName := config.MongoClientFun() + + if (gormClient == nil || gormClient.GetDb() == nil) || (mongoClient == nil || mongoClient.GetDb() == nil) { + return nil, dbClientFunNoConfig + } - if c.config.tableName == "" { + // 配置关系数据库 + if gormClient != nil || gormClient.GetDb() != nil { + + c.gormClient = gormClient + + if gormTableName == "" { return nil, errors.New("没有设置表名") + } else { + c.gormConfig.tableName = gormTableName } - err := c.gormClient.Table(c.config.tableName).AutoMigrate(&ginPostgresqlLog{}) - if err != nil { - return nil, errors.New("创建表失败：" + err.Error()) - } + c.gormAutoMigrate(ctx) - case logTypeMongo: + c.gormConfig.stats = true + } - if c.mongoClient.Db == nil { - return nil, errors.New("没有设置驱动") - } + // 配置非关系数据库 + if mongoClient != nil || mongoClient.GetDb() != nil { + + c.mongoClient = mongoClient - if c.config.databaseName == "" { + if mongoDatabaseName == "" { return nil, errors.New("没有设置库名") + } else { + c.mongoConfig.databaseName = mongoDatabaseName } - if c.config.collectionName == "" { + if mongoCollectionName == "" { return nil, errors.New("没有设置表名") + } else { + c.mongoConfig.collectionName = mongoCollectionName } - default: - return nil, errors.New("驱动为空") - } + // 创建时间序列集合 + //c.mongoCreateCollection(ctx) - hostname, _ := os.Hostname() + // 创建索引 + c.mongoCreateIndexes(ctx) - c.config.hostname = hostname - c.config.insideIp = goip.GetInsideIp(context.Background()) - c.config.goVersion = runtime.Version() + c.mongoConfig.stats = true + } return c, nil } @@ -120,3 +153,73 @@ func (c *GinClient) jsonUnmarshal(data string) (result interface{}) { _ = json.Unmarshal([]byte(data), &result) return } + +// Middleware 中间件 +func (c *GinClient) Middleware() gin.HandlerFunc { + return func(ginCtx *gin.Context) { + + // 开始时间 + startTime := gotime.Current().TimestampWithMillisecond() + requestTime := gotime.Current().Time + + // 获取 + data, _ := ioutil.ReadAll(ginCtx.Request.Body) + + // 复用 + ginCtx.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + blw := &bodyLogWriter{body: bytes.NewBufferString(""), ResponseWriter: ginCtx.Writer} + ginCtx.Writer = blw + + // 处理请求 + ginCtx.Next() + + // 响应 + responseCode := ginCtx.Writer.Status() + responseBody := blw.body.String() + + //结束时间 + endTime := gotime.Current().TimestampWithMillisecond() + + go func() { + + var dataJson = true + + // 解析请求内容 + var jsonBody map[string]interface{} + + // 判断是否有内容 + if len(data) > 0 { + err := json.Unmarshal(data, &jsonBody) + if err != nil { + dataJson = false + } + } + + clientIp := gorequest.ClientIp(ginCtx.Request) + var info = goip.AnalyseResult{} + + if c.ipService != nil { + info = c.ipService.Analyse(clientIp) + } + + var traceId = gotrace_id.GetGinTraceId(ginCtx) + + // 记录 + if c.gormConfig.stats { + if dataJson { + c.gormRecordJson(ginCtx, traceId, requestTime, data, responseCode, responseBody, startTime, endTime, info) + } else { + c.gormRecordXml(ginCtx, traceId, requestTime, data, responseCode, responseBody, startTime, endTime, info) + } + } + if c.mongoConfig.stats { + if dataJson { + c.mongoRecordJson(ginCtx, traceId, requestTime, data, responseCode, responseBody, startTime, endTime, info) + } else { + c.mongoRecordXml(ginCtx, traceId, requestTime, data, responseCode, responseBody, startTime, endTime, info) + } + } + }() + } +} diff --git a/utils/golog/gin_cofing.go b/utils/golog/gin_cofing.go new file mode 100644 index 00000000..8401f899 --- /dev/null +++ b/utils/golog/gin_cofing.go @@ -0,0 +1,31 @@ +package golog + +import ( + "context" + "github.com/dtapps/go-library" + "github.com/dtapps/go-library/utils/goip" + "go.mongodb.org/mongo-driver/version" + "runtime" +) + +func (c *GinClient) setConfig(ctx context.Context) { + + info := getSystem() + + c.config.systemHostname = info.SystemHostname + c.config.systemOs = info.SystemOs + c.config.systemVersion = info.SystemVersion + c.config.systemKernel = info.SystemKernel + c.config.systemKernelVersion = info.SystemKernelVersion + c.config.systemBootTime = info.SystemBootTime + c.config.cpuCores = info.CpuCores + c.config.cpuModelName = info.CpuModelName + c.config.cpuMhz = info.CpuMhz + + c.config.systemInsideIp = goip.GetInsideIp(ctx) + + c.config.sdkVersion = go_library.Version() + c.config.goVersion = runtime.Version() + + c.config.mongoSdkVersion = version.Driver +} diff --git a/utils/golog/gin_gorm.go b/utils/golog/gin_gorm.go index 2d5139cf..704ee03d 100644 --- a/utils/golog/gin_gorm.go +++ b/utils/golog/gin_gorm.go @@ -1,195 +1,156 @@ package golog import ( - "bytes" - "encoding/json" - "github.com/dtapps/go-library/utils/gojson" + "context" + "github.com/dtapps/go-library/utils/dorm" + "github.com/dtapps/go-library/utils/goip" "github.com/dtapps/go-library/utils/gotime" - "github.com/dtapps/go-library/utils/gotrace_id" "github.com/dtapps/go-library/utils/gourl" - "github.com/dtapps/go-library/utils/goxml" "github.com/gin-gonic/gin" - "gorm.io/datatypes" - "gorm.io/gorm" - "io/ioutil" - "log" - "net" "time" ) -// 模型结构体 +// 模型 type ginPostgresqlLog struct { - LogId uint `gorm:"primaryKey;comment:【记录】编号" json:"log_id,omitempty"` //【记录】编号 - TraceId string `gorm:"index;comment:【系统】跟踪编号" json:"trace_id,omitempty"` //【系统】跟踪编号 - RequestTime time.Time `gorm:"index;comment:【请求】时间" json:"request_time,omitempty"` //【请求】时间 - RequestUri string `gorm:"comment:【请求】请求链接 域名+路径+参数" json:"request_uri,omitempty"` //【请求】请求链接 域名+路径+参数 - RequestUrl string `gorm:"comment:【请求】请求链接 域名+路径" json:"request_url,omitempty"` //【请求】请求链接 域名+路径 - RequestApi string `gorm:"index;comment:【请求】请求接口 路径" json:"request_api,omitempty"` //【请求】请求接口 路径 - RequestMethod string `gorm:"index;comment:【请求】请求方式" json:"request_method,omitempty"` //【请求】请求方式 - RequestProto string `gorm:"comment:【请求】请求协议" json:"request_proto,omitempty"` //【请求】请求协议 - RequestUa string `gorm:"comment:【请求】请求UA" json:"request_ua,omitempty"` //【请求】请求UA - RequestReferer string `gorm:"comment:【请求】请求referer" json:"request_referer,omitempty"` //【请求】请求referer - RequestBody datatypes.JSON `gorm:"type:jsonb;comment:【请求】请求主体" json:"request_body,omitempty"` //【请求】请求主体 - RequestUrlQuery datatypes.JSON `gorm:"type:jsonb;comment:【请求】请求URL参数" json:"request_url_query,omitempty"` //【请求】请求URL参数 - RequestIp string `gorm:"index;comment:【请求】请求客户端Ip" json:"request_ip,omitempty"` //【请求】请求客户端Ip - RequestIpCountry string `gorm:"index;comment:【请求】请求客户端城市" json:"request_ip_country,omitempty"` //【请求】请求客户端城市 - RequestIpRegion string `gorm:"index;comment:【请求】请求客户端区域" json:"request_ip_region,omitempty"` //【请求】请求客户端区域 - RequestIpProvince string `gorm:"index;comment:【请求】请求客户端省份" json:"request_ip_province,omitempty"` //【请求】请求客户端省份 - RequestIpCity string `gorm:"index;comment:【请求】请求客户端城市" json:"request_ip_city,omitempty"` //【请求】请求客户端城市 - RequestIpIsp string `gorm:"index;comment:【请求】请求客户端运营商" json:"request_ip_isp,omitempty"` //【请求】请求客户端运营商 - RequestHeader datatypes.JSON `gorm:"type:jsonb;comment:【请求】请求头" json:"request_header,omitempty"` //【请求】请求头 - ResponseTime time.Time `gorm:"index;comment:【返回】时间" json:"response_time,omitempty"` //【返回】时间 - ResponseCode int `gorm:"index;comment:【返回】状态码" json:"response_code,omitempty"` //【返回】状态码 - ResponseMsg string `gorm:"comment:【返回】描述" json:"response_msg,omitempty"` //【返回】描述 - ResponseData datatypes.JSON `gorm:"type:jsonb;comment:【返回】数据" json:"response_data,omitempty"` //【返回】数据 - CostTime int64 `gorm:"comment:【系统】花费时间" json:"cost_time,omitempty"` //【系统】花费时间 - SystemHostName string `gorm:"index;comment:【系统】主机名" json:"system_host_name,omitempty"` //【系统】主机名 - SystemInsideIp string `gorm:"index;comment:【系统】内网ip" json:"system_inside_ip,omitempty"` //【系统】内网ip - GoVersion string `gorm:"index;comment:【程序】Go版本" json:"go_version,omitempty"` //【程序】Go版本 - SdkVersion string `gorm:"index;comment:【程序】Sdk版本" json:"sdk_version,omitempty"` //【程序】Sdk版本 + LogId uint `gorm:"primaryKey;comment:【记录】编号" json:"log_id,omitempty"` //【记录】编号 + TraceId string `gorm:"index;comment:【系统】跟踪编号" json:"trace_id,omitempty"` //【系统】跟踪编号 + RequestTime time.Time `gorm:"index;comment:【请求】时间" json:"request_time,omitempty"` //【请求】时间 + RequestUri string `gorm:"comment:【请求】请求链接 域名+路径+参数" json:"request_uri,omitempty"` //【请求】请求链接 域名+路径+参数 + RequestUrl string `gorm:"comment:【请求】请求链接 域名+路径" json:"request_url,omitempty"` //【请求】请求链接 域名+路径 + RequestApi string `gorm:"index;comment:【请求】请求接口 路径" json:"request_api,omitempty"` //【请求】请求接口 路径 + RequestMethod string `gorm:"index;comment:【请求】请求方式" json:"request_method,omitempty"` //【请求】请求方式 + RequestProto string `gorm:"comment:【请求】请求协议" json:"request_proto,omitempty"` //【请求】请求协议 + RequestUa string `gorm:"comment:【请求】请求UA" json:"request_ua,omitempty"` //【请求】请求UA + RequestReferer string `gorm:"comment:【请求】请求referer" json:"request_referer,omitempty"` //【请求】请求referer + RequestBody string `gorm:"comment:【请求】请求主体" json:"request_body,omitempty"` //【请求】请求主体 + RequestUrlQuery string `gorm:"comment:【请求】请求URL参数" json:"request_url_query,omitempty"` //【请求】请求URL参数 + RequestIp string `gorm:"default:0.0.0.0;index;comment:【请求】请求客户端Ip" json:"request_ip,omitempty"` //【请求】请求客户端Ip + RequestIpCountry string `gorm:"index;comment:【请求】请求客户端城市" json:"request_ip_country,omitempty"` //【请求】请求客户端城市 + RequestIpProvince string `gorm:"index;comment:【请求】请求客户端省份" json:"request_ip_province,omitempty"` //【请求】请求客户端省份 + RequestIpCity string `gorm:"index;comment:【请求】请求客户端城市" json:"request_ip_city,omitempty"` //【请求】请求客户端城市 + RequestIpIsp string `gorm:"index;comment:【请求】请求客户端运营商" json:"request_ip_isp,omitempty"` //【请求】请求客户端运营商 + RequestIpLongitude float64 `gorm:"index;comment:【请求】请求客户端经度" json:"request_ip_longitude,omitempty"` //【请求】请求客户端经度 + RequestIpLatitude float64 `gorm:"index;comment:【请求】请求客户端纬度" json:"request_ip_latitude,omitempty"` //【请求】请求客户端纬度 + RequestHeader string `gorm:"comment:【请求】请求头" json:"request_header,omitempty"` //【请求】请求头 + ResponseTime time.Time `gorm:"index;comment:【返回】时间" json:"response_time,omitempty"` //【返回】时间 + ResponseCode int `gorm:"index;comment:【返回】状态码" json:"response_code,omitempty"` //【返回】状态码 + ResponseMsg string `gorm:"comment:【返回】描述" json:"response_msg,omitempty"` //【返回】描述 + ResponseData string `gorm:"comment:【返回】数据" json:"response_data,omitempty"` //【返回】数据 + CostTime int64 `gorm:"comment:【系统】花费时间" json:"cost_time,omitempty"` //【系统】花费时间 + SystemHostName string `gorm:"index;comment:【系统】主机名" json:"system_host_name,omitempty"` //【系统】主机名 + SystemInsideIp string `gorm:"default:0.0.0.0;comment:【系统】内网ip" json:"system_inside_ip,omitempty"` //【系统】内网ip + SystemOs string `gorm:"index;comment:【系统】系统类型" json:"system_os,omitempty"` //【系统】系统类型 + SystemArch string `gorm:"index;comment:【系统】系统架构" json:"system_arch,omitempty"` //【系统】系统架构 + GoVersion string `gorm:"comment:【程序】Go版本" json:"go_version,omitempty"` //【程序】Go版本 + SdkVersion string `gorm:"comment:【程序】Sdk版本" json:"sdk_version,omitempty"` //【程序】Sdk版本 } -// gormRecord 记录日志 -func (c *GinClient) gormRecord(postgresqlLog ginPostgresqlLog) error { - - postgresqlLog.SystemHostName = c.config.hostname - if postgresqlLog.SystemInsideIp == "" { - postgresqlLog.SystemInsideIp = c.config.insideIp +// 创建模型 +func (c *GinClient) gormAutoMigrate(ctx context.Context) { + err := c.gormClient.GetDb().Table(c.gormConfig.tableName).AutoMigrate(&ginPostgresqlLog{}) + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("创建模型：%s", err) } - postgresqlLog.GoVersion = c.config.goVersion - - postgresqlLog.SdkVersion = Version - - return c.gormClient.Table(c.config.tableName).Create(&postgresqlLog).Error } -// GormQuery 查询 -func (c *GinClient) GormQuery() *gorm.DB { - return c.gormClient.Table(c.config.tableName) +// gormRecord 记录日志 +func (c *GinClient) gormRecord(data ginPostgresqlLog) { + + data.SystemHostName = c.config.systemHostname //【系统】主机名 + data.SystemInsideIp = c.config.systemInsideIp //【系统】内网ip + data.GoVersion = c.config.goVersion //【程序】Go版本 + data.SdkVersion = c.config.sdkVersion //【程序】Sdk版本 + data.SystemOs = c.config.systemOs //【系统】系统类型 + data.SystemArch = c.config.systemKernel //【系统】系统架构 + + err := c.gormClient.GetDb().Table(c.gormConfig.tableName).Create(&data).Error + if err != nil { + c.zapLog.WithTraceIdStr(data.TraceId).Sugar().Errorf("记录框架日志错误：%s", err) + c.zapLog.WithTraceIdStr(data.TraceId).Sugar().Errorf("记录框架日志数据：%+v", data) + } } -// GormMiddleware 中间件 -func (c *GinClient) GormMiddleware() gin.HandlerFunc { - return func(ginCtx *gin.Context) { - - // 开始时间 - startTime := gotime.Current().TimestampWithMillisecond() - requestTime := gotime.Current().Time - - // 获取 - data, _ := ioutil.ReadAll(ginCtx.Request.Body) - - // 复用 - ginCtx.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - - blw := &bodyLogWriter{body: bytes.NewBufferString(""), ResponseWriter: ginCtx.Writer} - ginCtx.Writer = blw - - // 处理请求 - ginCtx.Next() +func (c *GinClient) gormRecordJson(ginCtx *gin.Context, traceId string, requestTime time.Time, requestBody []byte, responseCode int, responseBody string, startTime, endTime int64, ipInfo goip.AnalyseResult) { + + data := ginPostgresqlLog{ + TraceId: traceId, //【系统】跟踪编号 + RequestTime: requestTime, //【请求】时间 + RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 + RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 + RequestMethod: ginCtx.Request.Method, //【请求】请求方式 + RequestProto: ginCtx.Request.Proto, //【请求】请求协议 + RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA + RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer + RequestUrlQuery: dorm.JsonEncodeNoError(ginCtx.Request.URL.Query()), //【请求】请求URL参数 + RequestIp: ipInfo.Ip, //【请求】请求客户端Ip + RequestIpCountry: ipInfo.Country, //【请求】请求客户端城市 + RequestIpProvince: ipInfo.Province, //【请求】请求客户端省份 + RequestIpCity: ipInfo.City, //【请求】请求客户端城市 + RequestIpIsp: ipInfo.Isp, //【请求】请求客户端运营商 + RequestIpLatitude: ipInfo.LocationLatitude, //【请求】请求客户端纬度 + RequestIpLongitude: ipInfo.LocationLongitude, //【请求】请求客户端经度 + RequestHeader: dorm.JsonEncodeNoError(ginCtx.Request.Header), //【请求】请求头 + ResponseTime: gotime.Current().Time, //【返回】时间 + ResponseCode: responseCode, //【返回】状态码 + ResponseData: responseBody, //【返回】数据 + CostTime: endTime - startTime, //【系统】花费时间 + } + if ginCtx.Request.TLS == nil { + data.RequestUri = "http://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } else { + data.RequestUri = "https://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } - // 响应 - responseCode := ginCtx.Writer.Status() - responseBody := blw.body.String() + if len(requestBody) > 0 { + data.RequestBody = dorm.JsonEncodeNoError(requestBody) //【请求】请求主体 + } - //结束时间 - endTime := gotime.Current().TimestampWithMillisecond() + c.gormRecord(data) +} - go func() { +func (c *GinClient) gormRecordXml(ginCtx *gin.Context, traceId string, requestTime time.Time, requestBody []byte, responseCode int, responseBody string, startTime, endTime int64, ipInfo goip.AnalyseResult) { + + data := ginPostgresqlLog{ + TraceId: traceId, //【系统】跟踪编号 + RequestTime: requestTime, //【请求】时间 + RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 + RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 + RequestMethod: ginCtx.Request.Method, //【请求】请求方式 + RequestProto: ginCtx.Request.Proto, //【请求】请求协议 + RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA + RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer + RequestUrlQuery: dorm.JsonEncodeNoError(ginCtx.Request.URL.Query()), //【请求】请求URL参数 + RequestIp: ipInfo.Ip, //【请求】请求客户端Ip + RequestIpCountry: ipInfo.Country, //【请求】请求客户端城市 + RequestIpProvince: ipInfo.Province, //【请求】请求客户端省份 + RequestIpCity: ipInfo.City, //【请求】请求客户端城市 + RequestIpIsp: ipInfo.Isp, //【请求】请求客户端运营商 + RequestIpLatitude: ipInfo.LocationLatitude, //【请求】请求客户端纬度 + RequestIpLongitude: ipInfo.LocationLongitude, //【请求】请求客户端经度 + RequestHeader: dorm.JsonEncodeNoError(ginCtx.Request.Header), //【请求】请求头 + ResponseTime: gotime.Current().Time, //【返回】时间 + ResponseCode: responseCode, //【返回】状态码 + ResponseData: responseBody, //【返回】数据 + CostTime: endTime - startTime, //【系统】花费时间 + } + if ginCtx.Request.TLS == nil { + data.RequestUri = "http://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } else { + data.RequestUri = "https://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } - // 解析请求内容 - var xmlBody map[string]string - var jsonBody map[string]interface{} - _ = json.Unmarshal(data, &jsonBody) - if len(jsonBody) <= 0 { - xmlBody = goxml.XmlDecode(string(data)) - } + if len(requestBody) > 0 { + data.RequestBody = dorm.XmlEncodeNoError(dorm.XmlDecodeNoError(requestBody)) //【请求】请求内容 + } - requestClientIpCountry, requestClientIpRegion, requestClientIpProvince, requestClientIpCity, requestClientIpIsp := "", "", "", "", "" - if c.ipService != nil { - if net.ParseIP(ginCtx.ClientIP()).To4() != nil { - // 判断是不是IPV4 - _, info := c.ipService.Ipv4(ginCtx.ClientIP()) - requestClientIpCountry = info.Country - requestClientIpRegion = info.Region - requestClientIpProvince = info.Province - requestClientIpCity = info.City - requestClientIpIsp = info.ISP - } else if net.ParseIP(ginCtx.ClientIP()).To16() != nil { - // 判断是不是IPV6 - info := c.ipService.Ipv6(ginCtx.ClientIP()) - requestClientIpCountry = info.Country - requestClientIpProvince = info.Province - requestClientIpCity = info.City - } - } + c.gormRecord(data) +} - // 记录 - if c.gormClient != nil { - host := "" - if ginCtx.Request.TLS == nil { - host = "http://" + ginCtx.Request.Host - } else { - host = "https://" + ginCtx.Request.Host - } - if len(jsonBody) > 0 { - err := c.gormRecord(ginPostgresqlLog{ - TraceId: gotrace_id.GetGinTraceId(ginCtx), //【系统】链编号 - RequestTime: requestTime, //【请求】时间 - RequestUri: host + ginCtx.Request.RequestURI, //【请求】请求链接 - RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 - RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 - RequestMethod: ginCtx.Request.Method, //【请求】请求方式 - RequestProto: ginCtx.Request.Proto, //【请求】请求协议 - RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA - RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer - RequestBody: datatypes.JSON(gojson.JsonEncodeNoError(jsonBody)), //【请求】请求主体 - RequestUrlQuery: datatypes.JSON(gojson.JsonEncodeNoError(ginCtx.Request.URL.Query())), //【请求】请求URL参数 - RequestIp: ginCtx.ClientIP(), //【请求】请求客户端Ip - RequestIpCountry: requestClientIpCountry, //【请求】请求客户端城市 - RequestIpRegion: requestClientIpRegion, //【请求】请求客户端区域 - RequestIpProvince: requestClientIpProvince, //【请求】请求客户端省份 - RequestIpCity: requestClientIpCity, //【请求】请求客户端城市 - RequestIpIsp: requestClientIpIsp, //【请求】请求客户端运营商 - RequestHeader: datatypes.JSON(gojson.JsonEncodeNoError(ginCtx.Request.Header)), //【请求】请求头 - ResponseTime: gotime.Current().Time, //【返回】时间 - ResponseCode: responseCode, //【返回】状态码 - ResponseData: datatypes.JSON(responseBody), //【返回】数据 - CostTime: endTime - startTime, //【系统】花费时间 - }) - if err != nil { - log.Println("log.gormRecord：", err.Error()) - } - } else { - err := c.gormRecord(ginPostgresqlLog{ - TraceId: gotrace_id.GetGinTraceId(ginCtx), //【系统】链编号 - RequestTime: requestTime, //【请求】时间 - RequestUri: host + ginCtx.Request.RequestURI, //【请求】请求链接 - RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 - RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 - RequestMethod: ginCtx.Request.Method, //【请求】请求方式 - RequestProto: ginCtx.Request.Proto, //【请求】请求协议 - RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA - RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer - RequestBody: datatypes.JSON(gojson.JsonEncodeNoError(xmlBody)), //【请求】请求主体 - RequestUrlQuery: datatypes.JSON(gojson.JsonEncodeNoError(ginCtx.Request.URL.Query())), //【请求】请求URL参数 - RequestIp: ginCtx.ClientIP(), //【请求】请求客户端Ip - RequestIpCountry: requestClientIpCountry, //【请求】请求客户端城市 - RequestIpRegion: requestClientIpRegion, //【请求】请求客户端区域 - RequestIpProvince: requestClientIpProvince, //【请求】请求客户端省份 - RequestIpCity: requestClientIpCity, //【请求】请求客户端城市 - RequestIpIsp: requestClientIpIsp, //【请求】请求客户端运营商 - RequestHeader: datatypes.JSON(gojson.JsonEncodeNoError(ginCtx.Request.Header)), //【请求】请求头 - ResponseTime: gotime.Current().Time, //【返回】时间 - ResponseCode: responseCode, //【返回】状态码 - ResponseData: datatypes.JSON(responseBody), //【返回】数据 - CostTime: endTime - startTime, //【系统】花费时间 - }) - if err != nil { - log.Println("log.gormRecord：", err.Error()) - } - } - } - }() +// GormDelete 删除 +func (c *GinClient) GormDelete(ctx context.Context, hour int64) error { + err := c.gormClient.GetDb().Table(c.gormConfig.tableName).Where("request_time < ?", gotime.Current().BeforeHour(hour).Format()).Delete(&ginPostgresqlLog{}).Error + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("删除失败：%s", err) } + return err } diff --git a/utils/golog/gin_mongo.go b/utils/golog/gin_mongo.go index 73645023..d1821d51 100644 --- a/utils/golog/gin_mongo.go +++ b/utils/golog/gin_mongo.go @@ -1,191 +1,247 @@ package golog import ( - "bytes" - "encoding/json" - "github.com/dtapps/go-library" + "context" "github.com/dtapps/go-library/utils/dorm" + "github.com/dtapps/go-library/utils/goip" "github.com/dtapps/go-library/utils/gotime" "github.com/dtapps/go-library/utils/gotrace_id" "github.com/dtapps/go-library/utils/gourl" - "github.com/dtapps/go-library/utils/goxml" "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "io/ioutil" - "net" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" ) +type ginMongoLogRequestIpLocationLocation struct { + Type string `json:"type,omitempty" bson:"type,omitempty"` // GeoJSON类型 + Coordinates []float64 `json:"coordinates,omitempty" bson:"coordinates,omitempty"` // 经度,纬度 +} + // 模型结构体 type ginMongoLog struct { - LogId primitive.ObjectID `json:"log_id,omitempty" bson:"_id,omitempty"` //【记录】编号 - TraceId string `json:"trace_id,omitempty" bson:"trace_id,omitempty"` //【系统】链编号 - RequestTime int64 `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 - RequestUri string `json:"request_uri,omitempty" bson:"request_uri,omitempty"` //【请求】请求链接 域名+路径+参数 - RequestUrl string `json:"request_url,omitempty" bson:"request_url,omitempty"` //【请求】请求链接 域名+路径 - RequestApi string `json:"request_api,omitempty" bson:"request_api,omitempty"` //【请求】请求接口 路径 - RequestMethod string `json:"request_method,omitempty" bson:"request_method,omitempty"` //【请求】请求方式 - RequestProto string `json:"request_proto,omitempty" bson:"request_proto,omitempty"` //【请求】请求协议 - RequestUa string `json:"request_ua,omitempty" bson:"request_ua,omitempty"` //【请求】请求UA - RequestReferer string `json:"request_referer,omitempty" bson:"request_referer,omitempty"` //【请求】请求referer - RequestBody interface{} `json:"request_body,omitempty" bson:"request_body,omitempty"` //【请求】请求主体 - RequestUrlQuery interface{} `json:"request_url_query,omitempty" bson:"request_url_query,omitempty"` //【请求】请求URL参数 - RequestIp string `json:"request_ip,omitempty" bson:"request_ip,omitempty"` //【请求】请求客户端Ip - RequestIpCountry string `json:"request_ip_country,omitempty" bson:"request_ip_country,omitempty"` //【请求】请求客户端城市 - RequestIpRegion string `json:"request_ip_region,omitempty" bson:"request_ip_region,omitempty"` //【请求】请求客户端区域 - RequestIpProvince string `json:"request_ip_province,omitempty" bson:"request_ip_province,omitempty"` //【请求】请求客户端省份 - RequestIpCity string `json:"request_ip_city,omitempty" bson:"request_ip_city,omitempty"` //【请求】请求客户端城市 - RequestIpIsp string `json:"request_ip_isp,omitempty" bson:"request_ip_isp,omitempty"` //【请求】请求客户端运营商 - RequestHeader interface{} `json:"request_header,omitempty" bson:"request_header,omitempty"` //【请求】请求头 - ResponseTime int64 `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 - ResponseCode int `json:"response_code,omitempty" bson:"response_code,omitempty"` //【返回】状态码 - ResponseMsg string `json:"response_msg,omitempty" bson:"response_msg,omitempty"` //【返回】描述 - ResponseData interface{} `json:"response_data,omitempty" bson:"response_data,omitempty"` //【返回】数据 - CostTime int64 `json:"cost_time,omitempty" bson:"cost_time,omitempty"` //【系统】花费时间 - SystemHostName string `json:"system_host_name,omitempty" bson:"system_host_name,omitempty"` //【系统】主机名 - SystemInsideIp string `json:"system_inside_ip,omitempty" bson:"system_inside_ip,omitempty"` //【系统】内网ip - GoVersion string `json:"go_version,omitempty" bson:"go_version,omitempty"` //【程序】Go版本 - SdkVersion string `json:"sdk_version,omitempty" bson:"sdk_version,omitempty"` //【程序】Sdk版本 + LogId primitive.ObjectID `json:"log_id,omitempty" bson:"_id,omitempty"` //【记录】编号 + TraceId string `json:"trace_id,omitempty" bson:"trace_id,omitempty"` //【记录】跟踪编号 + RequestTime string `json:"request_time,omitempty" bson:"request_time,omitempty"` //【请求】时间 + RequestUri string `json:"request_uri,omitempty" bson:"request_uri,omitempty"` //【请求】请求链接 域名+路径+参数 + RequestUrl string `json:"request_url,omitempty" bson:"request_url,omitempty"` //【请求】请求链接 域名+路径 + RequestApi string `json:"request_api,omitempty" bson:"request_api,omitempty"` //【请求】请求接口 路径 + RequestMethod string `json:"request_method,omitempty" bson:"request_method,omitempty"` //【请求】请求方式 + RequestProto string `json:"request_proto,omitempty" bson:"request_proto,omitempty"` //【请求】请求协议 + RequestUa string `json:"request_ua,omitempty" bson:"request_ua,omitempty"` //【请求】请求UA + RequestReferer string `json:"request_referer,omitempty" bson:"request_referer,omitempty"` //【请求】请求referer + RequestBody interface{} `json:"request_body,omitempty" bson:"request_body,omitempty"` //【请求】请求主体 + RequestUrlQuery interface{} `json:"request_url_query,omitempty" bson:"request_url_query,omitempty"` //【请求】请求URL参数 + RequestIp struct { + Ip string `json:"ip,omitempty" bson:"ip,omitempty"` //【请求】请求客户端Ip + Continent string `json:"continent,omitempty" bson:"continent,omitempty"` //【请求】请求客户端大陆 + Country string `json:"country,omitempty" bson:"country,omitempty"` //【请求】请求客户端国家 + Province string `json:"province,omitempty" bson:"province,omitempty"` //【请求】请求客户端省份 + City string `json:"city,omitempty" bson:"city,omitempty"` //【请求】请求客户端城市 + Isp string `json:"isp,omitempty" bson:"isp,omitempty"` //【请求】请求客户端运营商 + } `json:"request_ip,omitempty" bson:"request_ip,omitempty"` //【请求】请求客户端信息 + RequestIpLocation interface{} `json:"request_ip_location,omitempty" bson:"request_ip_location,omitempty"` //【请求】请求客户端位置 + RequestHeader interface{} `json:"request_header,omitempty" bson:"request_header,omitempty"` //【请求】请求头 + ResponseTime string `json:"response_time,omitempty" bson:"response_time,omitempty"` //【返回】时间 + ResponseCode int `json:"response_code,omitempty" bson:"response_code,omitempty"` //【返回】状态码 + ResponseMsg string `json:"response_msg,omitempty" bson:"response_msg,omitempty"` //【返回】描述 + ResponseData interface{} `json:"response_data,omitempty" bson:"response_data,omitempty"` //【返回】数据 + CostTime int64 `json:"cost_time,omitempty" bson:"cost_time,omitempty"` //【系统】花费时间 + System struct { + Hostname string `json:"hostname" bson:"hostname"` //【系统】主机名 + Os string `json:"os" bson:"os"` //【系统】系统类型 + Version string `json:"version" bson:"version"` //【系统】系统版本 + Kernel string `json:"kernel" bson:"kernel"` //【系统】系统内核 + KernelVersion string `json:"kernel_version" bson:"kernel_version"` //【系统】系统内核版本 + BootTime string `json:"boot_time" bson:"boot_time"` //【系统】系统开机时间 + CpuCores int `json:"cpu_cores,omitempty" bson:"cpu_cores,omitempty"` //【系统】CPU核数 + CpuModelName string `json:"cpu_model_name,omitempty" bson:"cpu_model_name,omitempty"` //【系统】CPU型号名称 + CpuMhz float64 `json:"cpu_mhz,omitempty" bson:"cpu_mhz,omitempty"` //【系统】CPU兆赫 + InsideIp string `json:"inside_ip" bson:"inside_ip"` //【系统】内网ip + OutsideIp string `json:"outside_ip" bson:"outside_ip"` //【系统】外网ip + GoVersion string `json:"go_version,omitempty" bson:"go_version,omitempty"` //【系统】go版本 + SdkVersion string `json:"sdk_version,omitempty" bson:"sdk_version,omitempty"` //【系统】sdk版本 + MongoVersion string `json:"mongo_version,omitempty" bson:"mongo_version,omitempty"` //【系统】mongo版本 + MongoSdkVersion string `json:"mongo_sdk_version,omitempty" bson:"mongo_sdk_version,omitempty"` //【系统】mongo sdk版本 + } `json:"system,omitempty" bson:"system,omitempty"` //【系统】信息 } -// 记录日志 -func (c *GinClient) mongoRecord(mongoLog ginMongoLog) error { +// 创建集合 +func (c *GinClient) mongoCreateCollection(ctx context.Context) { + err := c.mongoClient.Database(c.mongoConfig.databaseName).CreateCollection(ctx, c.mongoConfig.collectionName, options.CreateCollection().SetCollation(&options.Collation{ + Locale: "request_time", + Strength: -1, + })) + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Error("创建集合：", err) + } +} - mongoLog.SystemHostName = c.config.hostname - if mongoLog.SystemInsideIp == "" { - mongoLog.SystemInsideIp = c.config.insideIp +// 创建索引 +func (c *GinClient) mongoCreateIndexes(ctx context.Context) { + _, err := c.mongoClient.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).CreateManyIndexes(ctx, []mongo.IndexModel{ + { + Keys: bson.D{{ + Key: "trace_id", + Value: -1, + }}, + }, { + Keys: bson.D{{ + Key: "request_time", + Value: -1, + }}, + }, { + Keys: bson.D{{ + Key: "request_method", + Value: 1, + }}, + }, { + Keys: bson.D{{ + Key: "response_time", + Value: -1, + }}, + }, { + Keys: bson.D{{ + Key: "response_code", + Value: 1, + }}, + }, { + Keys: bson.D{{ + Key: "request_ip_location", + Value: "2dsphere", + }}, + }, + }) + if err != nil { + c.zapLog.WithTraceId(ctx).Sugar().Errorf("创建索引：%s", err) + } +} + +// MongoDelete 删除 +func (c *GinClient) MongoDelete(ctx context.Context, hour int64) (*mongo.DeleteResult, error) { + filter := bson.D{{"request_time", bson.D{{"$lt", gotime.SetCurrent(gotime.Current().BeforeHour(hour).Time)}}}} + return c.mongoClient.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).DeleteMany(ctx, filter) +} + +// 记录日志 +func (c *GinClient) mongoRecord(ctx context.Context, data ginMongoLog) { + + data.LogId = primitive.NewObjectID() //【记录】编号 + data.System.Hostname = c.config.systemHostname //【系统】主机名 + data.System.Os = c.config.systemOs //【系统】系统类型 + data.System.Version = c.config.systemVersion //【系统】系统版本 + data.System.Kernel = c.config.systemKernel //【系统】系统内核 + data.System.KernelVersion = c.config.systemKernelVersion //【系统】系统内核版本 + data.System.BootTime = gotime.SetCurrent(gotime.SetCurrentUnix(int64(c.config.systemBootTime)).Time).Format() //【系统】系统开机时间 + data.System.CpuCores = c.config.cpuCores //【系统】CPU核数 + data.System.CpuModelName = c.config.cpuModelName //【程序】CPU型号名称 + data.System.CpuMhz = c.config.cpuMhz //【系统】CPU兆赫 + data.System.InsideIp = c.config.systemInsideIp //【系统】内网ip + data.System.OutsideIp = c.config.systemOutsideIp //【系统】外网ip + data.System.GoVersion = c.config.goVersion //【系统】Go版本 + data.System.SdkVersion = c.config.sdkVersion //【系统】Sdk版本 + data.System.MongoVersion = c.config.mongoVersion //【系统】mongo版本 + data.System.MongoSdkVersion = c.config.mongoSdkVersion //【系统】mongo sdk版本 + + _, err := c.mongoClient.Database(c.mongoConfig.databaseName).Collection(c.mongoConfig.collectionName).InsertOne(ctx, data) + if err != nil { + c.zapLog.WithTraceIdStr(data.TraceId).Sugar().Errorf("保存框架日志错误：%s", err) + c.zapLog.WithTraceIdStr(data.TraceId).Sugar().Errorf("保存框架日志数据：%+v", data) } - mongoLog.GoVersion = c.config.goVersion +} - mongoLog.SdkVersion = go_library.Version() +func (c *GinClient) mongoRecordJson(ginCtx *gin.Context, traceId string, requestTime time.Time, requestBody []byte, responseCode int, responseBody string, startTime, endTime int64, ipInfo goip.AnalyseResult) { + + var ctx = gotrace_id.SetGinTraceIdContext(context.Background(), ginCtx) + + data := ginMongoLog{ + TraceId: traceId, //【记录】跟踪编号 + RequestTime: gotime.SetCurrent(requestTime).Format(), //【请求】时间 + RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 + RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 + RequestMethod: ginCtx.Request.Method, //【请求】请求方式 + RequestProto: ginCtx.Request.Proto, //【请求】请求协议 + RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA + RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer + RequestUrlQuery: ginCtx.Request.URL.Query(), //【请求】请求URL参数 + RequestHeader: ginCtx.Request.Header, //【请求】请求头 + ResponseTime: gotime.Current().Format(), //【返回】时间 + ResponseCode: responseCode, //【返回】状态码 + ResponseData: dorm.JsonDecodeNoError([]byte(responseBody)), //【返回】数据 + CostTime: endTime - startTime, //【系统】花费时间 + } + if ginCtx.Request.TLS == nil { + data.RequestUri = "http://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } else { + data.RequestUri = "https://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } - mongoLog.LogId = primitive.NewObjectID() + if len(requestBody) > 0 { + data.RequestBody = dorm.JsonDecodeNoError(requestBody) //【请求】请求主体 + } - _, err := c.mongoClient.Database(c.config.databaseName).Collection(c.config.collectionName).InsertOne(mongoLog) + //【请求】请求客户端信息 + data.RequestIp.Ip = ipInfo.Ip + data.RequestIp.Continent = ipInfo.Continent + data.RequestIp.Country = ipInfo.Country + data.RequestIp.Province = ipInfo.Province + data.RequestIp.City = ipInfo.City + data.RequestIp.Isp = ipInfo.Isp + if ipInfo.LocationLatitude != 0 && ipInfo.LocationLongitude != 0 { + data.RequestIpLocation = ginMongoLogRequestIpLocationLocation{ + Type: "Point", + Coordinates: []float64{ipInfo.LocationLongitude, ipInfo.LocationLatitude}, + } + } - return err + c.mongoRecord(ctx, data) } -// MongoQuery 查询 -func (c *GinClient) MongoQuery() *dorm.MongoClient { - return c.mongoClient.Database(c.config.databaseName).Collection(c.config.collectionName) -} +func (c *GinClient) mongoRecordXml(ginCtx *gin.Context, traceId string, requestTime time.Time, requestBody []byte, responseCode int, responseBody string, startTime, endTime int64, ipInfo goip.AnalyseResult) { + + var ctx = gotrace_id.SetGinTraceIdContext(context.Background(), ginCtx) + + data := ginMongoLog{ + TraceId: traceId, //【记录】跟踪编号 + RequestTime: gotime.SetCurrent(requestTime).Format(), //【请求】时间 + RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 + RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 + RequestMethod: ginCtx.Request.Method, //【请求】请求方式 + RequestProto: ginCtx.Request.Proto, //【请求】请求协议 + RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA + RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer + RequestUrlQuery: ginCtx.Request.URL.Query(), //【请求】请求URL参数 + RequestHeader: ginCtx.Request.Header, //【请求】请求头 + ResponseTime: gotime.Current().Format(), //【返回】时间 + ResponseCode: responseCode, //【返回】状态码 + ResponseData: dorm.JsonDecodeNoError([]byte(responseBody)), //【返回】数据 + CostTime: endTime - startTime, //【系统】花费时间 + } + if ginCtx.Request.TLS == nil { + data.RequestUri = "http://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } else { + data.RequestUri = "https://" + ginCtx.Request.Host + ginCtx.Request.RequestURI //【请求】请求链接 + } -// MongoMiddleware 中间件 -func (c *GinClient) MongoMiddleware() gin.HandlerFunc { - return func(ginCtx *gin.Context) { - - // 开始时间 - startTime := gotime.Current().TimestampWithMillisecond() - requestTime := gotime.Current().Time - - // 获取 - data, _ := ioutil.ReadAll(ginCtx.Request.Body) - - // 复用 - ginCtx.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - - blw := &bodyLogWriter{body: bytes.NewBufferString(""), ResponseWriter: ginCtx.Writer} - ginCtx.Writer = blw - - // 处理请求 - ginCtx.Next() - - // 响应 - responseCode := ginCtx.Writer.Status() - responseBody := blw.body.String() - - //结束时间 - endTime := gotime.Current().TimestampWithMillisecond() - - go func() { - - // 解析请求内容 - var xmlBody map[string]string - var jsonBody map[string]interface{} - _ = json.Unmarshal(data, &jsonBody) - if len(jsonBody) <= 0 { - xmlBody = goxml.XmlDecode(string(data)) - } - - requestClientIpCountry, requestClientIpRegion, requestClientIpProvince, requestClientIpCity, requestClientIpIsp := "", "", "", "", "" - if c.ipService != nil { - if net.ParseIP(ginCtx.ClientIP()).To4() != nil { - // 判断是不是IPV4 - _, info := c.ipService.Ipv4(ginCtx.ClientIP()) - requestClientIpCountry = info.Country - requestClientIpRegion = info.Region - requestClientIpProvince = info.Province - requestClientIpCity = info.City - requestClientIpIsp = info.ISP - } else if net.ParseIP(ginCtx.ClientIP()).To16() != nil { - // 判断是不是IPV6 - info := c.ipService.Ipv6(ginCtx.ClientIP()) - requestClientIpCountry = info.Country - requestClientIpProvince = info.Province - requestClientIpCity = info.City - } - } - - // 记录 - if c.mongoClient != nil { - host := "" - if ginCtx.Request.TLS == nil { - host = "http://" + ginCtx.Request.Host - } else { - host = "https://" + ginCtx.Request.Host - } - if len(jsonBody) > 0 { - c.mongoRecord(ginMongoLog{ - TraceId: gotrace_id.GetGinTraceId(ginCtx), //【系统】链编号 - RequestTime: gotime.SetCurrent(requestTime).Timestamp(), //【请求】时间 - RequestUri: host + ginCtx.Request.RequestURI, //【请求】请求链接 - RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 - RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 - RequestMethod: ginCtx.Request.Method, //【请求】请求方式 - RequestProto: ginCtx.Request.Proto, //【请求】请求协议 - RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA - RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer - RequestBody: jsonBody, //【请求】请求主体 - RequestUrlQuery: ginCtx.Request.URL.Query(), //【请求】请求URL参数 - RequestIp: ginCtx.ClientIP(), //【请求】请求客户端Ip - RequestIpCountry: requestClientIpCountry, //【请求】请求客户端城市 - RequestIpRegion: requestClientIpRegion, //【请求】请求客户端区域 - RequestIpProvince: requestClientIpProvince, //【请求】请求客户端省份 - RequestIpCity: requestClientIpCity, //【请求】请求客户端城市 - RequestIpIsp: requestClientIpIsp, //【请求】请求客户端运营商 - RequestHeader: ginCtx.Request.Header, //【请求】请求头 - ResponseTime: gotime.SetCurrent(gotime.Current().Time).Timestamp(), //【返回】时间 - ResponseCode: responseCode, //【返回】状态码 - ResponseData: c.jsonUnmarshal(responseBody), //【返回】数据 - CostTime: endTime - startTime, //【系统】花费时间 - }) - } else { - c.mongoRecord(ginMongoLog{ - TraceId: gotrace_id.GetGinTraceId(ginCtx), //【系统】链编号 - RequestTime: gotime.SetCurrent(requestTime).Timestamp(), //【请求】时间 - RequestUri: host + ginCtx.Request.RequestURI, //【请求】请求链接 - RequestUrl: ginCtx.Request.RequestURI, //【请求】请求链接 - RequestApi: gourl.UriFilterExcludeQueryString(ginCtx.Request.RequestURI), //【请求】请求接口 - RequestMethod: ginCtx.Request.Method, //【请求】请求方式 - RequestProto: ginCtx.Request.Proto, //【请求】请求协议 - RequestUa: ginCtx.Request.UserAgent(), //【请求】请求UA - RequestReferer: ginCtx.Request.Referer(), //【请求】请求referer - RequestBody: xmlBody, //【请求】请求主体 - RequestUrlQuery: ginCtx.Request.URL.Query(), //【请求】请求URL参数 - RequestIp: ginCtx.ClientIP(), //【请求】请求客户端Ip - RequestIpCountry: requestClientIpCountry, //【请求】请求客户端城市 - RequestIpRegion: requestClientIpRegion, //【请求】请求客户端区域 - RequestIpProvince: requestClientIpProvince, //【请求】请求客户端省份 - RequestIpCity: requestClientIpCity, //【请求】请求客户端城市 - RequestIpIsp: requestClientIpIsp, //【请求】请求客户端运营商 - RequestHeader: ginCtx.Request.Header, //【请求】请求头 - ResponseTime: gotime.SetCurrent(gotime.Current().Time).Timestamp(), //【返回】时间 - ResponseCode: responseCode, //【返回】状态码 - ResponseData: c.jsonUnmarshal(responseBody), //【返回】数据 - CostTime: endTime - startTime, //【系统】花费时间 - }) - } - } - }() + if len(requestBody) > 0 { + data.RequestBody = dorm.XmlDecodeNoError(requestBody) //【请求】请求主体 } + + //【请求】请求客户端信息 + data.RequestIp.Ip = ipInfo.Ip + data.RequestIp.Continent = ipInfo.Continent + data.RequestIp.Country = ipInfo.Country + data.RequestIp.Province = ipInfo.Province + data.RequestIp.City = ipInfo.City + data.RequestIp.Isp = ipInfo.Isp + if ipInfo.LocationLatitude != 0 && ipInfo.LocationLongitude != 0 { + data.RequestIpLocation = ginMongoLogRequestIpLocationLocation{ + Type: "Point", + Coordinates: []float64{ipInfo.LocationLongitude, ipInfo.LocationLatitude}, + } + } + + c.mongoRecord(ctx, data) } diff --git a/utils/golog/json.go b/utils/golog/json.go deleted file mode 100644 index f14ab938..00000000 --- a/utils/golog/json.go +++ /dev/null @@ -1,8 +0,0 @@ -package golog - -import "encoding/json" - -func JsonEncodeNoError(data interface{}) string { - jsons, _ := json.Marshal(data) - return string(jsons) -} diff --git a/utils/golog/log.go b/utils/golog/log.go deleted file mode 100644 index 397e9a17..00000000 --- a/utils/golog/log.go +++ /dev/null @@ -1,127 +0,0 @@ -package golog - -import ( - "github.com/natefinch/lumberjack" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "os" - "time" -) - -type ConfigGoLog struct { - LogPath string // 日志文件路径 - LogName string // 日志文件名 - LogLevel string // 日志级别 debug/info/warn/error，debug输出：debug/info/warn/error日志。 info输出：info/warn/error日志。 warn输出：warn/error日志。 error输出：error日志。 - MaxSize int // 单个文件大小,MB - MaxBackups int // 保存的文件个数 - MaxAge int // 保存的天数 0=不删除 - Compress bool // 压缩 - JsonFormat bool // 是否输出为json格式 - ShowLine bool // 显示代码行 - LogInConsole bool // 是否同时输出到控制台 -} - -type GoLog struct { - ConfigGoLog - Logger *zap.Logger -} - -func NewGoLog(config *ConfigGoLog) *GoLog { - g := &GoLog{} - g.LogPath = config.LogPath - g.LogName = config.LogName - g.LogLevel = config.LogLevel - g.MaxSize = config.MaxSize - g.MaxBackups = config.MaxBackups - g.MaxAge = config.MaxAge - g.Compress = config.Compress - g.JsonFormat = config.JsonFormat - g.ShowLine = config.ShowLine - g.LogInConsole = config.LogInConsole - - // 设置日志级别 - var level zapcore.Level - switch g.LogLevel { - case "debug": - level = zap.DebugLevel - case "info": - level = zap.InfoLevel - case "warn": - level = zap.WarnLevel - case "error": - level = zap.ErrorLevel - default: - level = zap.InfoLevel - } - - var ( - syncer zapcore.WriteSyncer - - // 自定义时间输出格式 - customTimeEncoder = func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { - enc.AppendString(t.Format("2006-01-02 15:04:05.000")) - } - - // 自定义日志级别显示 - customLevelEncoder = func(level zapcore.Level, enc zapcore.PrimitiveArrayEncoder) { - enc.AppendString(level.CapitalString()) - } - ) - - // 定义日志切割配置 - hook := lumberjack.Logger{ - Filename: g.LogPath + g.LogName, // 日志文件的位置 - MaxSize: g.MaxSize, // 在进行切割之前，日志文件的最大大小（以MB为单位） - MaxBackups: g.MaxBackups, // 保留旧文件的最大个数 - Compress: g.Compress, // 是否压缩 disabled by default - } - if g.MaxAge > 0 { - hook.MaxAge = g.MaxAge // days - } - - // 判断是否控制台输出日志 - if g.LogInConsole { - syncer = zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), zapcore.AddSync(&hook)) - } else { - syncer = zapcore.AddSync(&hook) - } - - // 定义zap配置信息 - encoderConfig := zapcore.EncoderConfig{ - TimeKey: "time", - LevelKey: "level", - NameKey: "logger", - CallerKey: "line", - MessageKey: "msg", - StacktraceKey: "stacktrace", - LineEnding: zapcore.DefaultLineEnding, - EncodeTime: customTimeEncoder, // 自定义时间格式 - EncodeLevel: customLevelEncoder, // 小写编码器 - EncodeCaller: zapcore.ShortCallerEncoder, // 全路径编码器 - EncodeDuration: zapcore.SecondsDurationEncoder, - EncodeName: zapcore.FullNameEncoder, - } - - var encoder zapcore.Encoder - // 判断是否json格式输出 - if g.JsonFormat { - encoder = zapcore.NewJSONEncoder(encoderConfig) - } else { - encoder = zapcore.NewConsoleEncoder(encoderConfig) - } - - core := zapcore.NewCore( - encoder, - syncer, - level, - ) - - g.Logger = zap.New(core) - - // 判断是否显示代码行号 - if g.ShowLine { - g.Logger = g.Logger.WithOptions(zap.AddCaller()) - } - - return g -} diff --git a/utils/golog/operation_attr.go b/utils/golog/operation_attr.go deleted file mode 100644 index 1e30b04e..00000000 --- a/utils/golog/operation_attr.go +++ /dev/null @@ -1,53 +0,0 @@ -package golog - -import ( - "github.com/dtapps/go-library/utils/dorm" - "github.com/dtapps/go-library/utils/goip" - "gorm.io/gorm" -) - -const ( - logTypeGorm = "gorm" - logTypeMongo = "mongo" -) - -// OperationAttr 操作属性 -type OperationAttr struct { - gormClient *gorm.DB // 数据库驱动 - mongoClient *dorm.MongoClient // 数据库驱动 - ipService *goip.Client // ip服务 - logType string // 类型 - tableName string // 表名 - databaseName string // 库名 - collectionName string // 表名 -} - -// WithGormClient 设置数据库驱动 -func WithGormClient(client *gorm.DB) *OperationAttr { - return &OperationAttr{gormClient: client, logType: logTypeGorm} -} - -// WithMongoClient 设置数据库驱动 -func WithMongoClient(client *dorm.MongoClient) *OperationAttr { - return &OperationAttr{mongoClient: client, logType: logTypeMongo} -} - -// WithTableName 设置表名 -func WithTableName(tableName string) *OperationAttr { - return &OperationAttr{tableName: tableName} -} - -// WithDatabaseName 设置库名 -func WithDatabaseName(databaseName string) *OperationAttr { - return &OperationAttr{databaseName: databaseName} -} - -// WithCollectionName 设置表名 -func WithCollectionName(collectionName string) *OperationAttr { - return &OperationAttr{collectionName: collectionName} -} - -// WithIpService 设置ip服务 -func WithIpService(ipService *goip.Client) *OperationAttr { - return &OperationAttr{ipService: ipService} -} diff --git a/utils/golog/system.go b/utils/golog/system.go deleted file mode 100644 index 41af3f57..00000000 --- a/utils/golog/system.go +++ /dev/null @@ -1,33 +0,0 @@ -package golog - -import ( - "os" - "runtime" -) - -type System struct { - Variable []string // 环境变量 - Hostname string // 主机名 - Twd string // 当前目录 - Uid int // 用户ID - EUid int // 有效用户ID - Gid int // 组ID - EGid int // 有效组ID - Pid int // 进程ID - PPid int // 父进程ID - Version string // 版本 -} - -func (s *System) Init() *System { - s.Variable = os.Environ() - s.Hostname, _ = os.Hostname() - s.Twd, _ = os.Getwd() - s.Uid = os.Getuid() - s.EUid = os.Geteuid() - s.Gid = os.Getgid() - s.EGid = os.Getegid() - s.Pid = os.Getpid() - s.PPid = os.Getppid() - s.Version = runtime.Version() - return s -} diff --git a/utils/golog/time.go b/utils/golog/time.go deleted file mode 100644 index e4310097..00000000 --- a/utils/golog/time.go +++ /dev/null @@ -1,71 +0,0 @@ -package golog - -import ( - "database/sql/driver" - "errors" - "fmt" - "github.com/dtapps/go-library/utils/gotime" - "gorm.io/gorm" - "gorm.io/gorm/schema" - "time" -) - -type TimeString struct { - Time time.Time -} - -// Value 插入数据，把时间转字符串 -func (t TimeString) Value() (driver.Value, error) { - return gotime.SetCurrent(t.Time).Format(), nil -} - -// Scan 查询数据，把字符串转时间 -func (t *TimeString) Scan(value interface{}) error { - - // 如果是空值，直接返回 - data, ok := value.(string) - if !ok { - return errors.New(fmt.Sprint("无法解析:", value)) - } - - // 解析时间 - result := gotime.SetCurrentParse(data) - - *t = TimeString{ - Time: result.Time, - } - - return nil -} - -// MarshalJSON JSON序列化 -func (t TimeString) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%s"`, gotime.SetCurrent(t.Time).Format())), nil -} - -// UnmarshalJSON JSON反序列化 -func (t *TimeString) UnmarshalJSON(data []byte) (err error) { - // 删除双引号 - if data[0] == '"' && data[len(data)-1] == '"' { - data = data[1 : len(data)-1] - } - result := gotime.SetCurrentParse(string(data)) - *t = TimeString{ - Time: result.Time, - } - return -} - -func (t TimeString) GormDBDataType(db *gorm.DB, field *schema.Field) string { - // 使用 field.Tag、field.TagSettings 获取字段的 tag - // 查看 https://github.com/go-gorm/gorm/blob/master/schema/field.go 获取全部的选项 - - // 根据不同的数据库驱动返回不同的数据类型 - switch db.Dialector.Name() { - case "mysql", "sqlite": - return "varchar" - case "postgres": - return "text" - } - return "" -} diff --git a/utils/golog/zap.go b/utils/golog/zap.go new file mode 100644 index 00000000..c0910572 --- /dev/null +++ b/utils/golog/zap.go @@ -0,0 +1,137 @@ +package golog + +import ( + "context" + "github.com/dtapps/go-library/utils/gotime" + "github.com/dtapps/go-library/utils/gotrace_id" + "github.com/natefinch/lumberjack" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "os" + "time" +) + +type ZapLogConfig struct { + LogPath string // 日志文件路径 + LogName string // 日志文件名 + MaxSize int // 单位为MB,默认为512MB + MaxBackups int // 保留旧文件的最大个数 + MaxAge int // 文件最多保存多少天 0=不删除 + LocalTime bool // 采用本地时间 + Compress bool // 是否压缩日志 + JsonFormat bool // 是否输出为json格式 + ShowLine bool // 显示代码行 + LogInConsole bool // 是否同时输出到控制台 +} + +type ZapLog struct { + config *ZapLogConfig + logger *zap.Logger + zapCore zapcore.Core +} + +func NewZapLog(config *ZapLogConfig) *ZapLog { + + zl := &ZapLog{config: config} + + var syncer zapcore.WriteSyncer + + if zl.config.LogPath != "" && zl.config.LogName != "" { + // 定义日志切割配置 + hook := lumberjack.Logger{ + Filename: zl.config.LogPath + zl.config.LogName, // ⽇志⽂件路径 + MaxSize: zl.config.MaxSize, // 单位为MB,默认为512MB + MaxBackups: zl.config.MaxBackups, // 保留旧文件的最大个数 + LocalTime: zl.config.LocalTime, // 采用本地时间 + Compress: zl.config.Compress, // 是否压缩日志 + } + if zl.config.MaxAge > 0 { + // 文件最多保存多少天 + hook.MaxAge = zl.config.MaxAge + } + if zl.config.LogInConsole { + // 在控制台和文件输出日志 + syncer = zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), zapcore.AddSync(&hook)) + } else { + // 在文件输出日志 + syncer = zapcore.AddSync(&hook) + } + } else { + // 在控制台输出日志 + syncer = zapcore.NewMultiWriteSyncer() + } + + // 自定义时间输出格式 + customTimeEncoder := func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(t.Format(gotime.DateTimeFormat)) + } + + // 自定义日志级别显示 + customLevelEncoder := func(level zapcore.Level, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(level.CapitalString()) + } + + // 定义zap配置信息 + encoderConf := zapcore.EncoderConfig{ + CallerKey: "caller_line", // 打印文件名和行数 + LevelKey: "level_name", + MessageKey: "msg", + TimeKey: "time", + NameKey: "logger", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeTime: customTimeEncoder, // 自定义时间格式 + EncodeLevel: customLevelEncoder, // 小写编码器 + EncodeCaller: zapcore.ShortCallerEncoder, // 全路径编码器 + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeName: zapcore.FullNameEncoder, + } + + // 判断是否json格式输出 + if zl.config.JsonFormat { + zl.zapCore = zapcore.NewCore(zapcore.NewJSONEncoder(encoderConf), + syncer, zap.NewAtomicLevelAt(zapcore.InfoLevel)) + } else { + zl.zapCore = zapcore.NewCore(zapcore.NewConsoleEncoder(encoderConf), + syncer, zap.NewAtomicLevelAt(zapcore.InfoLevel)) + } + + zl.logger = zl.withShowLine(zap.New(zl.zapCore)) + + return zl +} + +// 判断是否显示代码行号 +func (zl *ZapLog) withShowLine(logger *zap.Logger) *zap.Logger { + if zl.config.ShowLine { + logger = logger.WithOptions(zap.AddCaller()) + } + return logger +} + +// WithLogger 跟踪编号 +func (zl *ZapLog) WithLogger() *zap.Logger { + return zl.logger +} + +// WithTraceId 跟踪编号 +func (zl *ZapLog) WithTraceId(ctx context.Context) *zap.Logger { + logger := zl.logger + logger = logger.With(zapcore.Field{ + Key: "trace_id", + Type: zapcore.StringType, + String: gotrace_id.GetTraceIdContext(ctx), + }) + return logger +} + +// WithTraceIdStr 跟踪编号 +func (zl *ZapLog) WithTraceIdStr(traceId string) *zap.Logger { + logger := zl.logger + logger = logger.With(zapcore.Field{ + Key: "trace_id", + Type: zapcore.StringType, + String: traceId, + }) + return logger +} diff --git a/utils/gophp/chunk_split.go b/utils/gophp/chunk_split.go new file mode 100644 index 00000000..cd90580e --- /dev/null +++ b/utils/gophp/chunk_split.go @@ -0,0 +1,24 @@ +package gophp + +// ChunkSplit 将字符串拆分成更小的块 +func ChunkSplit(body string, chunklen uint, end string) string { + if end == "" { + end = "\r\n" + } + runes, erunes := []rune(body), []rune(end) + l := uint(len(runes)) + if l <= 1 || l < chunklen { + return body + end + } + ns := make([]rune, 0, len(runes)+len(erunes)) + var i uint + for i = 0; i < l; i += chunklen { + if i+chunklen > l { + ns = append(ns, runes[i:]...) + } else { + ns = append(ns, runes[i:i+chunklen]...) + } + ns = append(ns, erunes...) + } + return string(ns) +} diff --git a/utils/gorequest/const.go b/utils/gorequest/const.go index 8cf10a64..9adce6a9 100644 --- a/utils/gorequest/const.go +++ b/utils/gorequest/const.go @@ -1,7 +1,7 @@ package gorequest const ( - userAgentFormat = "%s/GO/%s" + userAgentFormat = "%s/%s/%s/%s" ) // 定义请求类型 diff --git a/utils/gorequest/header.type.go b/utils/gorequest/header.type.go new file mode 100644 index 00000000..d6262ab8 --- /dev/null +++ b/utils/gorequest/header.type.go @@ -0,0 +1,65 @@ +package gorequest + +// HeaderIsImg 判断是否为图片 +func (r *Response) HeaderIsImg() bool { + if r.ResponseHeader.Get("Content-Type") == "image/jpeg" || r.ResponseHeader.Get("Content-Type") == "image/png" || r.ResponseHeader.Get("Content-Type") == "image/jpg" { + return true + } + return false +} + +// HeaderIsJpeg 判断是否为jpeg图片 +func (r *Response) HeaderIsJpeg() bool { + if r.ResponseHeader.Get("Content-Type") == "image/jpeg" { + return true + } + return false +} + +// HeaderIsPng 判断是否为Png图片 +func (r *Response) HeaderIsPng() bool { + if r.ResponseHeader.Get("Content-Type") == "image/png" { + return true + } + return false +} + +// HeaderIsJpg 判断是否为Jpg图片 +func (r *Response) HeaderIsJpg() bool { + if r.ResponseHeader.Get("Content-Type") == "image/jpg" { + return true + } + return false +} + +// HeaderJson 判断是否为Json数据 +func (r *Response) HeaderJson() bool { + if r.ResponseHeader.Get("Content-Type") == "application/json" { + return true + } + return false +} + +// HeaderHtml 判断是否为Html +func (r *Response) HeaderHtml() bool { + if r.ResponseHeader.Get("Content-Type") == "text/html" || r.ResponseHeader.Get("Content-Type") == "application/xhtml+xml" { + return true + } + return false +} + +// HeaderTextHtml 判断是否为Html +func (r *Response) HeaderTextHtml() bool { + if r.ResponseHeader.Get("Content-Type") == "text/html" { + return true + } + return false +} + +// HeaderXHtml 判断是否为Html +func (r *Response) HeaderXHtml() bool { + if r.ResponseHeader.Get("Content-Type") == "application/xhtml+xml" { + return true + } + return false +} diff --git a/utils/gorequest/http.go b/utils/gorequest/http.go index 70779b50..56c145ce 100644 --- a/utils/gorequest/http.go +++ b/utils/gorequest/http.go @@ -7,11 +7,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dtapps/go-library" "github.com/dtapps/go-library/utils/gostring" "github.com/dtapps/go-library/utils/gotime" "github.com/dtapps/go-library/utils/gotrace_id" "io" "io/ioutil" + "log" "net/http" "net/url" "runtime" @@ -21,6 +23,7 @@ import ( // Response 返回内容 type Response struct { + RequestId string //【请求】编号 RequestUri string //【请求】链接 RequestParams Params //【请求】参数 RequestMethod string //【请求】方式 @@ -36,16 +39,17 @@ type Response struct { // App 实例 type App struct { - Uri string // 全局请求地址，没有设置url才会使用 - Error error // 错误 - httpUri string // 请求地址 - httpMethod string // 请求方法 - httpHeader Headers // 请求头 - httpParams Params // 请求参数 - responseContent Response // 返回内容 - httpContentType string // 请求内容类型 - debug bool // 是否开启调试模式 - p12Cert *tls.Certificate // p12证书内容 + Uri string // 全局请求地址，没有设置url才会使用 + Error error // 错误 + httpUri string // 请求地址 + httpMethod string // 请求方法 + httpHeader Headers // 请求头 + httpParams Params // 请求参数 + responseContent Response // 返回内容 + httpContentType string // 请求内容类型 + debug bool // 是否开启调试模式 + p12Cert *tls.Certificate // p12证书内容 + afferentSdkUserVersion string // 传入SDk版本 } // NewHttp 实例化 @@ -134,6 +138,11 @@ func (app *App) SetP12Cert(content *tls.Certificate) { app.p12Cert = content } +// AfferentSdkUserVersion 传入SDk版本 +func (app *App) AfferentSdkUserVersion(afferentSdkUserVersion string) { + app.afferentSdkUserVersion = afferentSdkUserVersion +} + // Get 发起GET请求 func (app *App) Get(ctx context.Context, uri ...string) (httpResponse Response, err error) { if len(uri) == 1 { @@ -194,7 +203,11 @@ func request(app *App, ctx context.Context) (httpResponse Response, err error) { } // SDK版本 - httpResponse.RequestHeader.Set("Sdk-User-Agent", fmt.Sprintf(userAgentFormat, runtime.GOOS, runtime.Version())) + if app.afferentSdkUserVersion == "" { + httpResponse.RequestHeader.Set("Sdk-User-Agent", fmt.Sprintf(userAgentFormat, runtime.GOOS, runtime.GOARCH, runtime.Version(), go_library.Version())) + } else { + httpResponse.RequestHeader.Set("Sdk-User-Agent", fmt.Sprintf(userAgentFormat, runtime.GOOS, runtime.GOARCH, runtime.Version(), go_library.Version())+"/"+app.afferentSdkUserVersion) + } // 请求类型 switch app.httpContentType { @@ -207,11 +220,11 @@ func request(app *App, ctx context.Context) (httpResponse Response, err error) { } // 跟踪编号 - traceId := gotrace_id.GetTraceIdContext(ctx) - if traceId == "" { - traceId = gostring.GetUuId() + httpResponse.RequestId = gotrace_id.GetTraceIdContext(ctx) + if httpResponse.RequestId == "" { + httpResponse.RequestId = gostring.GetUuId() } - httpResponse.RequestHeader.Set("X-Request-Id", traceId) + httpResponse.RequestHeader.Set("X-Request-Id", httpResponse.RequestId) // 请求内容 var reqBody io.Reader @@ -297,8 +310,8 @@ func request(app *App, ctx context.Context) (httpResponse Response, err error) { httpResponse.ResponseContentLength = resp.ContentLength if app.debug == true { - fmt.Printf("gorequest：%+v\n", httpResponse) - fmt.Printf("gorequest.body：%s\n", httpResponse.ResponseBody) + log.Printf("gorequest：%+v\n", httpResponse) + log.Printf("gorequest.body：%s\n", httpResponse.ResponseBody) } return httpResponse, err diff --git a/utils/gorequest/url.go b/utils/gorequest/url.go new file mode 100644 index 00000000..8437b6e7 --- /dev/null +++ b/utils/gorequest/url.go @@ -0,0 +1,78 @@ +package gorequest + +import ( + "net/url" + "strings" +) + +type UriParse struct { + uri string +} + +func NewUri(uri string) *UriParse { + return &UriParse{uri: uri} +} + +// ResponseUrlParse 返回参数 +type ResponseUrlParse struct { + Uri string `json:"uri"` // URI + Urn string `json:"urn"` // URN + Url string `json:"url"` // URL + Scheme string `json:"scheme"` // 协议 + Host string `json:"host"` // 主机 + Hostname string `json:"hostname"` // 主机名 + Port string `json:"port"` // 端口 + Path string `json:"path"` // 路径 + RawQuery string `json:"raw_query"` // 参数 ? + Fragment string `json:"fragment"` // 片段 # +} + +// Parse 解析URl +func (u *UriParse) Parse() (resp ResponseUrlParse) { + parse, err := url.Parse(u.uri) + if err != nil { + return + } + resp.Uri = u.uri + resp.Urn = parse.Host + parse.Path + resp.Url = parse.Scheme + "://" + parse.Host + parse.Path + resp.Scheme = parse.Scheme + resp.Host = parse.Host + resp.Hostname = parse.Hostname() + resp.Port = parse.Port() + resp.Path = parse.Path + resp.RawQuery = parse.RawQuery + resp.Fragment = parse.Fragment + return +} + +// UriFilterExcludeQueryString 过滤掉url中的参数 +func (u *UriParse) UriFilterExcludeQueryString(uri string) string { + URL, _ := url.Parse(uri) + clearUri := strings.ReplaceAll(uri, URL.RawQuery, "") + clearUri = strings.TrimRight(clearUri, "?") + return strings.TrimRight(clearUri, "/") +} + +// LenCode 编码 +func LenCode(s string) string { + escape := url.QueryEscape(s) + return escape +} + +// DeCode 解码 +func DeCode(s string) string { + unescape, _ := url.QueryUnescape(s) + return unescape +} + +// ParseQuery 获取URL参数 https://studygolang.com/articles/2876 +func ParseQuery(s string) map[string][]string { + u, err := url.Parse(s) + if err != nil { + return nil + } + urlParam := u.RawQuery + m, _ := url.ParseQuery(urlParam) + return m +} diff --git a/utils/gorequest/user_agent.go b/utils/gorequest/user_agent.go index 8880373b..94e44b88 100644 --- a/utils/gorequest/user_agent.go +++ b/utils/gorequest/user_agent.go @@ -1,7 +1,6 @@ package gorequest import ( - "fmt" "math/rand" "runtime" "time" @@ -14,20 +13,21 @@ func GetRandomUserAgent() string { } var userAgentList = []string{ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36", // Chrome 2022-02-14 - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0", // Firefox 2022-02-14 - "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", // IE 2022-02-14 - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Safari/537.36 HeyTapBrowser/40.7.35.1", // Chrome 2022-02-14 - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 OnePlusBrowser/30.5.0.8", // Chrome 2022-02-14 - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a1) Gecko/20060814 Firefox/51.0", // Firefox 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15", // Safari 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15", // Safari 2022-04-18 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36 Edg/98.0.1108.51", // Edge 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44", // Chrome 2022-04-18 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", // Chrome 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", // Chrome 2022-04-18 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:97.0) Gecko/20100101 Firefox/97.0", // Firefox 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:99.0) Gecko/20100101 Firefox/99.0", // Firefox 2022-04-18 + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", // IE 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", // Chrome 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.62", // Edge 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0", // Firefox 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Core/1.94.184.400 QQBrowser/11.3.5190.400", // QQ 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", // 360X 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", // 360 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 SE 2.X MetaSr 1.0", // 搜狗 2022-12-03 + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", // Chrome 2022-11-07 + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.35", // Edge 2022-11-07 + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", // Firefox 2022-11-07 + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", // Safari 2022-12-03 + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", // Chrome 2022-12-03 + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.62", // Edge 2022-12-03 + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0", // Firefox 2022-12-03 } // GetRandomUserAgentSystem 获取系统随机UA @@ -44,30 +44,25 @@ func GetRandomUserAgentSystem() string { } var userAgentListWindows = []string{ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36", // Chrome 2022-02-14 - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0", // Firefox 2022-02-14 - "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", // IE 2022-02-14 + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", // IE 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", // Chrome 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.62", // Edge 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0", // Firefox 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Core/1.94.184.400 QQBrowser/11.3.5190.400", // QQ 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", // 360X 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", // 360 2022-12-03 + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 SE 2.X MetaSr 1.0", // 搜狗 2022-12-03 } var userAgentListLinux = []string{ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Safari/537.36 HeyTapBrowser/40.7.35.1", // Chrome 2022-02-14 - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 OnePlusBrowser/30.5.0.8", // Chrome 2022-02-14 - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a1) Gecko/20060814 Firefox/51.0", // Firefox 2022-02-14 + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", // Chrome 2022-11-07 + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.35", // Edge 2022-11-07 + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", // Firefox 2022-11-07 } var userAgentListMac = []string{ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15", // Safari 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15", // Safari 2022-04-18 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36 Edg/98.0.1108.51", // Edge 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44", // Chrome 2022-04-18 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", // Chrome 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", // Chrome 2022-04-18 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:97.0) Gecko/20100101 Firefox/97.0", // Firefox 2022-02-14 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:99.0) Gecko/20100101 Firefox/99.0", // Firefox 2022-04-18 -} - -func DtaUa() string { - str := runtime.Version() - content := str[2 : len(str)-0] - return fmt.Sprintf("Go-dta-request/%v", content) + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", // Safari 2022-12-03 + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", // Chrome 2022-12-03 + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.62", // Edge 2022-12-03 + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0", // Firefox 2022-12-03 }