#4677 合并积分明细下载功能到里程碑分支

Merged
ychao_1983 merged 11 commits from fix-4463 into V20230912 8 months ago
  1. +18
    -0
      models/cloudbrain.go
  2. +15
    -0
      models/resource_specification.go
  3. +98
    -2
      models/reward_operate_record.go
  4. +32
    -0
      models/task_config.go
  5. +4
    -0
      routers/api/v1/api.go
  6. +38
    -0
      routers/reward/point/point.go
  7. +4
    -0
      services/ai_task_service/cluster/cloudbrain_two.go
  8. +103
    -1
      services/reward/record.go

+ 18
- 0
models/cloudbrain.go View File

@@ -320,6 +320,24 @@ type CloudbrainShow struct {
WorkServerNumber int
}

func (c *CloudbrainShow) GetChineseJobType() string {
switch JobType(c.JobType) {
case JobTypeDebug:
return "调试任务"
case JobTypeBenchmark, JobTypeModelSafety:
return "评测任务"
case JobTypeTrain:
return "训练任务"
case JobTypeInference:
return "推理任务"
case JobTypeOnlineInference:
return "在线推理"
case JobTypeSuperCompute:
return "超算任务"
}
return "未知类型"
}

type CloudbrainShow4Action struct {
ID int64
JobID string


+ 15
- 0
models/resource_specification.go View File

@@ -281,6 +281,21 @@ func (s *Specification) loadRelatedSpecs(jobType JobType, hasInternet SpecIntern
s.RelatedSpecs = r
}

func (s *Specification) ToShowString() string {
var specName string
specName += s.ComputeResource + ":" + fmt.Sprint(s.AccCardsNum) + "*" + s.AccCardType + ",CPU:" + fmt.Sprint(s.CpuCores)
if s.GPUMemGiB > 0 {
specName += ",显存:" + fmt.Sprint(s.GPUMemGiB) + "GB"
}
if s.MemGiB > 0 {
specName += ",内存:" + fmt.Sprint(s.MemGiB) + "GB"
}
if s.ShareMemGiB > 0 {
specName += ",共享内存:" + fmt.Sprint(s.ShareMemGiB) + "GB"
}
return specName
}

type GetAvailableCenterIdOpts struct {
UserId int64
JobType JobType


+ 98
- 2
models/reward_operate_record.go View File

@@ -6,6 +6,7 @@ import (
"fmt"
"strconv"
"strings"
"time"
"xorm.io/builder"
)

@@ -29,6 +30,18 @@ func (r SourceType) Name() string {
return ""
}
}
func (r SourceType) ChineseName() string {
switch r {
case SourceTypeAccomplishTask:
return "积分任务"
case SourceTypeAdminOperate:
return "管理员操作"
case SourceTypeRunCloudbrainTask:
return "运行云脑任务"
default:
return ""
}
}

type RewardType string

@@ -260,6 +273,7 @@ type RewardOperateRecordShow struct {
SourceType string
SourceTemplateId string
UserName string
UserId int64
LastOperateDate timeutil.TimeStamp
UnitPrice int64
SuccessCount int
@@ -268,6 +282,88 @@ type RewardOperateRecordShow struct {
AdminLog *RewardAdminLogShow
}

type Column struct {
Name string
Value string
}

func (r *RewardOperateRecordShow) ConvertToExcelColumn() []Column {
source := SourceType(r.SourceType).ChineseName()
action := TaskType(r.SourceTemplateId).ChineseName()
date := time.Unix(int64(r.LastOperateDate), 0).Format("2006-01-02 15:04:05")
var operator = "--"
var detail = r.Remark
if r.AdminLog != nil {
operator = r.AdminLog.CreatorName
}
result := make([]Column, 0)
if r.OperateType == string(OperateTypeIncrease) {

result = []Column{
{Name: "流水号", Value: r.SerialNo},
{Name: "用户名", Value: r.UserName},
{Name: "用户ID", Value: fmt.Sprint(r.UserId)},
{Name: "时间", Value: date},
{Name: "场景", Value: source},
{Name: "积分行为", Value: action},
{Name: "说明", Value: detail},
{Name: "操作者", Value: operator},
{Name: "数量", Value: fmt.Sprint(r.Amount)},
{Name: "积分余额", Value: fmt.Sprint(r.BalanceAfter)},
}
} else if r.OperateType == string(OperateTypeDecrease) {
var status string
switch r.Status {
case OperateStatusOperating:
status = "消耗中"
case OperateStatusSucceeded:
status = "已完成"
case OperateStatusFailed:
status = "失败"
}

var duration = "--"
var displayJobName = "--"
var aiCenter = "--"
var spec = "--"
if r.Cloudbrain != nil {
duration = r.Cloudbrain.Duration
displayJobName = r.Cloudbrain.DisplayJobName
if r.Cloudbrain.ResourceSpec != nil {
spec = "【" + r.Cloudbrain.GetChineseJobType() + "】" + "【" + r.Cloudbrain.ResourceSpec.ToShowString() + "】"
clusterName := ""
switch r.Cloudbrain.ResourceSpec.Cluster {
case "OpenI":
clusterName = "启智集群"
case "C2Net":
clusterName = "智算集群"
}

aiCenter = r.Cloudbrain.ResourceSpec.QueueCode + "(" + clusterName + "-" + r.Cloudbrain.ResourceSpec.AiCenterName + ")"
}
} else {
spec = r.Remark
}
result = []Column{
{Name: "流水号", Value: r.SerialNo},
{Name: "时间", Value: date},
{Name: "状态", Value: status},
{Name: "运行时长", Value: duration},
{Name: "用户名", Value: r.UserName},
{Name: "用户ID", Value: fmt.Sprint(r.UserId)},
{Name: "场景", Value: source},
{Name: "任务名称", Value: displayJobName},
{Name: "智算中心", Value: aiCenter},
{Name: "资源规格", Value: spec},
{Name: "操作者", Value: operator},
{Name: "资源单价", Value: fmt.Sprint(r.UnitPrice)},
{Name: "数量", Value: fmt.Sprint(r.SuccessCount)},
{Name: "总额", Value: fmt.Sprint(r.Amount)},
}
}
return result
}

func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) {
has, err := x.Get(tl)
if err != nil {
@@ -443,7 +539,7 @@ func GetAdminRewardRecordShowList(opts *RewardRecordListOpts) (RewardRecordShowL
err = x.Table("reward_operate_record").Cols("reward_operate_record.source_id", "reward_operate_record.serial_no",
"reward_operate_record.status", "reward_operate_record.operate_type", "reward_operate_record.amount",
"reward_operate_record.loss_amount", "reward_operate_record.remark", "reward_operate_record.source_type", "reward_operate_record.source_template_id",
"reward_operate_record.last_operate_unix as last_operate_date", "public.user.name as user_name",
"reward_operate_record.last_operate_unix as last_operate_date", "public.user.name as user_name", "public.user.id as user_id",
"point_account_log.balance_after").
Join("LEFT", "public.user", "reward_operate_record.user_id = public.user.id").
Join("LEFT", "point_account_log", " reward_operate_record.serial_no = point_account_log.source_id").
@@ -452,7 +548,7 @@ func GetAdminRewardRecordShowList(opts *RewardRecordListOpts) (RewardRecordShowL
err = x.Table("reward_operate_record").Cols("reward_operate_record.source_id", "reward_operate_record.serial_no",
"reward_operate_record.status", "reward_operate_record.operate_type", "reward_operate_record.amount",
"reward_operate_record.loss_amount", "reward_operate_record.remark", "reward_operate_record.source_type", "reward_operate_record.source_template_id",
"reward_operate_record.last_operate_unix as last_operate_date", "public.user.name as user_name",
"reward_operate_record.last_operate_unix as last_operate_date", "public.user.name as user_name", "public.user.id as user_id",
"reward_periodic_task.amount as unit_price", "reward_periodic_task.success_count").
Join("LEFT", "public.user", "reward_operate_record.user_id = public.user.id").
Join("LEFT", "reward_periodic_task", "reward_operate_record.serial_no = reward_periodic_task.operate_serial_no").


+ 32
- 0
models/task_config.go View File

@@ -29,6 +29,38 @@ const (
TaskPushCommits TaskType = "PushCommits"
)

func (t TaskType) ChineseName() string {
switch t {
case TaskCreatePublicRepo:
return "创建公开项目"
case TaskCreateIssue:
return "每日提出任务"
case TaskCreatePullRequest:
return "每日提出PR"
case TaskCommentIssue:
return "发表评论"
case TaskUploadAttachment:
return "上传数据集文件"
case TaskCreateNewModelTask:
return "导入新模型"
case TaskBindWechat:
return "完成微信扫码验证"
case TaskCreateCloudbrainTask:
return "每日运行云脑任务"
case TaskDatasetRecommended:
return "数据集被平台推荐"
case TaskCreateImage:
return "提交新公开镜像"
case TaskImageRecommend:
return "镜像被平台推荐"
case TaskChangeUserAvatar:
return "首次更换头像"
case TaskPushCommits:
return "每日commit"
}
return "--"
}

func GetTaskTypeFromAction(a ActionType) TaskType {
switch a {
case ActionCreateDebugGPUTask,


+ 4
- 0
routers/api/v1/api.go View File

@@ -59,6 +59,7 @@
package v1

import (
"code.gitea.io/gitea/routers/reward/point"
"net/http"
"strings"

@@ -721,6 +722,9 @@ func RegisterRoutes(m *macaron.Macaron) {

m.Group("/reward_point", func() {
m.Get("/is_admin", user.IsRewardPointAdmin)
m.Group("/list", func() {
m.Get("/export", HasRole(models.RewardPointAdmin), point.ExportAdminRewardList)
})
}, reqToken())

m.Group("/attachments", func() {


+ 38
- 0
routers/reward/point/point.go View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net/http"
"net/url"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
@@ -180,6 +181,43 @@ func GetAdminRewardList(ctx *context.Context) {
ctx.JSON(http.StatusOK, response.SuccessWithData(r))
}

func ExportAdminRewardList(ctx *context.Context) {
opts, err := buildAdminRewardRecordListOpts(ctx)
if err != nil {
log.Error("buildAdminRewardRecordListOpts error.%v", err)
ctx.JSON(http.StatusOK, response.ServerError(err.Error()))
return
}

username := ctx.Query("userName")
if username != "" {
user, err := models.GetUserByName(username)
if err != nil {
log.Error("GetUserByName error.%v", err)
if models.IsErrUserNotExist(err) {
ctx.JSON(http.StatusOK, response.ServerError("user not exist"))
} else {
ctx.JSON(http.StatusOK, response.ServerError(err.Error()))
}
return
}
opts.UserId = user.ID
opts.UserName = user.Name
}

xlsx, fileName, err := reward.GenerateAdminRewardExcel(opts)
if err != nil {
log.Error("GenerateAdminReward2Response error.%v", err)
ctx.JSON(http.StatusOK, response.ServerError(err.Error()))
return
}
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName))
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
log.Info("writer reward exel error." + err.Error())
}
}

func buildAdminRewardRecordListOpts(ctx *context.Context) (*models.RewardRecordListOpts, error) {
operateType := ctx.Query("operate")
sourceType := ctx.Query("source")


+ 4
- 0
services/ai_task_service/cluster/cloudbrain_two.go View File

@@ -227,6 +227,10 @@ func (c CloudbrainTwoClusterAdapter) QueryNoteBook(opts entity.JobIdAndVersionId
log.Error("GetNotebook(%s) failed:%v", task.DisplayJobName, err)
return nil, err
}
if result == nil {
log.Error("GetNotebook(%s) from cloudbrain 2 failed:result is empty", task.DisplayJobName)
return nil, errors.New("result is empty")
}
return convertCloudbrainTwo2QueryRes(result), nil
}



+ 103
- 1
services/reward/record.go View File

@@ -3,6 +3,10 @@ package reward
import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"errors"
"fmt"
"github.com/360EntSecGroup-Skylar/excelize/v2"
"math"
)

type RecordResponse struct {
@@ -12,6 +16,105 @@ type RecordResponse struct {
Page int
}

func GenerateAdminRewardExcel(opts *models.RewardRecordListOpts) (*excelize.File, string, error) {
opts.Page = 1
opts.PageSize = 100
recordList, total, err := models.GetAdminRewardRecordShowList(opts)
if err != nil {
log.Error("export increase point excel error.opts=%+v err=%v", opts, err)
return nil, "", err
}
if len(recordList) == 0 {
log.Error("no content to export.opts=%+v ", opts)
return nil, "", errors.New("no content to export")
}
xlsx := excelize.NewFile()
sheetName := "积分获取明细"
if opts.OperateType == models.OperateTypeDecrease {
sheetName = "积分消耗明细"
}
xlsx.NewSheet(sheetName)
xlsx.DeleteSheet("Sheet1")
for i, v := range recordList[0].ConvertToExcelColumn() {
cellRef := fmt.Sprintf("%s%d", excelColumnName(i), 1)
xlsx.SetCellValue(sheetName, cellRef, v.Name)
}

currentRow := int64(1)
currentRow = writeIncreasePointList2Excel(recordList, sheetName, xlsx, currentRow)
totalPage := int64(math.Ceil(float64(total) / float64(opts.PageSize)))
for page := 2; int64(page) <= totalPage; page++ {
opts.Page = page
recordList, _, err = models.GetAdminRewardRecordShowList(opts)
if err != nil {
log.Error("export increase point excel error.opts=%+v err=%v", opts, err)
return nil, "", err
}
if len(recordList) == 0 {
break
}
currentRow = writeIncreasePointList2Excel(recordList, sheetName, xlsx, currentRow)
}
filename := sheetName + ".xlsx"

return xlsx, filename, nil
}

func writeIncreasePointList2Excel(list models.RewardRecordShowList, sheetName string, xlsx *excelize.File, currentRow int64) int64 {
for i := 0; i < len(list); i++ {
currentRow = writeIncreasePoint2Excel(list[i], sheetName, xlsx, currentRow)
}
return currentRow
}

func writeIncreasePoint2Excel(record *models.RewardOperateRecordShow, sheetName string, xlsx *excelize.File, currentRow int64) int64 {
columns := record.ConvertToExcelColumn()
currentRow++
for i := 0; i < len(columns); i++ {
cellRef := fmt.Sprintf("%s%d", excelColumnName(i), currentRow)
xlsx.SetCellValue(sheetName, cellRef, columns[i].Value)
}
return currentRow
}

func initIncreasePointExcel() (string, *excelize.File) {
xlsx := excelize.NewFile()
sheetName := "积分明细"
xlsx.NewSheet(sheetName)
xlsx.DeleteSheet("Sheet1")
dataHeader := getIncreasePointExcelHeader()
for i, v := range dataHeader {
cellRef := fmt.Sprintf("%s%d", excelColumnName(i), 1)
xlsx.SetCellValue(sheetName, cellRef, v)
}
return sheetName, xlsx
}

// 将列索引转换为 Excel 列名
func excelColumnName(index int) string {
dividend := index + 1
columnName := ""
for dividend > 0 {
modulo := (dividend - 1) % 26
columnName = string(rune('A'+modulo)) + columnName
dividend = (dividend - modulo) / 26
}
return columnName
}

func getIncreasePointExcelHeader() []string {
excelHeader := make([]string, 0)
excelHeader = append(excelHeader, "流水号")
excelHeader = append(excelHeader, "用户名")
excelHeader = append(excelHeader, "时间")
excelHeader = append(excelHeader, "场景")
excelHeader = append(excelHeader, "积分行为")
excelHeader = append(excelHeader, "数量")
excelHeader = append(excelHeader, "积分余额")

return excelHeader
}

func GetRewardRecordList(opts *models.RewardRecordListOpts) (*RecordResponse, error) {
var l models.RewardRecordShowList
var n int64
@@ -23,7 +126,6 @@ func GetRewardRecordList(opts *models.RewardRecordListOpts) (*RecordResponse, er
}
if err != nil {
log.Error("GetRewardRecordList error. %v", err)

return nil, err
}
if len(l) == 0 {


Loading…
Cancel
Save