From 692ac2d751f1879b9fdaf462499eabd2f594b3ab Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 2 Feb 2024 11:14:04 +0800 Subject: [PATCH 01/55] =?UTF-8?q?#5180=20=E9=80=9A=E7=94=A8=E4=BB=BB?= =?UTF-8?q?=E5=8A=A11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- entity/ai_task.go | 1 + entity/cluster.go | 37 ++++ models/action.go | 1 + models/cloudbrain.go | 1 + services/ai_task_service/cluster/c2net.go | 80 +++++++++ .../ai_task_service/cluster/cloudbrain_one.go | 4 + .../ai_task_service/cluster/cloudbrain_two.go | 4 + .../ai_task_service/cluster/cluster_base.go | 1 + .../task/grampus_general_task.go | 158 ++++++++++++++++++ 9 files changed, 287 insertions(+) create mode 100644 services/ai_task_service/task/grampus_general_task.go diff --git a/entity/ai_task.go b/entity/ai_task.go index f582fc1ed3..b86dca5960 100644 --- a/entity/ai_task.go +++ b/entity/ai_task.go @@ -348,6 +348,7 @@ type AITaskBaseConfig struct { DatasetsMaxNum int ModelMaxNum int ModelLimitSizeGB int + AutoStopDuration int64 } func GetAITaskConfigFromCloudbrainConfig(config *models.CloudbrainConfig) *AITaskBaseConfig { diff --git a/entity/cluster.go b/entity/cluster.go index bd941d8c32..6c7a24c0c4 100644 --- a/entity/cluster.go +++ b/entity/cluster.go @@ -50,6 +50,43 @@ type CreateNoteBookTaskResponse struct { Status string UserID string } +type CreateGeneralTaskRequest struct { + Name string + Description string + Tasks []GeneralTask + PrimitiveDatasetName string + RepoName string +} + +type GeneralTask struct { + AutoStopDuration int64 + Name string + Capacity int + Queues []models.ResourceQueue + Code []ContainerData + Datasets []ContainerData + PreTrainModel []ContainerData + OutPut []ContainerData + ImageId string + ImageUrl string + ResourceSpecId string + BootFile string + Spec *models.Specification + EnvVariables models.GrampusEnvVarReq +} + +type CreateGeneralTaskResponse struct { + StartedAt int64 + RunSec int64 + CompletedAt int64 + CreatedAt int64 + UpdatedAt int64 + Desc string + JobID string + Name string + Status string + UserID string +} type RestartNoteBookTaskResponse struct { JobId string `json:"newId"` diff --git a/models/action.go b/models/action.go index 85fa2c184e..6178553c0b 100755 --- a/models/action.go +++ b/models/action.go @@ -78,6 +78,7 @@ const ( ActionCreateGrampusMETAXDebugTask //49 ActionCreateGrampusGPUInferenceTask //50 ActionCreateGrampusILUVATARInferenceTask //51 + ActionCreateAITask ) // Action represents user operation type and other information to diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 79facc7148..ca38f09dc8 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -80,6 +80,7 @@ const ( JobTypeInference JobType = "INFERENCE" JobTypeOnlineInference JobType = "ONLINEINFERENCE" JobTypeSuperCompute JobType = "HPC" + JobTypeGeneral JobType = "GENERAL" //notebook ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 diff --git a/services/ai_task_service/cluster/c2net.go b/services/ai_task_service/cluster/c2net.go index f75969d7ed..ef3d9846da 100644 --- a/services/ai_task_service/cluster/c2net.go +++ b/services/ai_task_service/cluster/c2net.go @@ -67,6 +67,25 @@ func (c C2NetClusterAdapter) CreateOnlineInfer(req entity.CreateNoteBookTaskRequ return convertGrampus2NoteBookRes(jobResult), nil } +func (c C2NetClusterAdapter) CreateGeneralTask(req entity.CreateGeneralTaskRequest) (*entity.CreateGeneralTaskResponse, error) { + log.Info("start to CreateGeneralTask req=%+v", req) + newReq, err := convertGeneralTaskReq2Grampus(req) + if err != nil { + log.Error("CreateGeneralTask err.req=%+v err=%v", req, err) + return nil, err + } + jobResult, err := grampus.CreateNotebookJob(newReq) + if err != nil { + log.Error("CreateGeneralTask failed: %v", err.Error()) + return nil, err + } + if jobResult.ErrorCode > 0 { + log.Error("CreateGeneralTask call grampus err.req.Name = %s ErrorCode = %d ErrorMsg = %s", req.Name, jobResult.ErrorCode, jobResult.ErrorMsg) + return nil, errors.New(fmt.Sprintf("CreateGeneralTask err[%d%s]", jobResult.ErrorCode, jobResult.ErrorMsg)) + } + return convertGrampus2GeneralTaskRes(jobResult), nil +} + func (c C2NetClusterAdapter) GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { processType := req.ComputeSource.FullName images, err := grampus.GetImages(processType, string(req.JobType)) @@ -166,6 +185,20 @@ func convertOnlineInfer2Grampus(req entity.CreateNoteBookTaskRequest) (models.Cr return models.CreateGrampusInferenceRequest{Name: req.Name, Tasks: tasks}, nil } +func convertGeneralTaskReq2Grampus(req entity.CreateGeneralTaskRequest) (models.CreateGrampusNotebookRequest, error) { + tasks := make([]models.GrampusNotebookTask, len(req.Tasks)) + for i := 0; i < len(req.Tasks); i++ { + t := req.Tasks[i] + task, err := convertGeneralTask2Grampus(t) + if err != nil { + return models.CreateGrampusNotebookRequest{}, err + } + tasks[i] = task + } + + return models.CreateGrampusNotebookRequest{Name: req.Name, Tasks: tasks}, nil +} + func generateCommand(repoName, bootFile, datasetName string) string { //prepare @@ -280,6 +313,37 @@ func convertNoteBookTask2Grampus(t entity.NoteBookTask, command string) (models. }, nil } +func convertGeneralTask2Grampus(t entity.GeneralTask) (models.GrampusNotebookTask, error) { + code := models.GrampusDataset{} + codeArray := convertContainerArray2GrampusArray(t.Code) + if codeArray != nil && len(codeArray) > 0 { + code = codeArray[0] + } + output := models.GrampusDataset{} + outputArray := convertContainerArray2GrampusArray(t.OutPut) + if outputArray != nil && len(outputArray) > 0 { + output = outputArray[0] + } + centerIds, err := getGrampusAvailableCenterIds(t.Queues, t.ImageId, *models.GetComputeSourceInstance(t.Spec.ComputeResource), models.JobTypeDebug) + if err != nil { + return models.GrampusNotebookTask{}, err + } + return models.GrampusNotebookTask{ + Name: t.Name, + ResourceSpecId: t.Spec.SourceSpecId, + ImageId: t.ImageId, + ImageUrl: t.ImageUrl, + Datasets: convertContainerArray2GrampusArray(t.Datasets), + PreTrainModel: convertContainerArray2GrampusArray(t.PreTrainModel), + Code: code, + OutPut: output, + EnvVariables: t.EnvVariables, + AutoStopDuration: t.AutoStopDuration, + Capacity: t.Capacity, + CenterID: centerIds, + }, nil +} + func getGrampusAvailableCenterIds(queues []models.ResourceQueue, imageId string, computeSource models.ComputeSource, jobType models.JobType) ([]string, error) { if len(queues) == 0 { return []string{}, nil @@ -385,6 +449,22 @@ func convertGrampus2NoteBookRes(res *models.GrampusNotebookResponse) *entity.Cre } } +func convertGrampus2GeneralTaskRes(res *models.GrampusNotebookResponse) *entity.CreateGeneralTaskResponse { + jobInfo := res.JobInfo + return &entity.CreateGeneralTaskResponse{ + StartedAt: jobInfo.StartedAt, + RunSec: jobInfo.RunSec, + CompletedAt: jobInfo.CompletedAt, + CreatedAt: jobInfo.CreatedAt, + UpdatedAt: jobInfo.UpdatedAt, + Desc: jobInfo.Desc, + JobID: jobInfo.JobID, + Name: jobInfo.Name, + Status: jobInfo.Status, + UserID: jobInfo.UserID, + } +} + func (c C2NetClusterAdapter) RestartNoteBook(jobId string) (*entity.RestartNoteBookTaskResponse, error) { res, err := grampus.RestartNotebookJob(jobId) if err != nil { diff --git a/services/ai_task_service/cluster/cloudbrain_one.go b/services/ai_task_service/cluster/cloudbrain_one.go index f67c95ff82..fa6a88e9a5 100644 --- a/services/ai_task_service/cluster/cloudbrain_one.go +++ b/services/ai_task_service/cluster/cloudbrain_one.go @@ -38,6 +38,10 @@ func (c CloudbrainOneClusterAdapter) CreateOnlineInfer(req entity.CreateNoteBook return nil, nil } +func (c CloudbrainOneClusterAdapter) CreateGeneralTask(req entity.CreateGeneralTaskRequest) (*entity.CreateGeneralTaskResponse, error) { + return nil, nil +} + func (c CloudbrainOneClusterAdapter) GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { return nil, true, nil } diff --git a/services/ai_task_service/cluster/cloudbrain_two.go b/services/ai_task_service/cluster/cloudbrain_two.go index e11513a491..0273defdcf 100644 --- a/services/ai_task_service/cluster/cloudbrain_two.go +++ b/services/ai_task_service/cluster/cloudbrain_two.go @@ -102,6 +102,10 @@ func (c CloudbrainTwoClusterAdapter) CreateOnlineInfer(req entity.CreateNoteBook return nil, nil } +func (c CloudbrainTwoClusterAdapter) CreateGeneralTask(req entity.CreateGeneralTaskRequest) (*entity.CreateGeneralTaskResponse, error) { + return nil, nil +} + var cloudbrainTwoNotebookImages []entity.ClusterImage func (c CloudbrainTwoClusterAdapter) GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { diff --git a/services/ai_task_service/cluster/cluster_base.go b/services/ai_task_service/cluster/cluster_base.go index e877dfa2ad..0f623bf560 100644 --- a/services/ai_task_service/cluster/cluster_base.go +++ b/services/ai_task_service/cluster/cluster_base.go @@ -51,4 +51,5 @@ type ClusterAdapter interface { GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) GetTrainImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) CreateOnlineInfer(req entity.CreateNoteBookTaskRequest) (*entity.CreateNoteBookTaskResponse, error) + CreateGeneralTask(req entity.CreateGeneralTaskRequest) (*entity.CreateGeneralTaskResponse, error) } diff --git a/services/ai_task_service/task/grampus_general_task.go b/services/ai_task_service/task/grampus_general_task.go new file mode 100644 index 0000000000..ce7e9e51f5 --- /dev/null +++ b/services/ai_task_service/task/grampus_general_task.go @@ -0,0 +1,158 @@ +package task + +import ( + "strings" + + "code.gitea.io/gitea/entity" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/ai_task_service/context" +) + +type GrampusGeneralTaskTemplate struct { + DefaultAITaskTemplate +} + +func init() { + t := &GrampusGeneralTaskTemplate{ + DefaultAITaskTemplate: DefaultAITaskTemplate{ + ClusterType: entity.C2Net, + JobType: models.JobTypeGeneral, + Config: GetGeneralTaskConfig, + }, + } + RegisterTask(models.JobTypeGeneral, entity.C2Net, t) +} + +func GetGeneralTaskConfig(opts entity.AITaskConfigKey) *entity.AITaskBaseConfig { + codePath := "/code" + datasetPath := "/dataset" + pretrainModelPath := "/pretrainmodel" + + config := &entity.AITaskBaseConfig{ + ContainerSteps: map[entity.ContainerDataType]*entity.ContainerBuildOpts{ + entity.ContainerCode: { + ContainerPath: codePath, + ReadOnly: false, + AcceptStorageType: []entity.StorageType{entity.MINIO, entity.OBS}, + VolumeFolder: true, + }, + entity.ContainerDataset: { + ContainerPath: datasetPath, + ReadOnly: true, + AcceptStorageType: []entity.StorageType{entity.MINIO, entity.OBS}, + }, + entity.ContainerPreTrainModel: { + ContainerPath: pretrainModelPath, + ReadOnly: true, + AcceptStorageType: []entity.StorageType{entity.MINIO, entity.OBS}, + }, + }, + ActionType: models.ActionCreateAITask, + DatasetsLimitSizeGB: setting.DebugAttachSize, + DatasetsMaxNum: setting.MaxDatasetNum, + ModelLimitSizeGB: setting.DEBUG_MODEL_SIZE_LIMIT_GB, + ModelMaxNum: setting.DEBUG_MODEL_NUM_LIMIT, + } + return config +} + +func (t GrampusGeneralTaskTemplate) Create(ctx *context.CreationContext) (*entity.CreateTaskRes, *response.BizError) { + c := &CreateOperator{} + err := c.Next(t.CheckParamFormat). + Next(t.CheckMultiRequest). + Next(t.CheckDisplayJobName). + //Next(t.CheckNotebookCount). + Next(t.LoadSpec). + Next(t.CheckPointBalance). + Next(t.CheckDatasets). + Next(t.CheckBranchExists). + Next(t.CheckModels). + Next(t.InsertCloudbrainRecord4Async). + AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async). + Operate(ctx) + if err != nil { + log.Error("create GrampusNoteBookTask err.%v", err) + return nil, err + } + return &entity.CreateTaskRes{ID: ctx.NewCloudbrain.ID}, nil +} + +func (g GrampusGeneralTaskTemplate) CallCreationAPI(ctx *context.CreationContext) *response.BizError { + c := g.GetMyCluster() + if c == nil { + log.Error("Get cluster failed") + return response.SYSTEM_ERROR + } + form := ctx.Request + imageUrl := strings.TrimSpace(form.ImageUrl) + if form.ImageID != "" { + imageUrl = "" + } + req := entity.CreateGeneralTaskRequest{ + Name: form.JobName, + Tasks: []entity.GeneralTask{ + { + Name: form.JobName, + ResourceSpecId: ctx.Spec.SourceSpecId, + ImageId: form.ImageID, + ImageUrl: imageUrl, + Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), + PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), + Code: ctx.GetContainerDataArray(entity.ContainerCode), + EnvVariables: models.GrampusEnvVarReq{}, + Capacity: setting.Capacity, + Queues: ctx.Queues, + Spec: ctx.Spec, + }, + }, + } + createTime := timeutil.TimeStampNow() + res, err := c.CreateGeneralTask(req) + if err != nil { + log.Error("GrampusGeneralTaskTemplate CreateGeneralTask err.req=%+v err=%v", req, err) + return response.NewBizError(err) + } + if res.JobID == "" { + log.Error("GrampusGeneralTaskTemplate CreateGeneralTask failed.Cloudbrain.JobID=%s", ctx.SourceCloudbrain.JobID) + return response.CREATE_FAILED + } + ctx.Response = &entity.CreationResponse{ + JobID: res.JobID, + Status: res.Status, + CreateTime: createTime, + } + return nil +} + +func (g GrampusGeneralTaskTemplate) GetImages(computeSource models.ComputeSource, centerId ...string) ([]entity.ClusterImage, bool, *response.BizError) { + c := g.GetMyCluster() + if c == nil { + log.Error("Get cluster failed") + return nil, false, response.SYSTEM_ERROR + } + + var images []entity.ClusterImage + var customFlag bool + var err error + if g.JobType == models.JobTypeDebug { + images, customFlag, err = c.GetNotebookImages(entity.GetImageReq{ + ComputeSource: computeSource, + JobType: g.JobType, + }, centerId...) + } else { + images, customFlag, err = c.GetTrainImages(entity.GetImageReq{ + ComputeSource: computeSource, + JobType: g.JobType, + }, centerId...) + } + if err != nil { + log.Error("GetImages err.computeSource=%s err =%v", computeSource.Name, err) + return nil, false, response.NewBizError(err) + } + return images, customFlag, nil +} -- 2.34.1 From 5eb4580996fbfee0f8ebcd94750bdff8ca54fe3c Mon Sep 17 00:00:00 2001 From: chenshihai Date: Sun, 4 Feb 2024 16:31:49 +0800 Subject: [PATCH 02/55] add general task --- routers/repo/grampus_general.go | 28 +++++++ routers/routes/routes.go | 8 ++ templates/repo/grampus/general/list.tmpl | 8 ++ templates/repo/grampus/general/new.tmpl | 1 + templates/repo/grampus/general/show.tmpl | 1 + web_src/vuepages/const/index.js | 2 +- web_src/vuepages/langs/config/en-US.js | 2 + web_src/vuepages/langs/config/zh-CN.js | 2 + web_src/vuepages/pages/cloudbrain/configs.js | 77 +++++++++++++++++++- 9 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 routers/repo/grampus_general.go create mode 100644 templates/repo/grampus/general/list.tmpl create mode 100644 templates/repo/grampus/general/new.tmpl create mode 100644 templates/repo/grampus/general/show.tmpl diff --git a/routers/repo/grampus_general.go b/routers/repo/grampus_general.go new file mode 100644 index 0000000000..e8e04f0276 --- /dev/null +++ b/routers/repo/grampus_general.go @@ -0,0 +1,28 @@ +package repo + +import ( + "net/http" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" +) + +const ( + tplGrampusGeneralIndex base.TplName = "repo/grampus/general/list" + tplGrampusGeneralShow base.TplName = "repo/grampus/general/show" + tplGrampusGeneralNew base.TplName = "repo/grampus/general/new" +) + +func GrampusGeneralNew(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + ctx.HTML(http.StatusOK, tplGrampusGeneralNew) +} + +func GrampusGeneralShow(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + ctx.HTML(http.StatusOK, tplGrampusGeneralShow) +} + +func GrampusGeneralIndex(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + ctx.HTML(http.StatusOK, tplGrampusGeneralIndex) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 19c2f54704..c9ad7eaf66 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1370,6 +1370,14 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.GrampusOnlineInferNew) }) + m.Group("/general", func() { + m.Get("", reqRepoCloudBrainReader, repo.GrampusGeneralIndex) + m.Group("/:id", func() { + m.Get("", reqRepoCloudBrainReader, repo.GrampusGeneralShow) + }) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.GrampusGeneralNew) + }) + m.Group("/train-job", func() { m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.GrampusTrainJobShow) diff --git a/templates/repo/grampus/general/list.tmpl b/templates/repo/grampus/general/list.tmpl new file mode 100644 index 0000000000..294d59d4a3 --- /dev/null +++ b/templates/repo/grampus/general/list.tmpl @@ -0,0 +1,8 @@ +{{template "base/head" .}} + +
+ {{template "repo/header" .}} +
+
+ +{{template "base/footer" .}} diff --git a/templates/repo/grampus/general/new.tmpl b/templates/repo/grampus/general/new.tmpl new file mode 100644 index 0000000000..e36addbb2e --- /dev/null +++ b/templates/repo/grampus/general/new.tmpl @@ -0,0 +1 @@ +{{ template "repo/cloudbrain/cloudbraincreate" .}} diff --git a/templates/repo/grampus/general/show.tmpl b/templates/repo/grampus/general/show.tmpl new file mode 100644 index 0000000000..73ba5b964e --- /dev/null +++ b/templates/repo/grampus/general/show.tmpl @@ -0,0 +1 @@ +{{ template "repo/cloudbrain/cloudbraindetail" .}} diff --git a/web_src/vuepages/const/index.js b/web_src/vuepages/const/index.js index 1c832260e0..bd354aae86 100644 --- a/web_src/vuepages/const/index.js +++ b/web_src/vuepages/const/index.js @@ -6,7 +6,7 @@ export const POINT_ACTIONS = [ { k: 'CreatePublicRepo', v: i18n.t('createPublicProject') }, { k: 'CreateIssue', v: i18n.t('dailyPutforwardTasks') }, { k: 'CreatePullRequest', v: i18n.t('dailyPR') }, { k: 'CommentIssue', v: i18n.t('comment') }, { k: 'UploadAttachment', v: i18n.t('uploadDatasetFile') }, { k: 'CreateNewModelTask', v: i18n.t('importNewModel') }, { k: 'BindWechat', v: i18n.t('completeWechatCodeScanningVerification') }, { k: 'CreateCloudbrainTask', v: i18n.t('dailyRunCloudbrainTasks') }, { k: 'DatasetRecommended', v: i18n.t('datasetRecommendedByThePlatform') }, { k: 'CreateImage', v: i18n.t('submitNewPublicImage') }, { k: 'ImageRecommend', v: i18n.t('imageRecommendedByThePlatform') }, { k: 'ChangeUserAvatar', v: i18n.t('firstChangeofAvatar') }, { k: 'PushCommits', v: i18n.t('dailyCommit') }, ]; -export const JOB_TYPE = [{ k: 'DEBUG', v: i18n.t('debugTask') }, { k: 'TRAIN', v: i18n.t('trainTask') }, { k: 'INFERENCE', v: i18n.t('inferenceTask') }, { k: 'BENCHMARK', v: i18n.t('benchmarkTask') }, { k: 'ONLINEINFERENCE', v: i18n.t('onlineinfer') }, { k: 'HPC', v: i18n.t('superComputeTask') }]; +export const JOB_TYPE = [{ k: 'DEBUG', v: i18n.t('debugTask') }, { k: 'TRAIN', v: i18n.t('trainTask') }, { k: 'INFERENCE', v: i18n.t('inferenceTask') }, { k: 'BENCHMARK', v: i18n.t('benchmarkTask') }, { k: 'ONLINEINFERENCE', v: i18n.t('onlineinfer') }, { k: 'HPC', v: i18n.t('superComputeTask') }, { k: 'GENERAL', v: i18n.t('generalTask') }]; // 资源管理 export const CLUSTERS = [{ k: 'OpenI', v: i18n.t('resourcesManagement.OpenI') }, { k: 'C2Net', v: i18n.t('resourcesManagement.C2Net') }]; diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js index 167afd5bd5..d42ec59fb6 100644 --- a/web_src/vuepages/langs/config/en-US.js +++ b/web_src/vuepages/langs/config/en-US.js @@ -42,6 +42,7 @@ const en = { benchmarkTask: 'Benchmark Task', onlineinfer: "Online Inference", superComputeTask: "HPC Task", + generalTask: "General Task", createPublicProject: 'Create Public Projects', dailyPutforwardTasks: 'Daily Put Forward Tasks', dailyPR: 'Daily PR', @@ -664,6 +665,7 @@ const en = { bootFileTips: 'The startup file is the entry file that your program executes, and it must be a file ending in .py', viewSample: 'View sample', tabTitOnlineInference: 'Online Inference', + tabTitGeneral: 'General Task', allResultDownload: 'All result download', downloadDisplayMaxCountTips: 'Display up to {count} files or folders in a single directory', file_sync_ing: "File synchronization in progress, please wait", diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js index 38f43ada95..b375d353d6 100644 --- a/web_src/vuepages/langs/config/zh-CN.js +++ b/web_src/vuepages/langs/config/zh-CN.js @@ -41,6 +41,7 @@ const zh = { benchmarkTask: "评测任务", onlineinfer: "在线推理", superComputeTask: "超算任务", + generalTask: "通用任务", createPublicProject: "创建公开项目", dailyPutforwardTasks: "每日提出任务", dailyPR: "每日提出PR", @@ -679,6 +680,7 @@ const zh = { bootFileTips: '启动文件是您程序执行的入口文件,必须是以.py结尾的文件。比如train.py、main.py、example/train.py、case/main.py。', viewSample: '查看样例', tabTitOnlineInference: '在线推理', + tabTitGeneral: '通用任务', allResultDownload: '全部结果下载', downloadDisplayMaxCountTips: '单目录下最多显示{count}个文件或文件夹', file_sync_ing: "文件同步中,请稍侯", diff --git a/web_src/vuepages/pages/cloudbrain/configs.js b/web_src/vuepages/pages/cloudbrain/configs.js index c8ea00e04c..109a98c247 100644 --- a/web_src/vuepages/pages/cloudbrain/configs.js +++ b/web_src/vuepages/pages/cloudbrain/configs.js @@ -567,6 +567,36 @@ export const CreatePageConfigs = { }], }] }], + // 通用任务 + 'GENERAL': [{ + url: 'grampus/general', + clusters: ['C2Net'], + 'C2Net': [{ + url: 'grampus/general/create?type=0', + computerResouces: ['GPU'], + 'GPU': [{ + url: 'grampus/general/create?type=0', + clusterType: 'C2Net', + tips2: i18n.t('cloudbrainObj.pathTips3', { + code: '/code', + dataset: '/dataset', + model: '/pretrainmodel', + }), + hideTips2: true, + form: { + taskName: { required: true, }, + taskDescr: { required: false, }, + taskType: {}, + branchName: {}, + model: { required: false, multiple: true, useExceedSize: true }, + imagev1: { required: true, type: 2 }, + dataset: { required: false, type: 0, useExceedSize: true }, + networkType: { required: true }, + spec: {}, + }, + }], + }] + }], }; export const ListPageConfigs = { @@ -576,7 +606,8 @@ export const ListPageConfigs = { { key: 'TRAIN', label: i18n.t('cloudbrainObj.tabTitTrain'), url: 'modelarts/train-job?listType=all', }, { key: 'INFERENCE', label: i18n.t('cloudbrainObj.tabTitInference'), url: 'modelarts/inference-job', }, { key: 'BENCHMARK', label: i18n.t('cloudbrainObj.tabTitBenchmark'), url: 'cloudbrain/benchmark', }, - { key: 'ONLINEINFERENCE', label: i18n.t('cloudbrainObj.tabTitOnlineInference'), url: 'grampus/onlineinfer', } + { key: 'ONLINEINFERENCE', label: i18n.t('cloudbrainObj.tabTitOnlineInference'), url: 'grampus/onlineinfer', }, + { key: 'GENERAL', label: i18n.t('cloudbrainObj.tabTitGeneral'), url: 'grampus/general', } ], pages: [{ jobType: 'DEBUG', @@ -632,6 +663,18 @@ export const ListPageConfigs = { emptyTip0: true, emptyTip2: i18n.t('cloudbrainObj.onlineInferEmptyTip2'), emptyTip3: i18n.t('cloudbrainObj.debugTaskEmptyTip3', { url: 'https://openi.pcl.ac.cn/docs/index.html#/cloudbrain/infer/online-inference' }), + }, { + jobType: 'GENERAL', + sortList: getSortList(['', 'GPU']), + jobTypeName: getListValueWithKey(JOB_TYPE, 'GENERAL'), + url: 'grampus/general', + createUrl: 'grampus/general/create?type=0', + columns: ['taskName', 'status', 'createTime', 'clusterAndComputeResource', 'creator'], + operations: ['debug', 'redebug', 'stop', 'delete', 'debugMore'], + emptyTitle: i18n.t('cloudbrainObj.debugTaskEmptyTitle'), + emptyTip1: i18n.t('cloudbrainObj.debugTaskEmptyTip1'), + emptyTip2: i18n.t('cloudbrainObj.debugTaskEmptyTip2'), + emptyTip3: i18n.t('cloudbrainObj.debugTaskEmptyTip3', { url: 'https://openi.pcl.ac.cn/docs/index.html#/cloudbrain/debug/debug' }), }] }; @@ -1290,6 +1333,38 @@ export const DetailPageConfigs = { }], }] }], + // 通用任务 + 'GENERAL': [{ + listUrl: 'grampus/general', + clusters: ['C2Net'], + 'C2Net': [{ + 'GPU': [{ + detailUrl: 'grampus/general/', + summary: [], + operations: [], + tabs: [{ + name: 'configInfo', + fields: [ + 'taskName', 'imagev1', + 'status', 'spec', + 'creator', 'aiCenter', + 'branch', 'hasInternet', + 'computerRes', 'codePath', + 'createTime', 'datasetPath', + 'startTime', 'modelPath', + 'endTime', '', + 'duration', '', + 'descr', '', + 'failedReason', + 'dataset', + 'modelList', + ] + }, { + name: 'operationProfile' + }], + }], + }] + }], }; export const getCreatePageConfigs = (url) => { -- 2.34.1 From bea13482f14763749424ec4f174d1486da5827e5 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Sun, 4 Feb 2024 17:30:25 +0800 Subject: [PATCH 03/55] update general task config --- web_src/vuepages/pages/cloudbrain/configs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/vuepages/pages/cloudbrain/configs.js b/web_src/vuepages/pages/cloudbrain/configs.js index 109a98c247..742b0ecefc 100644 --- a/web_src/vuepages/pages/cloudbrain/configs.js +++ b/web_src/vuepages/pages/cloudbrain/configs.js @@ -589,7 +589,7 @@ export const CreatePageConfigs = { taskType: {}, branchName: {}, model: { required: false, multiple: true, useExceedSize: true }, - imagev1: { required: true, type: 2 }, + imagev2: { required: true, relatedSpec: true }, dataset: { required: false, type: 0, useExceedSize: true }, networkType: { required: true }, spec: {}, -- 2.34.1 From 15e7ec5e5915ac6b08e4d7262bf51bb5352e2078 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Sun, 4 Feb 2024 17:43:21 +0800 Subject: [PATCH 04/55] fix bug --- .../ai_task_service/task/grampus_general_task.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/services/ai_task_service/task/grampus_general_task.go b/services/ai_task_service/task/grampus_general_task.go index ce7e9e51f5..7233987d6c 100644 --- a/services/ai_task_service/task/grampus_general_task.go +++ b/services/ai_task_service/task/grampus_general_task.go @@ -139,17 +139,10 @@ func (g GrampusGeneralTaskTemplate) GetImages(computeSource models.ComputeSource var images []entity.ClusterImage var customFlag bool var err error - if g.JobType == models.JobTypeDebug { - images, customFlag, err = c.GetNotebookImages(entity.GetImageReq{ - ComputeSource: computeSource, - JobType: g.JobType, - }, centerId...) - } else { - images, customFlag, err = c.GetTrainImages(entity.GetImageReq{ - ComputeSource: computeSource, - JobType: g.JobType, - }, centerId...) - } + images, customFlag, err = c.GetNotebookImages(entity.GetImageReq{ + ComputeSource: computeSource, + JobType: models.JobTypeDebug, + }, centerId...) if err != nil { log.Error("GetImages err.computeSource=%s err =%v", computeSource.Name, err) return nil, false, response.NewBizError(err) -- 2.34.1 From 1ace4c0e68c0646beb008afb139b7f1ed11fe898 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 7 Feb 2024 14:50:05 +0800 Subject: [PATCH 05/55] update ai tasks list page --- templates/admin/cloudbrain/list.tmpl | 577 +----------------- templates/user/dashboard/cloudbrains.tmpl | 528 +--------------- .../vuepages/pages/cloudbrain/tasks/index.vue | 529 ++++++++++++++++ .../cloudbrain/tasks/vp-cloudbrain-tasks.js | 17 + web_src/vuepages/pages/cloudbrain/tools.js | 3 + 5 files changed, 556 insertions(+), 1098 deletions(-) mode change 100755 => 100644 templates/admin/cloudbrain/list.tmpl create mode 100644 web_src/vuepages/pages/cloudbrain/tasks/index.vue create mode 100644 web_src/vuepages/pages/cloudbrain/tasks/vp-cloudbrain-tasks.js diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl old mode 100755 new mode 100644 index 7d54a9e0f5..01cf7ecbc5 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -1,579 +1,10 @@ {{template "base/head" .}} - - -
-
-
-
-
-
-
-
-
- -
+
- {{template "admin/navbar" .}}
- {{template "base/alert" .}} -
-
- {{template "admin/cloudbrain/search" .}} - -
- -
- -
-
-
- {{$.i18n.Tr "repo.cloudbrain_task"}} -
- -
- {{$.i18n.Tr "repo.modelarts.cluster"}} -
-
- {{$.i18n.Tr "repo.modelarts.status"}} -
-
- {{$.i18n.Tr "repo.cloudbrain_task_type"}} -
-
- {{$.i18n.Tr "repo.modelarts.createtime"}} -
-
- {{$.i18n.Tr "repo.cloudbrain_status_runtime"}} -
-
- {{$.i18n.Tr "repo.modelarts.computing_resources"}} -
- -
- {{$.i18n.Tr "repo.modelarts.ai_center"}} -
- -
- {{$.i18n.Tr "repo.modelarts.card_type"}} -
-
- {{$.i18n.Tr "repo.cloudbrain_creator"}} -
-
- {{$.i18n.Tr "repository"}} -
-
- {{.i18n.Tr "admin.cloudbrain.cloudbrain_name"}} -
-
- {{$.i18n.Tr "repo.cloudbrain_operate"}} -
-
-
- {{range .Tasks}} - {{if .Repo}} -
-
- - {{$JobID := '0'}} - {{if eq .JobType "DEBUG" "TRAIN" "INFERENCE" "HPC" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY" "SNN4ECOSET" "SIM2BRAIN_SNN" "ONLINEINFERENCE"}} - {{$JobID = .Cloudbrain.ID}} - {{else}} - {{$JobID = .JobID}} - {{end}} - -
- {{if eq .JobType "DEBUG"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "HPC"}} - - {{.DisplayJobName}} - - {{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE") (eq .JobType "SNN4ECOSET") (eq .JobType "SIM2BRAIN_SNN")}} - - {{.DisplayJobName}} - - {{else if eq .JobType "INFERENCE"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "TRAIN"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "BENCHMARK" "MODELSAFETY"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "ONLINEINFERENCE"}} - - {{.DisplayJobName}} - - {{end}} -
- -
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} -
- -
- - - - {{.Status}} - {{if eq .Status "WAITING"}}{{end}} - - -
- -
- {{.JobType}} -
- - -
- {{TimeSinceUnix1 .Cloudbrain.CreatedUnix}} -
- -
- {{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}} -
- -
- {{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}} -
- -
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} -
- -
- -
- - -
- {{if .User.Name}} - - {{else}} - - {{end}} -
- - - -
- - {{.JobName}} - -
-
- {{if eq .JobType "DEBUG"}} -
-
- {{$.CsrfTokenHtml}} - {{if eq .Status "RUNNING" "WAITING" "CREATING" "STARTING"}} - - {{$.i18n.Tr "repo.debug"}} - - {{else}} - {{if not .BootFile}} - - {{$.i18n.Tr "repo.debug_again"}} - - {{end}} - {{end}} -
-
- {{end}} - {{if eq .JobType "ONLINEINFERENCE"}} -
-
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.online_debug"}} - -
-
- {{end}} - {{if eq .JobType "HPC"}} -
-
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.start_use"}} - -
-
- {{end}} - -
- {{if eq .JobType "MODELSAFETY"}} -
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.stop"}} - -
- {{else}} - {{if eq .JobType "DEBUG" "HPC" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE" "SNN4ECOSET" "SIM2BRAIN_SNN" "ONLINEINFERENCE"}} -
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.stop"}} - -
- {{else}} - - {{$.i18n.Tr "repo.stop"}} - - {{end}} - {{end}} -
- - {{if and (eq .JobType "TRAIN") (not .FineTune)}} - - {{end}} - - {{if eq .JobType "MODELSAFETY"}} -
- {{$.CsrfTokenHtml}} - - - {{$.i18n.Tr "repo.delete"}} - -
- {{else}} -
- {{$.CsrfTokenHtml}} - - - {{$.i18n.Tr "repo.delete"}} - -
- {{end}} -
-
-
- {{else}} - {{$JobID := '0'}} - {{if eq .JobType "DEBUG" "TRAIN" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "SNN4ECOSET" "SIM2BRAIN_SNN"}} - {{$JobID = .Cloudbrain.ID}} - {{else}} - {{$JobID = .JobID}} - {{end}} -
-
- -
- {{if eq .JobType "DEBUG"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "INFERENCE"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "TRAIN"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "BENCHMARK"}} - - {{.DisplayJobName}} - - {{end}} -
- -
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} -
- -
- - - - {{.Status}} - {{if eq .Status "WAITING"}}{{end}} - - -
- -
- {{.JobType}} -
- - -
- {{TimeSinceUnix1 .Cloudbrain.CreatedUnix}} -
- -
- {{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}} -
- -
- {{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}} -
- -
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} -
- -
- - {{if .CardType}}{{.CardType}}{{else}}--{{end}} - -
- -
- {{if .User.Name}} - - {{else}} - - {{end}} -
- -
- -- -
- -
- - {{.JobName}} - - -
-
- {{if eq .JobType "DEBUG"}} -
-
- {{$.CsrfTokenHtml}} - {{if eq .Status "RUNNING" "WAITING" "CREATING" "STARTING"}} - - {{$.i18n.Tr "repo.debug"}} - - {{else}} - - {{$.i18n.Tr "repo.debug_again"}} - - {{end}} -
-
- {{end}} - - - -
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.delete"}} - -
-
-
-
- - {{end}} - {{end}} -
-
-
-
- - -
-
-
-
-
- -
- -
-
- -{{template "base/footer" .}} \ No newline at end of file + +{{template "base/footer" .}} diff --git a/templates/user/dashboard/cloudbrains.tmpl b/templates/user/dashboard/cloudbrains.tmpl index 5d902a441e..cc61f34a7e 100755 --- a/templates/user/dashboard/cloudbrains.tmpl +++ b/templates/user/dashboard/cloudbrains.tmpl @@ -1,529 +1,7 @@ {{template "base/head" .}} - -
- +
- - {{template "admin/cloudbrain/search_dashboard" .}} -
- {{template "base/alert" .}} -
-
-
- -
- -
-
-
- {{$.i18n.Tr "repo.cloudbrain_task"}} -
- -
- {{$.i18n.Tr "repo.modelarts.cluster"}} -
-
- {{$.i18n.Tr "repo.modelarts.status"}} -
-
- {{$.i18n.Tr "repo.cloudbrain_task_type"}} -
-
- {{$.i18n.Tr "repo.modelarts.createtime"}} -
-
- {{$.i18n.Tr "repo.cloudbrain_status_runtime"}} -
-
- {{$.i18n.Tr "repo.modelarts.computing_resources"}} -
- - -
- {{$.i18n.Tr "repo.modelarts.ai_center"}} -
- -
- {{$.i18n.Tr "repo.modelarts.card_type"}} -
-
- {{$.i18n.Tr "repository"}} -
- -
- {{$.i18n.Tr "repo.cloudbrain_operate"}} -
-
-
- {{range .Tasks}} - {{if .Repo}} -
-
- - {{$JobID := '0'}} - {{if eq .JobType "DEBUG" "TRAIN" "INFERENCE" "HPC" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY" "SNN4ECOSET" "SIM2BRAIN_SNN" "ONLINEINFERENCE"}} - {{$JobID = .Cloudbrain.ID}} - {{else}} - {{$JobID = .JobID}} - {{end}} - -
- {{if eq .JobType "DEBUG"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "HPC"}} - - {{.DisplayJobName}} - - {{else if (eq .JobType "SNN4IMAGENET" "BRAINSCORE" "SNN4ECOSET" "SIM2BRAIN_SNN")}} - - {{.DisplayJobName}} - - {{else if eq .JobType "INFERENCE"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "TRAIN"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "BENCHMARK" "MODELSAFETY"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "ONLINEINFERENCE"}} - - {{.DisplayJobName}} - - {{end}} -
- -
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} -
- -
- - - - {{.Status}} - {{if eq .Status "WAITING"}}{{end}} - - -
- - - {{$JobType := $.i18n.Tr (printf "cloudbrain.%s" .JobType)}} -
- {{$JobType}} -
- -
- {{TimeSinceUnix1 .Cloudbrain.CreatedUnix}} -
- -
- {{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}} -
- -
- {{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}} -
- - -
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} -
- -
- -
- - - -
- {{if eq .JobType "DEBUG"}} -
-
- {{$.CsrfTokenHtml}} - {{if eq .Status "RUNNING" "WAITING" "CREATING" "STARTING"}} - - {{$.i18n.Tr "repo.debug"}} - - {{else}} - {{if not .BootFile}} - - {{$.i18n.Tr "repo.debug_again"}} - - {{end}} - {{end}} -
-
- {{end}} - {{if eq .JobType "ONLINEINFERENCE"}} -
-
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.online_debug"}} - -
-
- {{end}} - {{if eq .JobType "HPC"}} -
-
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.start_use"}} - -
-
- {{end}} - -
- {{if eq .JobType "MODELSAFETY"}} -
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.stop"}} - -
- {{else}} - {{if eq .JobType "DEBUG" "HPC" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE" "SNN4ECOSET" "SIM2BRAIN_SNN" "ONLINEINFERENCE"}} -
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.stop"}} - -
- {{else}} - - {{$.i18n.Tr "repo.stop"}} - - {{end}} - {{end}} -
- {{if eq .JobType "BENCHMARK"}} - - {{end}} - - {{if and (eq .JobType "TRAIN") (not .FineTune)}} - - {{end}} - - {{if eq .JobType "MODELSAFETY"}} -
- {{$.CsrfTokenHtml}} - - - {{$.i18n.Tr "repo.delete"}} - -
- {{else}} -
- {{$.CsrfTokenHtml}} - - - {{$.i18n.Tr "repo.delete"}} - -
- {{end}} -
-
-
- {{else}} - {{$JobID := '0'}} - {{if eq .JobType "DEBUG" "TRAIN" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "SNN4ECOSET" "SIM2BRAIN_SNN"}} - {{$JobID = .Cloudbrain.ID}} - {{else}} - {{$JobID = .JobID}} - {{end}} -
-
- -
- {{if eq .JobType "DEBUG"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "INFERENCE"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "TRAIN"}} - - {{.DisplayJobName}} - - {{else if eq .JobType "BENCHMARK"}} - - {{.DisplayJobName}} - - {{end}} -
- -
- {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} -
- -
- - - - {{.Status}} - {{if eq .Status "WAITING"}}{{end}} - - -
- - {{$JobType := $.i18n.Tr (printf "cloudbrain.%s" .JobType)}} -
- {{$JobType}} -
- -
- {{TimeSinceUnix1 .Cloudbrain.CreatedUnix}} -
- -
- {{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}} -
- -
- {{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}} -
- - -
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} -
- -
- - {{if .CardType}}{{.CardType}}{{else}}--{{end}} - -
- - -
- -- -
-
- {{if eq .JobType "DEBUG"}} -
-
- {{$.CsrfTokenHtml}} - {{if eq .Status "RUNNING" "WAITING" "CREATING" "STARTING"}} - - {{$.i18n.Tr "repo.debug"}} - - {{else}} - - {{$.i18n.Tr "repo.debug_again"}} - - {{end}} -
-
- {{end}} - - - -
- {{$.CsrfTokenHtml}} - - {{$.i18n.Tr "repo.delete"}} - -
-
-
-
- - {{end}} - {{end}} -
-
-
-
-
-
-
- - -
-
- -
- -
- - +
- - + {{template "base/footer" .}} diff --git a/web_src/vuepages/pages/cloudbrain/tasks/index.vue b/web_src/vuepages/pages/cloudbrain/tasks/index.vue new file mode 100644 index 0000000000..d8201d148a --- /dev/null +++ b/web_src/vuepages/pages/cloudbrain/tasks/index.vue @@ -0,0 +1,529 @@ + + + + + diff --git a/web_src/vuepages/pages/cloudbrain/tasks/vp-cloudbrain-tasks.js b/web_src/vuepages/pages/cloudbrain/tasks/vp-cloudbrain-tasks.js new file mode 100644 index 0000000000..48e5425001 --- /dev/null +++ b/web_src/vuepages/pages/cloudbrain/tasks/vp-cloudbrain-tasks.js @@ -0,0 +1,17 @@ +import Vue from 'vue'; +import ElementUI from 'element-ui'; +import 'element-ui/lib/theme-chalk/index.css'; +import localeEn from 'element-ui/lib/locale/lang/en'; +import localeZh from 'element-ui/lib/locale/lang/zh-CN'; +import { i18n, lang } from '~/langs'; +import App from './index.vue'; + +Vue.use(ElementUI, { + locale: lang === 'zh-CN' ? localeZh : localeEn, + size: 'small', +}); + +new Vue({ + i18n, + render: (h) => h(App), +}).$mount('#__vue-root'); diff --git a/web_src/vuepages/pages/cloudbrain/tools.js b/web_src/vuepages/pages/cloudbrain/tools.js index 9d79a49685..6e086a73c2 100644 --- a/web_src/vuepages/pages/cloudbrain/tools.js +++ b/web_src/vuepages/pages/cloudbrain/tools.js @@ -188,6 +188,9 @@ export class CloudBrainTools { case 'HPC': link += `supercompute/job/${taskInfo.id}`; break; + case 'GENERAL': + link += `grampus/general/${taskInfo.id}`; + break; default: break; } -- 2.34.1 From 255d352ba233efd3bb4a5fef68b861f515222b98 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 28 Feb 2024 15:28:25 +0800 Subject: [PATCH 06/55] =?UTF-8?q?=E4=B8=BA=E9=80=9A=E7=94=A8=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E9=80=82=E9=85=8D=E8=8E=B7=E5=8F=96=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=AE=80=E5=86=B5=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ai_task_service/task/grampus_general_task.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/services/ai_task_service/task/grampus_general_task.go b/services/ai_task_service/task/grampus_general_task.go index 7233987d6c..dcc50ade52 100644 --- a/services/ai_task_service/task/grampus_general_task.go +++ b/services/ai_task_service/task/grampus_general_task.go @@ -149,3 +149,17 @@ func (g GrampusGeneralTaskTemplate) GetImages(computeSource models.ComputeSource } return images, customFlag, nil } + +func (g GrampusGeneralTaskTemplate) GetOperationProfile(cloudbrainId int64) (*entity.OperationProfile, *response.BizError) { + c := g.GetMyCluster() + if c == nil { + log.Error("Get cluster failed,cloudbrainId=%d", cloudbrainId) + return nil, response.SYSTEM_ERROR + } + s, err := GetOperationProfile(cloudbrainId, c.GetNoteBookOperationProfile) + if err != nil { + log.Error("GetOperationProfile err.cloudbrainId=%d err =%v", cloudbrainId, err) + return nil, nil + } + return s, nil +} -- 2.34.1 From 24a5170f04c6dcc3cd33e01b9d1b7383576ddc54 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 5 Mar 2024 17:34:06 +0800 Subject: [PATCH 07/55] =?UTF-8?q?#5100=20=E9=87=8D=E6=9E=84=E4=B8=AA?= =?UTF-8?q?=E4=BA=BA=E4=BA=91=E8=84=91=E4=BB=BB=E5=8A=A1=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- entity/ai_task.go | 23 ++++- routers/admin/cloudbrains.go | 1 + routers/ai_task/ai_task.go | 37 +++++++- routers/api/v1/api.go | 1 + services/ai_task_service/task/task_list.go | 84 ++++++++++++++++++- services/ai_task_service/task/task_service.go | 2 +- 6 files changed, 143 insertions(+), 5 deletions(-) diff --git a/entity/ai_task.go b/entity/ai_task.go index b86dca5960..80e45f1072 100644 --- a/entity/ai_task.go +++ b/entity/ai_task.go @@ -228,6 +228,8 @@ type AITaskBriefInfo struct { IsFileNotebook bool `json:"is_file_notebook"` IsFineTuneTask bool `json:"is_fine_tune_task"` APPName string `json:"app_name"` + AccCardType string `json:"acc_card_type"` + JobId string `json:"job_id"` } func (a *AITaskBriefInfo) Tr(language string) { @@ -247,9 +249,15 @@ type AITaskInfo4List struct { Creator UserBriefInfo `json:"creator"` CanModify bool `json:"can_modify"` CanDelete bool `json:"can_delete"` + RepoName string `json:"repo_name"` + OwnerName string `json:"owner_name"` } -func ConvertCloudbrainToAITaskBriefInfo(task *models.Cloudbrain) *AITaskBriefInfo { +func ConvertCloudbrainToAITaskBriefInfo(task *models.Cloudbrain, isAdmin bool) *AITaskBriefInfo { + jobID := "" + if isAdmin { + jobID = task.JobID + } return &AITaskBriefInfo{ ID: task.ID, JobType: task.JobType, @@ -266,6 +274,8 @@ func ConvertCloudbrainToAITaskBriefInfo(task *models.Cloudbrain) *AITaskBriefInf IsFileNotebook: task.IsFileNoteBookTask(), IsFineTuneTask: task.FineTune, APPName: task.AppName, + AccCardType: task.Spec.AccCardType, + JobId: jobID, } } @@ -339,6 +349,17 @@ type GetTaskListReq struct { Operator *models.User IsRepoOwner bool } +type GetMyTaskListReq struct { + models.ListOptions + ComputeSource *models.ComputeSource + JobType string + JobStatus string + User *models.User + IsRepoOwner bool + Keyword string + AICenter string + Cluster string +} type AITaskBaseConfig struct { ContainerSteps map[ContainerDataType]*ContainerBuildOpts `json:"container_configs"` diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index bc2b820ecd..5402bebc94 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -117,6 +117,7 @@ func CloudBrains(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = ciTasks ctx.Data["CanCreate"] = true + ctx.Data["Keyword"] = keyword ctx.HTML(200, tplCloudBrains) diff --git a/routers/ai_task/ai_task.go b/routers/ai_task/ai_task.go index 05b3584ec3..afa90e6f50 100644 --- a/routers/ai_task/ai_task.go +++ b/routers/ai_task/ai_task.go @@ -468,7 +468,7 @@ func GetAITaskList(ctx *context.Context) { if jobType != "" { jobTypes = append(jobTypes, jobType) } - result, err := task.GetAITaskList(entity.GetTaskListReq{ + result, err := task.GetRepoAITaskList(entity.GetTaskListReq{ ListOptions: models.ListOptions{ PageSize: setting.UI.IssuePagingNum, Page: page, @@ -576,3 +576,38 @@ func GenerateSDKCode(ctx *context.Context) { code := task.GenerateSDKCode(datasetNames, pretrainModelNames, parameterKeys, models.JobType(jobType)) ctx.JSON(http.StatusOK, response.OuterSuccessWithData(map[string]string{"code": code})) } + +func GetMyAITaskList(ctx *context.Context) { + jobType := ctx.Query("job_type") + jobStatus := ctx.Query("job_status") + aiCenter := ctx.Query("ai_center") + cluster := ctx.Query("cluster") + computeSourceName := ctx.Query("compute_source") + keyword := strings.Trim(ctx.Query("q"), " ") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + computeSource := models.GetComputeSourceInstance(computeSourceName) + result, err := task.GetMyAITaskList(entity.GetMyTaskListReq{ + ListOptions: models.ListOptions{ + PageSize: setting.UI.IssuePagingNum, + Page: page, + }, + ComputeSource: computeSource, + JobType: jobType, + User: ctx.User, + IsRepoOwner: ctx.Repo.IsOwner(), + JobStatus: jobStatus, + Keyword: keyword, + AICenter: aiCenter, + Cluster: cluster, + }) + if err != nil { + log.Error("GetMyAITaskList error,err=%v", err) + ctx.JSON(http.StatusOK, response.OuterTrBizError(err, ctx)) + return + } + result.CanCreateTask = cloudbrain.CanCreateOrDebugJob(ctx) + ctx.JSON(http.StatusOK, response.OuterSuccessWithData(result)) +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index fd09df35aa..a6a87812ee 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -679,6 +679,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/ai_task", func() { m.Get("/generate_sdk_code", ai_task.GenerateSDKCode) + m.Get("/my_list", reqToken(), ai_task.GetMyAITaskList) }) // Miscellaneous diff --git a/services/ai_task_service/task/task_list.go b/services/ai_task_service/task/task_list.go index 4b6a5fbb1d..61da5bcd35 100644 --- a/services/ai_task_service/task/task_list.go +++ b/services/ai_task_service/task/task_list.go @@ -8,7 +8,7 @@ import ( "code.gitea.io/gitea/routers/response" ) -func GetAITaskList(req entity.GetTaskListReq) (*entity.AITaskListRes, *response.BizError) { +func GetRepoAITaskList(req entity.GetTaskListReq) (*entity.AITaskListRes, *response.BizError) { page := req.Page if page <= 0 { page = 1 @@ -34,7 +34,7 @@ func GetAITaskList(req entity.GetTaskListReq) (*entity.AITaskListRes, *response. for i := 0; i < len(tasks); i++ { r[i] = &entity.AITaskInfo4List{ - Task: entity.ConvertCloudbrainToAITaskBriefInfo(&tasks[i].Cloudbrain), + Task: entity.ConvertCloudbrainToAITaskBriefInfo(&tasks[i].Cloudbrain, false), Creator: *entity.ConvertUserToBrief(&tasks[i].User), CanModify: tasks[i].CanUserModify(req.Operator), CanDelete: tasks[i].CanUserDelete(req.Operator, req.IsRepoOwner), @@ -48,3 +48,83 @@ func GetAITaskList(req entity.GetTaskListReq) (*entity.AITaskListRes, *response. Page: page, }, nil } + +func GetMyAITaskList(req entity.GetMyTaskListReq) (*entity.AITaskListRes, *response.BizError) { + var jobTypes []string + jobTypeNot := false + if req.JobType == string(models.JobTypeBenchmark) { + jobTypes = models.AllBenchMarkJobType() + } else if req.JobType != "all" && req.JobType != "" { + jobTypes = append(jobTypes, req.JobType) + } + + var jobStatuses []string + jobStatusNot := false + if req.JobStatus == "other" { + jobStatusNot = true + jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), + string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), + string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) + } else if req.JobStatus != "all" && req.JobStatus != "" { + jobStatuses = append(jobStatuses, req.JobStatus) + } + + repos, _, err := models.SearchRepository(&models.SearchRepoOptions{ + Actor: req.User, + OwnerID: req.User.ID, + Private: true, + }) + if err != nil { + return nil, response.NewBizError(err) + } + var repoIDList []int64 + for i, _ := range repos { + repoIDList = append(repoIDList, repos[i].ID) + } + computeSourceName := "" + if req.ComputeSource != nil { + computeSourceName = req.ComputeSource.GetCloudbrainFormat() + } + tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: req.Page, + PageSize: req.PageSize, + }, + Keyword: req.Keyword, + UserID: req.User.ID, + JobTypeNot: jobTypeNot, + JobStatusNot: jobStatusNot, + JobStatus: jobStatuses, + JobTypes: jobTypes, + NeedRepoInfo: true, + IsLatestVersion: modelarts.IsLatestVersion, + RepoIDList: repoIDList, + ComputeResource: computeSourceName, + Type: models.TypeCloudBrainAll, + AiCenter: req.AICenter, + Cluster: req.Cluster, + }) + if err != nil { + log.Error("GetMyAITaskList query cloudbrains err. req=%+v,err=%v", req, err) + return nil, response.NewBizError(err) + } + models.LoadSpecs4CloudbrainInfo(tasks) + r := make([]*entity.AITaskInfo4List, len(tasks)) + + for i := 0; i < len(tasks); i++ { + r[i] = &entity.AITaskInfo4List{ + Task: entity.ConvertCloudbrainToAITaskBriefInfo(&tasks[i].Cloudbrain, false), + CanModify: tasks[i].CanUserModify(req.User), + CanDelete: tasks[i].CanUserDelete(req.User, req.IsRepoOwner), + RepoName: tasks[i].Repo.Name, + OwnerName: tasks[i].Repo.OwnerName, + } + } + + return &entity.AITaskListRes{ + Tasks: r, + Total: count, + PageSize: req.PageSize, + Page: req.Page, + }, nil +} diff --git a/services/ai_task_service/task/task_service.go b/services/ai_task_service/task/task_service.go index c03fa321de..e9be594561 100644 --- a/services/ai_task_service/task/task_service.go +++ b/services/ai_task_service/task/task_service.go @@ -206,7 +206,7 @@ func QueryTaskBriefInfo(id int64) (*entity.AITaskBriefInfo, error) { log.Error("QueryTaskBriefInfo GetCloudbrainByCloudbrainID err.id=%d err=%v", id, err) return nil, err } - return entity.ConvertCloudbrainToAITaskBriefInfo(task), nil + return entity.ConvertCloudbrainToAITaskBriefInfo(task, false), nil } func UpdateAITaskFromRemote(task *models.Cloudbrain, remoteFunc QueryFunc) error { -- 2.34.1 From eb204dcb1f456b2aca774272fa16651215d422e3 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 5 Mar 2024 17:58:42 +0800 Subject: [PATCH 08/55] =?UTF-8?q?#5100=20=E9=87=8D=E6=9E=84=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E5=90=8E=E5=8F=B0=E4=BA=91=E8=84=91=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/ai_task/ai_task.go | 34 +++++++++++ routers/api/v1/api.go | 3 + services/ai_task_service/task/task_list.go | 67 ++++++++++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/routers/ai_task/ai_task.go b/routers/ai_task/ai_task.go index afa90e6f50..023c1d76fd 100644 --- a/routers/ai_task/ai_task.go +++ b/routers/ai_task/ai_task.go @@ -611,3 +611,37 @@ func GetMyAITaskList(ctx *context.Context) { result.CanCreateTask = cloudbrain.CanCreateOrDebugJob(ctx) ctx.JSON(http.StatusOK, response.OuterSuccessWithData(result)) } + +func GetAITaskList4Admin(ctx *context.Context) { + jobType := ctx.Query("job_type") + jobStatus := ctx.Query("job_status") + aiCenter := ctx.Query("ai_center") + cluster := ctx.Query("cluster") + computeSourceName := ctx.Query("compute_source") + keyword := strings.Trim(ctx.Query("q"), " ") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + computeSource := models.GetComputeSourceInstance(computeSourceName) + result, err := task.GetAITaskList4Admin(entity.GetMyTaskListReq{ + ListOptions: models.ListOptions{ + PageSize: setting.UI.IssuePagingNum, + Page: page, + }, + ComputeSource: computeSource, + JobType: jobType, + User: ctx.User, + JobStatus: jobStatus, + Keyword: keyword, + AICenter: aiCenter, + Cluster: cluster, + }) + if err != nil { + log.Error("GetMyAITaskList error,err=%v", err) + ctx.JSON(http.StatusOK, response.OuterTrBizError(err, ctx)) + return + } + result.CanCreateTask = cloudbrain.CanCreateOrDebugJob(ctx) + ctx.JSON(http.StatusOK, response.OuterSuccessWithData(result)) +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index a6a87812ee..3bedc5accd 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1467,6 +1467,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Combo("").Post(bind(models.OperateRoleReq{}), admin.AddUserRole). Delete(bind(models.OperateRoleReq{}), admin.DeleteUserRole) }) + m.Group("/ai_task", func() { + m.Get("/list", ai_task.GetAITaskList4Admin) + }) }, reqToken(), reqSiteAdmin()) m.Group("/topics", func() { diff --git a/services/ai_task_service/task/task_list.go b/services/ai_task_service/task/task_list.go index 61da5bcd35..03d2248425 100644 --- a/services/ai_task_service/task/task_list.go +++ b/services/ai_task_service/task/task_list.go @@ -128,3 +128,70 @@ func GetMyAITaskList(req entity.GetMyTaskListReq) (*entity.AITaskListRes, *respo Page: req.Page, }, nil } + +func GetAITaskList4Admin(req entity.GetMyTaskListReq) (*entity.AITaskListRes, *response.BizError) { + var jobTypes []string + jobTypeNot := false + if req.JobType == string(models.JobTypeBenchmark) { + jobTypes = models.AllBenchMarkJobType() + } else if req.JobType != "all" && req.JobType != "" { + jobTypes = append(jobTypes, req.JobType) + } + + var jobStatuses []string + jobStatusNot := false + if req.JobStatus == "other" { + jobStatusNot = true + jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), + string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), + string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) + } else if req.JobStatus != "all" && req.JobStatus != "" { + jobStatuses = append(jobStatuses, req.JobStatus) + } + + computeSourceName := "" + if req.ComputeSource != nil { + computeSourceName = req.ComputeSource.GetCloudbrainFormat() + } + tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: req.Page, + PageSize: req.PageSize, + }, + Keyword: req.Keyword, + JobTypeNot: jobTypeNot, + JobStatusNot: jobStatusNot, + JobStatus: jobStatuses, + JobTypes: jobTypes, + NeedRepoInfo: true, + IsLatestVersion: modelarts.IsLatestVersion, + ComputeResource: computeSourceName, + Type: models.TypeCloudBrainAll, + AiCenter: req.AICenter, + Cluster: req.Cluster, + }) + if err != nil { + log.Error("GetMyAITaskList query cloudbrains err. req=%+v,err=%v", req, err) + return nil, response.NewBizError(err) + } + models.LoadSpecs4CloudbrainInfo(tasks) + r := make([]*entity.AITaskInfo4List, len(tasks)) + + for i := 0; i < len(tasks); i++ { + r[i] = &entity.AITaskInfo4List{ + Task: entity.ConvertCloudbrainToAITaskBriefInfo(&tasks[i].Cloudbrain, true), + Creator: *entity.ConvertUserToBrief(&tasks[i].User), + CanModify: true, + CanDelete: true, + RepoName: tasks[i].Repo.Name, + OwnerName: tasks[i].Repo.OwnerName, + } + } + + return &entity.AITaskListRes{ + Tasks: r, + Total: count, + PageSize: req.PageSize, + Page: req.Page, + }, nil +} -- 2.34.1 From 2fcb1b19c8e499a355ac89d886fb92c8b9f25966 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 6 Mar 2024 10:04:58 +0800 Subject: [PATCH 09/55] update task list --- web_src/vuepages/apis/modules/cloudbrain.js | 20 ++ web_src/vuepages/pages/cloudbrain/configs.js | 4 +- .../vuepages/pages/cloudbrain/tasks/index.vue | 195 ++++++++++++++---- 3 files changed, 175 insertions(+), 44 deletions(-) diff --git a/web_src/vuepages/apis/modules/cloudbrain.js b/web_src/vuepages/apis/modules/cloudbrain.js index 26c6f27b97..2d3ecb21de 100644 --- a/web_src/vuepages/apis/modules/cloudbrain.js +++ b/web_src/vuepages/apis/modules/cloudbrain.js @@ -249,3 +249,23 @@ export const setAiTaskExportDataset = (data) => { data: Qs.stringify(data), }); } + +// 个人中心-云脑任务列表 +// job_type,job_status,ai_center,cluster,compute_source,q,page,pageSize +export const getMyAiTasks = (params) => { + return service({ + url: `/api/v1/ai_task/my_list`, + method: 'get', + params: { ...params }, + }); +} + +// 管理后台-云脑任务列表 +// job_type,job_status,ai_center,cluster,compute_source,q,page,pageSize +export const getAdminAiTasks = (params) => { + return service({ + url: `/api/v1/admin/ai_task/list`, + method: 'get', + params: { ...params }, + }); +} diff --git a/web_src/vuepages/pages/cloudbrain/configs.js b/web_src/vuepages/pages/cloudbrain/configs.js index 742b0ecefc..1ccd7011ff 100644 --- a/web_src/vuepages/pages/cloudbrain/configs.js +++ b/web_src/vuepages/pages/cloudbrain/configs.js @@ -670,7 +670,7 @@ export const ListPageConfigs = { url: 'grampus/general', createUrl: 'grampus/general/create?type=0', columns: ['taskName', 'status', 'createTime', 'clusterAndComputeResource', 'creator'], - operations: ['debug', 'redebug', 'stop', 'delete', 'debugMore'], + operations: ['debug', 'stop', 'delete'], emptyTitle: i18n.t('cloudbrainObj.debugTaskEmptyTitle'), emptyTip1: i18n.t('cloudbrainObj.debugTaskEmptyTip1'), emptyTip2: i18n.t('cloudbrainObj.debugTaskEmptyTip2'), @@ -1359,8 +1359,6 @@ export const DetailPageConfigs = { 'dataset', 'modelList', ] - }, { - name: 'operationProfile' }], }], }] diff --git a/web_src/vuepages/pages/cloudbrain/tasks/index.vue b/web_src/vuepages/pages/cloudbrain/tasks/index.vue index d8201d148a..cbe5ceb09e 100644 --- a/web_src/vuepages/pages/cloudbrain/tasks/index.vue +++ b/web_src/vuepages/pages/cloudbrain/tasks/index.vue @@ -14,12 +14,12 @@
-
+
- +
@@ -47,35 +47,43 @@ class="ri-download-line middle aligned icon">下载此报告
-
+
- + - + + - + + - + + + header-align="center" width="190"> +