#5179 path_unite_all

Merged
liuzx merged 180 commits from path_unite_all into V20240116 3 months ago
  1. +58
    -50
      entity/ai_task.go
  2. +1
    -0
      entity/cluster.go
  3. +1
    -0
      entity/container.go
  4. +3
    -0
      entity/creation.go
  5. +38
    -2
      manager/client/grampus/grampus.go
  6. +25
    -0
      models/ai_model_manage.go
  7. +141
    -23
      models/cloudbrain.go
  8. +1
    -1
      models/model_migrate_record.go
  9. +1
    -0
      models/resource_specification.go
  10. +1
    -1
      modules/grampus/grampus.go
  11. +7
    -0
      modules/redis/redis_key/ai_model_redis_key.go
  12. +4
    -0
      modules/setting/setting.go
  13. +40
    -0
      modules/storage/obs.go
  14. +7
    -3
      options/locale/locale_en-US.ini
  15. +6
    -2
      options/locale/locale_zh-CN.ini
  16. +9
    -0
      routers/ai_task/ai_task.go
  17. +5
    -1
      routers/api/v1/api.go
  18. +1
    -1
      routers/repo/ai_model_convert.go
  19. +55
    -4
      routers/repo/ai_model_manage.go
  20. +32
    -23
      routers/repo/ai_model_square.go
  21. +5
    -2
      routers/repo/attachment_model.go
  22. +39
    -40
      routers/repo/modelarts.go
  23. +3
    -1
      routers/response/response_list.go
  24. +4
    -0
      routers/routes/routes.go
  25. +74
    -0
      services/ai_model/model_version.go
  26. +65
    -232
      services/ai_task_service/cluster/c2net.go
  27. +14
    -13
      services/ai_task_service/cluster/cloudbrain_one.go
  28. +51
    -8
      services/ai_task_service/cluster/cloudbrain_two.go
  29. +1
    -1
      services/ai_task_service/cluster/common.go
  30. +2
    -1
      services/ai_task_service/container_builder/code_builder.go
  31. +6
    -6
      services/ai_task_service/container_builder/common.go
  32. +8
    -5
      services/ai_task_service/container_builder/dataset_builder.go
  33. +1
    -1
      services/ai_task_service/container_builder/log_path_builder.go
  34. +1
    -1
      services/ai_task_service/container_builder/output_path_builder.go
  35. +62
    -102
      services/ai_task_service/container_builder/pre_model_builder.go
  36. +4
    -2
      services/ai_task_service/context/context.go
  37. +12
    -2
      services/ai_task_service/storage_helper/client.go
  38. +13
    -1
      services/ai_task_service/storage_helper/minio.go
  39. +13
    -1
      services/ai_task_service/storage_helper/obs.go
  40. +67
    -5
      services/ai_task_service/storage_helper/repo.go
  41. +5
    -1
      services/ai_task_service/task/cloudbrain_one_notebook_task.go
  42. +1
    -1
      services/ai_task_service/task/cloudbrain_one_train_task.go
  43. +1
    -1
      services/ai_task_service/task/cloudbrain_two_inference_task.go
  44. +13
    -5
      services/ai_task_service/task/cloudbrain_two_notebook_task.go
  45. +1
    -1
      services/ai_task_service/task/cloudbrain_two_train_task.go
  46. +1
    -1
      services/ai_task_service/task/grampus_inference_task.go
  47. +13
    -9
      services/ai_task_service/task/grampus_notebook_task.go
  48. +18
    -6
      services/ai_task_service/task/grampus_online_infer_task.go
  49. +3
    -2
      services/ai_task_service/task/grampus_train_task.go
  50. +49
    -31
      services/ai_task_service/task/opt_handler.go
  51. +138
    -0
      services/ai_task_service/task/sdk_util.go
  52. +1
    -1
      services/ai_task_service/task/super_compute_task.go
  53. +1
    -1
      services/ai_task_service/task/task_config.go
  54. +6
    -1
      services/ai_task_service/task/task_creation_info.go
  55. +25
    -45
      services/ai_task_service/task/task_extend.go
  56. +73
    -54
      services/ai_task_service/task/task_service.go
  57. +1
    -1
      templates/admin/cloudbrain/imagecommit.tmpl
  58. +12
    -7
      templates/repo/datasets/index.tmpl
  59. +0
    -1
      templates/repo/debugjob/index.tmpl
  60. +4
    -0
      templates/repo/home.tmpl
  61. +1
    -1
      templates/user/dashboard/dashboard.tmpl
  62. +3
    -3
      web_src/js/features/clipboard.js
  63. +198
    -0
      web_src/js/features/globalModalDlg.js
  64. +1
    -1
      web_src/js/features/highlight.js
  65. +10
    -0
      web_src/js/features/i18nVue.js
  66. +40
    -6
      web_src/js/index.js
  67. +44
    -0
      web_src/vuepages/apis/modules/common.js
  68. +161
    -0
      web_src/vuepages/components/CommonTipsDialog.vue
  69. +11
    -8
      web_src/vuepages/components/cloudbrain/AIEngineSelect.vue
  70. +16
    -13
      web_src/vuepages/components/cloudbrain/AlgBechmarkType.vue
  71. +13
    -40
      web_src/vuepages/components/cloudbrain/BootFile.vue
  72. +10
    -7
      web_src/vuepages/components/cloudbrain/BranchName.vue
  73. +11
    -14
      web_src/vuepages/components/cloudbrain/DatasetSelect.vue
  74. +44
    -34
      web_src/vuepages/components/cloudbrain/FormTop.vue
  75. +8
    -12
      web_src/vuepages/components/cloudbrain/ImageSelectV1.vue
  76. +11
    -16
      web_src/vuepages/components/cloudbrain/ImageSelectV2.vue
  77. +12
    -14
      web_src/vuepages/components/cloudbrain/ModelSelect.vue
  78. +69
    -83
      web_src/vuepages/components/cloudbrain/ModelSelectV2.vue
  79. +18
    -15
      web_src/vuepages/components/cloudbrain/NetworkType.vue
  80. +28
    -22
      web_src/vuepages/components/cloudbrain/RunParameters.vue
  81. +129
    -0
      web_src/vuepages/components/cloudbrain/SDKCode.vue
  82. +33
    -34
      web_src/vuepages/components/cloudbrain/SpecSelect.vue
  83. +11
    -8
      web_src/vuepages/components/cloudbrain/TaskDescr.vue
  84. +12
    -8
      web_src/vuepages/components/cloudbrain/TaskName.vue
  85. +14
    -9
      web_src/vuepages/components/cloudbrain/WorkServerNum.vue
  86. +92
    -61
      web_src/vuepages/components/cloudbrain/cloudbrain.less
  87. +56
    -5
      web_src/vuepages/components/cloudbrain/details/ConfigInfo.vue
  88. +22
    -6
      web_src/vuepages/langs/config/en-US.js
  89. +22
    -8
      web_src/vuepages/langs/config/zh-CN.js
  90. +140
    -132
      web_src/vuepages/pages/cloudbrain/configs.js
  91. +209
    -129
      web_src/vuepages/pages/cloudbrain/create/index.vue
  92. +97
    -2
      web_src/vuepages/pages/cloudbrain/list/index.vue
  93. +17
    -29
      web_src/vuepages/pages/computingpower/domestic/index.vue
  94. +27
    -0
      web_src/vuepages/pages/dataset/codedialog/index.vue
  95. +17
    -0
      web_src/vuepages/pages/dataset/codedialog/vp-dataset-code-dialog.js
  96. +47
    -37
      web_src/vuepages/pages/modelmanage/components/ModelHeader.vue
  97. +34
    -9
      web_src/vuepages/pages/modelmanage/files/index.vue
  98. +19
    -8
      web_src/vuepages/pages/modelmanage/fileupload/index.vue
  99. +14
    -3
      web_src/vuepages/pages/modelmanage/graph/index.vue
  100. +11
    -2
      web_src/vuepages/pages/modelmanage/graph/model-graph.css

+ 58
- 50
entity/ai_task.go View File

@@ -31,14 +31,11 @@ type CreateReq struct {
Cluster ClusterType `json:"cluster" binding:"Required"`
WorkServerNumber int `json:"work_server_number"`
BranchName string `json:"branch_name"`
PreTrainModelUrl string `json:"pretrain_model_url"`
PretrainModelCkptName string `json:"pretrain_model_ckpt_name"`
ImageUrl string `json:"image_url"`
ImageID string `json:"image_id"`
ImageName string `json:"image_name"`
PretrainModelName string `json:"pretrain_model_name"`
PretrainModelVersion string `json:"pretrain_model_version"`
PretrainModelId string `json:"pretrain_model_id"`
PretrainModelId string `json:"pretrain_model_id_str"`
Description string `json:"description"`
LabelName string `json:"label_names"`
DatasetUUIDStr string `json:"dataset_uuid_str"`
@@ -57,6 +54,7 @@ type CreateReq struct {
FileBranchName string
IsRestartRequest bool
DatasetNames string
ModelNames string
}

type CreationResponse struct {
@@ -83,12 +81,13 @@ func (r *QueryAITaskRes) TryToRemoveDatasetAndModelInfo(currentUser *models.User
if r.Task != nil {
r.Task.TryToRemoveDatasets(currentUser)
r.Task.TryToRemovePretrainModelList(currentUser)
r.Task.TryToRemoveSDKCode(currentUser)
}
if r.EarlyVersionList != nil {
for _, t := range r.EarlyVersionList {
t.TryToRemoveDatasets(currentUser)
t.TryToRemovePretrainModelList(currentUser)
t.TryToRemoveSDKCode(currentUser)
}
}
}
@@ -104,49 +103,48 @@ func (r *QueryAITaskRes) Tr(language string) {
}

type AITaskDetailInfo struct {
ID int64 `json:"id"`
JobID string `json:"job_id"`
Status string `json:"status"`
DetailedStatus string `json:"detailed_status"`
JobType string `json:"job_type"`
Cluster string `json:"cluster"`
DisplayJobName string `json:"display_job_name"`
FormattedDuration string `json:"formatted_duration"`
ComputeSource string `json:"compute_source"`
AICenter string `json:"ai_center"`
BootFile string `json:"boot_file"`
PreVersionName string `json:"pre_version_name"`
CurrentVersionName string `json:"current_version_name"`
WorkServerNumber int `json:"work_server_number"`
Spec *structs.SpecificationShow `json:"spec"`
DatasetList []*models.DatasetDownload `json:"dataset_list"`
PretrainModelList []*models.ModelDownload `json:"pretrain_model_list"`
Parameters *models.Parameters `json:"parameters"`
CreatedUnix timeutil.TimeStamp `json:"created_unix"`
CodePath string `json:"code_path"`
DatasetPath string `json:"dataset_path"`
PretrainModelPath string `json:"pretrain_model_path"`
PretrainModelUrl string `json:"pretrain_model_url"`
OutputPath string `json:"output_path"`
CodeUrl string `json:"code_url"`
PretrainModelName string `json:"pretrain_model_name"`
PretrainModelVersion string `json:"pretrain_model_version"`
PretrainCkptName string `json:"pretrain_model_ckpt_name"`
PretrainModelId string `json:"pretrain_model_id"`
StartTime timeutil.TimeStamp `json:"start_time"`
EndTime timeutil.TimeStamp `json:"end_time"`
Description string `json:"description"`
CommitID string `json:"commit_id"`
BranchName string `json:"branch_name"`
ImageUrl string `json:"image_url"`
ImageID string `json:"image_id"`
ImageName string `json:"image_name"`
CreatorName string `json:"creator_name"`
EngineName string `json:"engine_name"`
FailedReason string `json:"failed_reason"`
UserId int64 `json:"-"`
AppName string `json:"app_name"`
HasInternet int `json:"has_internet"`
ID int64 `json:"id"`
JobID string `json:"job_id"`
Status string `json:"status"`
DetailedStatus string `json:"detailed_status"`
JobType string `json:"job_type"`
Cluster string `json:"cluster"`
DisplayJobName string `json:"display_job_name"`
FormattedDuration string `json:"formatted_duration"`
ComputeSource string `json:"compute_source"`
AICenter string `json:"ai_center"`
BootFile string `json:"boot_file"`
PreVersionName string `json:"pre_version_name"`
CurrentVersionName string `json:"current_version_name"`
WorkServerNumber int `json:"work_server_number"`
Spec *structs.SpecificationShow `json:"spec"`
DatasetList []*models.DatasetDownload `json:"dataset_list"`
PretrainModelList []*models.Model4Show `json:"pretrain_model_list"`
SDKCode string `json:"sdk_code"`
Parameters *models.Parameters `json:"parameters"`
CreatedUnix timeutil.TimeStamp `json:"created_unix"`
CodePath string `json:"code_path"`
DatasetPath string `json:"dataset_path"`
PretrainModelPath string `json:"pretrain_model_path"`
PretrainModelUrl string `json:"pretrain_model_url"`
OutputPath string `json:"output_path"`
CodeUrl string `json:"code_url"`
PretrainModelName string `json:"pretrain_model_name"`
PretrainModelId string `json:"pretrain_model_id"`
StartTime timeutil.TimeStamp `json:"start_time"`
EndTime timeutil.TimeStamp `json:"end_time"`
Description string `json:"description"`
CommitID string `json:"commit_id"`
BranchName string `json:"branch_name"`
ImageUrl string `json:"image_url"`
ImageID string `json:"image_id"`
ImageName string `json:"image_name"`
CreatorName string `json:"creator_name"`
EngineName string `json:"engine_name"`
FailedReason string `json:"failed_reason"`
UserId int64 `json:"-"`
AppName string `json:"app_name"`
HasInternet int `json:"has_internet"`
}

func (a *AITaskDetailInfo) Tr(language string) {
@@ -160,7 +158,12 @@ func (a *AITaskDetailInfo) TryToRemoveDatasets(currentUser *models.User) {
}
func (a *AITaskDetailInfo) TryToRemovePretrainModelList(currentUser *models.User) {
if currentUser == nil || a.UserId == 0 || (!currentUser.IsAdmin && currentUser.ID != a.UserId) {
a.PretrainModelList = []*models.ModelDownload{}
a.PretrainModelList = []*models.Model4Show{}
}
}
func (a *AITaskDetailInfo) TryToRemoveSDKCode(currentUser *models.User) {
if currentUser == nil || a.UserId == 0 || (!currentUser.IsAdmin && currentUser.ID != a.UserId) {
a.SDKCode = ""
}
}

@@ -267,7 +270,10 @@ func ConvertCloudbrainToAITaskBriefInfo(task *models.Cloudbrain) *AITaskBriefInf
}

type NotebookDataset struct {
DatasetUrl string `json:"dataset_url"`
DatasetUrl string `json:"dataset_url"`
DatasetName string `json:"dataset_name"`
ContainerPath string `json:"containerPath"`
ReadOnly bool `json:"readOnly"`
}

type QueryLogOpts struct {
@@ -340,6 +346,8 @@ type AITaskBaseConfig struct {
IsActionUseJobId bool `json:"is_action_use_job_id"`
DatasetsLimitSizeGB int
DatasetsMaxNum int
ModelMaxNum int
ModelLimitSizeGB int
}

func GetAITaskConfigFromCloudbrainConfig(config *models.CloudbrainConfig) *AITaskBaseConfig {


+ 1
- 0
entity/cluster.go View File

@@ -35,6 +35,7 @@ type NoteBookTask struct {
ResourceSpecId string
BootFile string
Spec *models.Specification
EnvVariables models.GrampusEnvVarReq
}

type CreateNoteBookTaskResponse struct {


+ 1
- 0
entity/container.go View File

@@ -20,6 +20,7 @@ type ContainerData struct {
S3DownloadUrl string `json:"s3DownloadUrl"`
Size int64 `json:"size"`
IsOverwrite bool `json:"isOverwrite"`
IsNeedUnzip bool `json:"isNeedUnzip"`
StorageType StorageType
}



+ 3
- 0
entity/creation.go View File

@@ -28,6 +28,9 @@ type ImageRequiredInfo struct {

type AITaskCreationConfig struct {
DatasetMaxSize int `json:"dataset_max_size"`
DatasetsMaxNum int `json:"dataset_max_num"`
ModelMaxSize int `json:"model_max_size"`
ModelMaxNum int `json:"model_max_num"`
}

type SpecificationInfo struct {


+ 38
- 2
manager/client/grampus/grampus.go View File

@@ -30,6 +30,7 @@ const (
urlGetAiCenter = urlOpenApiV1 + "sharescreen/aicenter"
urlGetImages = urlOpenApiV1 + "image"
urlNotebookJob = urlOpenApiV1 + "notebook"
urlInferenceJob = urlOpenApiV1 + "inference"

errorIllegalToken = 1005
errorCannotStopCreatingJob = 5008
@@ -93,6 +94,41 @@ func getToken() error {
return nil
}

func CreateInferenceJob(req models.CreateGrampusInferenceRequest) (*models.GrampusNotebookResponse, error) {
checkSetting()
client := getRestyClient()
var result models.GrampusNotebookResponse
reqJson, _ := json.Marshal(req)
log.Info("Online infer REQ:" + string(reqJson))
retry := 0

sendjob:
_, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetBody(req).
SetResult(&result).
Post(HOST + urlInferenceJob)

if err != nil {
log.Error("resty CreateInferenceJob: %v", err)
return nil, models.NetworkError{}
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("CreateInferenceJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("CreateNotebookJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}
log.Info("CreateInferenceJob success.req.JobName = %s ,result=%+v", req.Name, result)
return &result, nil
}

func CreateNotebookJob(req models.CreateGrampusNotebookRequest) (*models.GrampusNotebookResponse, error) {
checkSetting()
client := getRestyClient()
@@ -169,7 +205,7 @@ func GetNotebookJob(jobID string) (*models.GrampusNotebookResponse, error) {
retry := 0

sendjob:
_, err := client.R().
body, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + urlNotebookJob + "/" + jobID)
@@ -177,7 +213,7 @@ sendjob:
if err != nil {
return nil, fmt.Errorf("resty GetNotebookJob: %v", err)
}
log.Info("%+v", body)
if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
log.Info("retry get token")


+ 25
- 0
models/ai_model_manage.go View File

@@ -872,3 +872,28 @@ func QueryModelRepoByModelID(modelId string) (*Repository, error) {
}
return r, nil
}

func QueryModelMapsByIds(ids []string) (map[string]*AiModelManage, error) {
sess := x.NewSession()
defer sess.Close()
re := make([]*AiModelManage, 0)
err := sess.Table(new(AiModelManage)).In("id", ids).Find(&re)
if err != nil {
return nil, err
}
resultMap := make(map[string]*AiModelManage, 0)
for _, m := range re {
resultMap[m.ID] = m
}
return resultMap, nil
}

//created_unix
func QueryModelIdsByPaging(pageSize, pageNum int, sort string) ([]string, error) {
sess := x.NewSession()
defer sess.Close()
re := make([]string, 0)
start := (pageNum - 1) * pageSize
err := sess.Table("ai_model_manage").Cols("id").OrderBy(sort).Limit(pageSize, start).Find(&re)
return re, err
}

+ 141
- 23
models/cloudbrain.go View File

@@ -149,6 +149,24 @@ const (
OpenICluster = "OpenI"
C2NetCluster = "C2Net"

//cloudbrain two sdk PathValue
LocalCodePath = "/home/ma-user/work/code"
LocalDatasetPath = "/home/ma-user/work/dataset"
LocalPretrainModelPath = "/home/ma-user/work/pretrainmodel"
LocalOutputPath = "/home/ma-user/work/output"

DataDownloadMethodMount = "MOUNT"
DataDownloadMethodMoxing = "MOXING"

CodeNeedUnzipTrue = "true"
CodeNeedUnzipFalse = "false"

DatasetNeedUnzipTrue = "true"
DatasetNeedUnzipFalse = "false"

PretrainModelNeedUnzipTrue = "true"
PretrainModelNeedUnzipFalse = "false"

//AI center
AICenterOfCloudBrainOne = "OpenIOne"
AICenterOfCloudBrainTwo = "OpenITwo"
@@ -378,6 +396,37 @@ func (task *Cloudbrain) ToShow() *CloudbrainShow {
return c
}

func (task *Cloudbrain) HasUseModel(modelId string) bool {
modelIDArray := task.GetModelIdArray()
if modelIDArray == nil || len(modelIDArray) == 0 {
return false
}
for _, id := range modelIDArray {
if id == modelId {
return true
}
}
return false
}

func (task *Cloudbrain) GetModelIdArray() []string {
if task.ModelId == "" {
return []string{}
}
modelIdStr := strings.TrimSuffix(task.ModelId, ";")
modelIDArray := strings.Split(modelIdStr, ";")
return modelIDArray
}

func (task *Cloudbrain) GetModelNameArray() []string {
if task.ModelName == "" {
return []string{}
}
modelNameStr := strings.TrimSuffix(task.ModelName, ";")
modelNameArray := strings.Split(modelNameStr, ";")
return modelNameArray
}

func (task *Cloudbrain) GetStandardComputeSource() string {
return GetComputeSourceStandardFormat(task.ComputeResource)
}
@@ -1163,15 +1212,16 @@ type CloudBrainResult struct {
}

type CreateNotebook2Params struct {
JobName string `json:"name"`
Description string `json:"description"`
Duration int64 `json:"duration"` //ms
Feature string `json:"feature"`
PoolID string `json:"pool_id"`
Flavor string `json:"flavor"`
ImageID string `json:"image_id"`
WorkspaceID string `json:"workspace_id"`
Volume VolumeReq `json:"volume"`
JobName string `json:"name"`
Description string `json:"description"`
Duration int64 `json:"duration"` //ms
Feature string `json:"feature"`
PoolID string `json:"pool_id"`
Flavor string `json:"flavor"`
ImageID string `json:"image_id"`
WorkspaceID string `json:"workspace_id"`
Volume VolumeReq `json:"volume"`
EnvVariables CloudBrain2EnvVarReq `json:"env_variables"`
}

type CreateNotebookWithoutPoolParams struct {
@@ -1185,6 +1235,20 @@ type CreateNotebookWithoutPoolParams struct {
Volume VolumeReq `json:"volume"`
}

type CloudBrain2EnvVarReq struct {
CodeObsUrl string `json:"CODE_URL"`
DatasetObsUrl string `json:"DATASET_URL"`
PretrainedModelObsUrl string `json:"PRETRAIN_MODEL_URL"`
OutputObsUrl string `json:"OUTPUT_URL"`
LocalCodePath string `json:"LOCAL_CODE_PATH"`
LocalDatasetPath string `json:"LOCAL_DATASET_PATH"`
LocalPretrainModelPath string `json:"LOCAL_PRETRAIN_MODEL_PATH"`
LocalOutputPath string `json:"LOCAL_OUTPUT_PATH"`
DataDownloadMethod string `json:"DATA_DOWNLOAD_METHOD"`
CodeNeedUnzip string `json:"CODE_NEED_UNZIP"`
DatasetNeedUnzip string `json:"DATASET_NEED_UNZIP"`
PretrainModelNeedUnzip string `json:"PRETRAIN_MODEL_NEED_UNZIP"`
}
type VolumeReq struct {
Capacity int `json:"capacity"`
Category string `json:"category"`
@@ -1606,6 +1670,15 @@ type ModelDownload struct {
IsDelete bool `json:"is_delete"`
}

type Model4Show struct {
ID string `json:"id"`
Name string `json:"name"`
RepositoryLink string `json:"repository_link"`
IsDelete bool `json:"is_delete"`
//DownloadLink string `json:"download_link"`
Size int64 `json:"size"`
}

type DataSource struct {
DatasetID string `json:"dataset_id"`
DatasetVersion string `json:"dataset_version"`
@@ -2035,22 +2108,46 @@ type GetGrampusDebugJobEventsResponse struct {
}

type GrampusTasks struct {
Command string `json:"command"`
Command string `json:"command"`
Name string `json:"name"`
ImageId string `json:"imageId"`
ResourceSpecId string `json:"resourceSpecId"`
ImageUrl string `json:"imageUrl"`
CenterID []string `json:"centerID"`
CenterName []string `json:"centerName"`
ReplicaNum int `json:"replicaNum"`
Datasets []GrampusDataset `json:"datasets"`
Models []GrampusDataset `json:"models"`
Code GrampusDataset `json:"code"`
BootFile string `json:"bootFile"`
OutPut GrampusDataset `json:"output"`
WorkServerNumber int `json:"nodeCount"`
RunParams map[string]interface{} `json:"runParams"`
}
type GrampusNotebookTask struct {
AutoStopDuration int64 `json:"autoStopDuration"`
Name string `json:"name"`
ImageId string `json:"imageId"`
ResourceSpecId string `json:"resourceSpecId"`
ImageUrl string `json:"imageUrl"`
Capacity int `json:"capacity"`
CenterID []string `json:"centerID"`
CenterName []string `json:"centerName"`
ReplicaNum int `json:"replicaNum"`
Datasets []GrampusDataset `json:"datasets"`
Models []GrampusDataset `json:"models"`
PoolId string `json:"poolId"`
Code GrampusDataset `json:"code"`
BootFile string `json:"bootFile"`
Datasets []GrampusDataset `json:"datasets"`
PreTrainModel []GrampusDataset `json:"models"`
OutPut GrampusDataset `json:"output"`
WorkServerNumber int `json:"nodeCount"`
CodeUrl string `json:"codeUrl"`
DataUrl string `json:"dataUrl"`
ImageId string `json:"imageId"`
ImageUrl string `json:"imageUrl"`
ResourceSpecId string `json:"resourceSpecId"`
Token string `json:"token"`
Url string `json:"url"`
Status string `json:"status"`
Command string `json:"command"`
EnvVariables GrampusEnvVarReq `json:"envVariables"`
}
type GrampusNotebookTask struct {

type GrampusInferenceTask struct {
AutoStopDuration int64 `json:"autoStopDuration"`
Name string `json:"name"`
Capacity int `json:"capacity"`
@@ -2059,6 +2156,7 @@ type GrampusNotebookTask struct {
PoolId string `json:"poolId"`
Code GrampusDataset `json:"code"`
Datasets []GrampusDataset `json:"datasets"`
PreTrainModel []GrampusDataset `json:"models"`
OutPut GrampusDataset `json:"output"`
CodeUrl string `json:"codeUrl"`
DataUrl string `json:"dataUrl"`
@@ -2069,6 +2167,8 @@ type GrampusNotebookTask struct {
Url string `json:"url"`
Status string `json:"status"`
Command string `json:"command"`
EnvVariables GrampusEnvVarReq `json:"envVariables"`
BootFile string `json:"bootFile"`
}

type GrampusDataset struct {
@@ -2081,6 +2181,18 @@ type GrampusDataset struct {
GetBackEndpoint string `json:"getBackEndpoint"`
Size int64 `json:"size"`
IsOverwrite bool `json:"isOverwrite"`
IsNeedUnzip bool `json:"isNeedUnzip"`
}

type GrampusEnvVarReq struct {
MoxingRequired string `json:"MOXING_REQUIRED"`
UploadOpeniRequired string `json:"UPLOAD_OPENI_REQUIRED"`
UnzipRequired string `json:"UNZIP_REQUIRED"`
CodePathValue string `json:"CODE_PATH"`
DatasetPathValue string `json:"DATASET_PATH"`
PretrainedModelPathValue string `json:"PRETRAIN_MODEL_PATH"`
OutputPathValue string `json:"OUTPUT_PATH"`
OutputObsUrl string `json:"OUTPUT_URL"`
}

type CreateGrampusJobRequest struct {
@@ -2093,6 +2205,11 @@ type CreateGrampusNotebookRequest struct {
Tasks []GrampusNotebookTask `json:"tasks"`
}

type CreateGrampusInferenceRequest struct {
Name string `json:"name"`
Tasks []GrampusInferenceTask `json:"tasks"`
}

type GetTrainJobMetricStatisticResult struct {
TrainJobResult
Interval int `json:"interval"` //查询的时间间隔,单位为分钟
@@ -2459,8 +2576,9 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) {
func updateReferenceCount(cloudbrain *Cloudbrain) {
increaseDatasetUseCount(cloudbrain.Uuid)
increaseImageUseCount(cloudbrain.Image)
increaseModelReference(cloudbrain.ModelId)

for _, id := range cloudbrain.GetModelIdArray() {
increaseModelReference(id)
}
}

func increaseImageUseCount(image string) {
@@ -3641,13 +3759,13 @@ func LoadSpecs4CloudbrainInfo(tasks []*CloudbrainInfo) error {

func GetCloudBrainByModelId(modelId string) ([]*Cloudbrain, error) {
cloudBrains := make([]*Cloudbrain, 0)
err := x.AllCols().Where("model_id=?", modelId).OrderBy("created_unix desc").Find(&cloudBrains)
err := x.AllCols().Where("model_id like ?", "%"+modelId+"%").OrderBy("created_unix desc").Find(&cloudBrains)
return cloudBrains, err
}

func GetCloudBrainByRepoIdAndModelName(repoId int64, modelName string) ([]*Cloudbrain, error) {
cloudBrains := make([]*Cloudbrain, 0)
err := x.AllCols().Where("model_name=? and repo_id=?", modelName, repoId).OrderBy("created_unix asc").Find(&cloudBrains)
err := x.AllCols().Where("model_name like ? and repo_id=?", "%"+modelName+"%", repoId).OrderBy("created_unix asc").Find(&cloudBrains)
return cloudBrains, err
}



+ 1
- 1
models/model_migrate_record.go View File

@@ -89,7 +89,7 @@ type ModelMigrateRecord struct {
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
DeletedAt time.Time `xorm:"deleted"`
Remark string
Remark string `xorm:"TEXT"`
}

func (r *ModelMigrateRecord) IsFinished() bool {


+ 1
- 0
models/resource_specification.go View File

@@ -357,6 +357,7 @@ func (s *Specification) GetAvailableQueues(opts GetAvailableCenterIdOpts) []Reso
relatedSpecs := s.findRelatedSpecs(specOpts, opts.UserId)

if len(relatedSpecs) == 0 {
log.Info("check centerIds opt%+v relatedSpecs is empty", opts)
return make([]ResourceQueue, 0)
}



+ 1
- 1
modules/grampus/grampus.go View File

@@ -36,7 +36,7 @@ const (
BucketRemote = "grampus"
RemoteModelPath = "/output"
autoStopDurationMs = 4 * 60 * 60 * 1000
CommandGpuDebug = "! [ -x \"$(command -v jupyter)\" ] && pip install jupyterlab==3 -i https://pypi.tuna.tsinghua.edu.cn/simple;jupyter lab --ServerApp.shutdown_no_activity_timeout=%s --TerminalManager.cull_inactive_timeout=%s --TerminalManager.cull_interval=%s --MappingKernelManager.cull_idle_timeout=%s --MappingKernelManager.cull_interval=%s --MappingKernelManager.cull_connected=True --MappingKernelManager.cull_busy=True --no-browser --ip=0.0.0.0 --allow-root --notebook-dir='%s' --port=$OCTOPUS_NOTEBOOK_PORT --LabApp.token='' --LabApp.allow_origin='*' --LabApp.base_url=$OCTOPUS_NOTEBOOK_BASE_URL;"
CommandGpuDebug = "jupyter lab --ServerApp.shutdown_no_activity_timeout=%s --TerminalManager.cull_inactive_timeout=%s --TerminalManager.cull_interval=%s --MappingKernelManager.cull_idle_timeout=%s --MappingKernelManager.cull_interval=%s --MappingKernelManager.cull_connected=True --MappingKernelManager.cull_busy=True --no-browser --ip=0.0.0.0 --allow-root --notebook-dir='%s' --port=$OCTOPUS_NOTEBOOK_PORT --LabApp.token='' --LabApp.allow_origin='*' --LabApp.base_url=$OCTOPUS_NOTEBOOK_BASE_URL;"
)

var (


+ 7
- 0
modules/redis/redis_key/ai_model_redis_key.go View File

@@ -0,0 +1,7 @@
package redis_key

const AI_MODEL_REDIS_PREFIX = "ai_model"

func AIModelMetaUpdateLock(modelId string) string {
return KeyJoin(AI_MODEL_REDIS_PREFIX, modelId, "meta", "lock")
}

+ 4
- 0
modules/setting/setting.go View File

@@ -743,6 +743,8 @@ var (
OUTPUT_SHOW_MAX_KEY int
OUTPUT_DOWNLOAD_MAX_KEY int
SPECIFICATION_SPECIAL_QUEUE string
DEBUG_MODEL_NUM_LIMIT int
DEBUG_MODEL_SIZE_LIMIT_GB int

//wenxin url
BaiduWenXin = struct {
@@ -1617,6 +1619,8 @@ func NewContext() {
OUTPUT_SHOW_MAX_KEY = sec.Key("OUTPUT_SHOW_MAX_KEY").MustInt(100)
OUTPUT_DOWNLOAD_MAX_KEY = sec.Key("OUTPUT_DOWNLOAD_MAX_KEY").MustInt(1000)
SPECIFICATION_SPECIAL_QUEUE = sec.Key("SPECIFICATION_SPECIAL_QUEUE").MustString("{}")
DEBUG_MODEL_NUM_LIMIT = sec.Key("DEBUG_MODEL_NUM_LIMIT").MustInt(5)
DEBUG_MODEL_SIZE_LIMIT_GB = sec.Key("DEBUG_MODEL_SIZE_LIMIT_GB").MustInt(20)

sec = Cfg.Section("benchmark")
IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)


+ 40
- 0
modules/storage/obs.go View File

@@ -902,3 +902,43 @@ func GetDirsSomeFile(bucket string, prefixRootPath string, prefix string, marker
}
return fileInfos, marker, nil
}

func SetObsObjectMetaData(bucket, key string, metaData map[string]string) error {
log.Info("SetObsObjectMetaData bucket=%s key=%s metaData=%v", bucket, key, metaData)
input := &obs.SetObjectMetadataInput{}
input.Bucket = bucket
input.Key = key
input.Metadata = metaData
_, err := ObsCli.SetObjectMetadata(input)
if err != nil {
if obsError, ok := err.(obs.ObsError); ok {
log.Info("Message:%s\n", obsError.Message)
if obsError.StatusCode == 404 {
putInput := &obs.PutObjectInput{}
putInput.Bucket = bucket
putInput.Key = key
ObsCli.PutObject(putInput)
_, err = ObsCli.SetObjectMetadata(input)
}
}
}
return err
}

func GetObsObjectMetaData(bucket, key string) (map[string]string, error) {
log.Info("GetObsObjectMetaData bucket=%s key=%s", bucket, key)
input := &obs.GetObjectMetadataInput{}
input.Bucket = bucket
input.Key = key
output, err := ObsCli.GetObjectMetadata(input)
if err != nil {
if obsError, ok := err.(obs.ObsError); ok {
log.Info("Message:%s\n", obsError.Message)
}
return nil, err
}
if output == nil {
return map[string]string{}, nil
}
return output.Metadata, err
}

+ 7
- 3
options/locale/locale_en-US.ini View File

@@ -418,7 +418,7 @@ new_email_address = New email address
openi_community_really_awesome = OpenI, Really Awesome!
protocol_header=Please read the following content carefully:
protocol_title=Dear OpenI User
protocol_context=Thank you for your continuous support to the Openl Qizhi Community AI Collaboration Platform. In order to protect your usage rights and ensure network security, we updated the Openl Qizhi Community AI Collaboration Platform Usage Agreement in January 2024. The updated agreement specifies that users are prohibited from using intranet penetration tools. After you check and agree, you can continue to use our services. Thank you for your cooperation and understanding.
protocol_context=Thank you for your continuous support to the Openl Qizhi Community AI Collaboration Platform. In order to protect your usage rights and ensure network security, we updated the Openl Qizhi Community AI Collaboration Platform Usage Agreement in January 2024. The updated agreement specifies that users are prohibited from using intranet penetration tools. After you click "Agree and continue", you can continue to use our services. Thank you for your cooperation and understanding.
protocol_context_sub=For more agreement content, please refer to the<u><font color="# 3291f8"><a href="/home/term" target="_blank">《Openl Qizhi Community AI Collaboration Platform Usage Agreement》</a></font></u>
protocol_confirm=Agree and continue
protocol_cancel=Disagree, exit
@@ -990,6 +990,7 @@ dataset_available_clusters = Available Clusters
dataset_upload_time = Upload Time
download = Download
modify_description = Modify Description
copy_code = Copy code
set_public = Set Public
set_private = Set Private
annotation = Image Annotation
@@ -1084,6 +1085,7 @@ language_other = Other
datasets = Datasets
datasets.desc = Enable Dataset
cloudbrain_helper=Use GPU/NPU resources to open notebooks, model training tasks, etc.
code_use_resource=How to access data resources in code
cloudbrain.exitinfo=Exit Information
cloudbrain.platform=Platform
cloudbrain.endtime=End Time
@@ -1151,7 +1153,7 @@ images.name_placerholder = Please enter the image name
images.descr_placerholder = The description should not exceed 1000 characters
image.label_tooltips = Example Python 3.7, Tensorflow 2.0, cuda 10, pytorch 1.6
images.public_tooltips = After the image is set to public, it can be seen by other users.
images.submit_tooltips = The code directory /code, dataset directory /dataset will not be submitted with the image, and other directories will be packaged into the image.
images.submit_tooltips = The code directory /tmp/code, dataset directory /tmp/dataset will not be submitted with the image, and other directories will be packaged into the image.
images.name_format_err=The format of image tag is wrong.
images.name_rule50 = Please enter letters, numbers, _ and - up to 50 characters and starts with a letter.
images.name_rule100 = Please enter letters, numbers, _ and - up to 100 characters and cannot end with a dash (-).
@@ -3491,15 +3493,17 @@ boot_file_must_python = The boot file must be a python file
stop_failed = Fail to stop the job, please try again later.
can_not_restart = The task was not scheduled successfully before, so it cannot be restart.
dataset_size_over_limit = The size of dataset exceeds limitation (%dGB)
model_size_over_limit = The size of model exceeds limitation (%dGB)
boot_file_must_python = The boot file must be a python file
boot_file_not_exist = The boot file is not exists.
branch_not_exists = The branch does not exist. Please refresh and select again.
dataset_number_over_limit = The dataset count exceed the limit
model_number_over_limit = The model count exceed the limit
result_cleared=The files of the task have been cleared, can not restart or retrain any more, please create a new task instead
model_not_exist=The model in the task does not exist or has been deleted
too_many_notebook=A user can have up to 5 debug tasks, please try again after delete some debug tasks.
can_not_stop_creating_job=AI task is creating, can not be stopped.
no_center_match=Can not match a AI center, please select other specification.
no_center_match=Can not match an AI center, please select other specification.

[common_error]
system_error = System error.Please try again later


+ 6
- 2
options/locale/locale_zh-CN.ini View File

@@ -421,7 +421,7 @@ new_email_address=新邮箱地址
openi_community_really_awesome=启智社区 确实给力
protocol_header=请仔细阅读下方内容:
protocol_title=尊敬的启智用户
protocol_context=感谢您一直以来对Openl启智社区AI协作平台的支持。为了保障您的使用权益和确保网络安全,我们于2024年1月份更新了《Openl启智社区AI协作平台使用协议》。更新后的协议明确了用户<font color="#ff2525">禁止使用内网穿透工具</font>的条例。在您勾选同意后,便可以继续使用我们的服务。感谢您的合作与理解。
protocol_context=感谢您一直以来对Openl启智社区AI协作平台的支持。为了保障您的使用权益和确保网络安全,我们于2024年1月份更新了《Openl启智社区AI协作平台使用协议》。更新后的协议明确了用户<font color="#ff2525">禁止使用内网穿透工具</font>的条例。您单击“同意并继续”后,便可以继续使用我们的服务。感谢您的合作与理解。
protocol_context_sub=更多协议内容,请参考<u><font color="#3291f8"><a href="/home/term" target="_blank">《Openl启智社区AI协作平台使用协议》</a></font></u>
protocol_confirm=同意并继续
protocol_cancel=不同意,退出
@@ -994,6 +994,7 @@ dataset_available_clusters = 可用集群
dataset_upload_time = 上传时间
download = 下载
modify_description = 修改描述
copy_code = 复制代码
set_public = 设为公开
set_private = 设为私有
annotation = 图片标注
@@ -1088,6 +1089,7 @@ language_other=其它
datasets=数据集
datasets.desc=数据集功能
cloudbrain_helper=使用GPU/NPU资源,开启Notebook、模型训练任务等
code_use_resource=代码中如何访问数据资源

model_manager = 模型
model_base = 大模型基地
@@ -1150,7 +1152,7 @@ images.name_placerholder = 请输入镜像Tag
images.descr_placerholder = 描述字数不超过1000个字符
image.label_tooltips = 如Python 3.7, Tensorflow 2.0, cuda 10, pytorch 1.6
images.public_tooltips = 镜像设置为公开后,可被其他用户看到。
images.submit_tooltips = 代码目录/code,数据集目录/dataset不会随镜像提交,其他目录都会打包到镜像中。
images.submit_tooltips = 代码目录/tmp/code,数据集目录/tmp/dataset不会随镜像提交,其他目录都会打包到镜像中。
images.name_format_err=镜像Tag格式错误。
images.name_rule50 = 请输入字母、数字、_和-,最长50个字符,且以字母开头。
images.name_rule100 = 请输入字母、数字、_和-,最长100个字符,且不能以中划线(-)结尾。
@@ -3514,10 +3516,12 @@ boot_file_must_python = 启动文件必须是python文件
stop_failed = 任务停止失败,请稍后再试
can_not_restart = 这个任务之前没有调度成功,不能再次调试。
dataset_size_over_limit = 数据集大小超过限制(%dGB)
model_size_over_limit = 模型大小超过限制(%dGB)
boot_file_must_python = 启动文件必须是python文件
boot_file_not_exist =启动文件不存在
branch_not_exists = 代码分支不存在,请刷新后重试
dataset_number_over_limit = 选择的数据集文件数量超出限制
model_number_over_limit = 选择的模型数量超出限制
result_cleared=源任务的文件已被清理,无法再次调试或复用训练结果,请新建任务。
model_not_exist=选择的预训练模型不存在或者已被删除
too_many_notebook=每个用户最多只能创建5个调试任务,请删除历史任务再新建。


+ 9
- 0
routers/ai_task/ai_task.go View File

@@ -567,3 +567,12 @@ func handCreateReq(req *entity.CreateReq) {
req.WorkServerNumber = 1
}
}

func GenerateSDKCode(ctx *context.Context) {
datasetNames := ctx.QueryStrings("dataset_name")
pretrainModelNames := ctx.QueryStrings("pretrain_model_name")
parameterKeys := ctx.QueryStrings("param_key")
jobType := ctx.Query("job_type")
code := task.GenerateSDKCode(datasetNames, pretrainModelNames, parameterKeys, models.JobType(jobType))
ctx.JSON(http.StatusOK, response.OuterSuccessWithData(map[string]string{"code": code}))
}

+ 5
- 1
routers/api/v1/api.go View File

@@ -660,7 +660,6 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/creation/required", reqWeChatStandard(), reqRepoWriter(models.UnitTypeCloudBrain), ai_task.GetCreationRequiredInfo)
m.Get("/creation/image_by_spec", reqWeChatStandard(), reqRepoWriter(models.UnitTypeCloudBrain), ai_task.GetImageInfoBySelectedSpec)
m.Post("/output/reschedule", reqRepoWriter(models.UnitTypeCloudBrain), ai_task.RetryModelSchedule)

}, reqToken(), context.RepoRef())
m.Group("/ai_task", func() {
m.Get("", reqRepoReader(models.UnitTypeCloudBrain), ai_task.GetAITaskInfo)
@@ -677,6 +676,11 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/resource_usage", reqRepoReader(models.UnitTypeCloudBrain), reqAITaskInRepo(), ai_task.GetAITaskResourceUsage)
})
}, repoAssignment())

m.Group("/ai_task", func() {
m.Get("/generate_sdk_code", ai_task.GenerateSDKCode)
})

// Miscellaneous
if setting.API.EnableSwagger {
m.Get("/swagger", misc.Swagger)


+ 1
- 1
routers/repo/ai_model_convert.go View File

@@ -420,7 +420,7 @@ func createGpuTrainJob(modelConvert *models.AiModelConvert, ctx *context.Context
log.Info("command=" + command)
codePath := setting.JobPath + modelConvert.ID + CodeMountPath
codeTmpPath := setting.JobPath + modelConvert.ID + CodeMountPath + "tmp"
uploader := storage_helper.SelectUploaderFromStorageType(entity.MINIO)
uploader := storage_helper.SelectStorageHelperFromStorageType(entity.MINIO)
codeRemoteDir := path.Join(uploader.GetJobDefaultObjectKeyPrefix(modelConvert.ID), "code")
log.Info("codePath=" + codePath)
log.Info("codeTmpPath=" + codeTmpPath)


+ 55
- 4
routers/repo/ai_model_manage.go View File

@@ -2,6 +2,10 @@ package repo

import (
"archive/zip"
"code.gitea.io/gitea/routers/response"
"code.gitea.io/gitea/services/ai_model"
"code.gitea.io/gitea/services/cloudbrain/modelmanage"
"code.gitea.io/gitea/services/cloudbrain/resource"
"encoding/json"
"errors"
"fmt"
@@ -12,10 +16,6 @@ import (
"regexp"
"strings"

"code.gitea.io/gitea/services/cloudbrain/resource"

"code.gitea.io/gitea/services/cloudbrain/modelmanage"

"code.gitea.io/gitea/services/repository"

"code.gitea.io/gitea/models"
@@ -167,6 +167,7 @@ func asyncToCopyModel(aiTask *models.Cloudbrain, id string, modelSelectedFile st
insertModelFile(id)
}
}
ai_model.UpdateModelMeta(id)
}

func insertModelFile(id string) {
@@ -555,6 +556,7 @@ func DeleteModelFile(ctx *context.Context) {
ModelID: id,
}
models.DeleteModelFile(modelFile)
ai_model.UpdateModelMeta(id)
}
}

@@ -585,6 +587,55 @@ func DeleteModel(ctx *context.Context) {
}
}

func UpdateAllModelMeta(ctx *context.Context) {
log.Info("Start to update all model meta")
ids := ctx.QueryStrings("model_id")
updateAll := ctx.QueryBool("update_all")
if !updateAll && len(ids) == 0 {
ctx.JSON(http.StatusOK, response.OuterSuccessWithData(map[string]int{"count": 0}))
return
}
count := 0
if updateAll {
pageSize := 100
pageNum := 1
for {
ids, err := models.QueryModelIdsByPaging(pageSize, pageNum, "created_unix")
if err != nil {
log.Error("UpdateAllModelMeta QueryModelIdsByPaging err.%v", err)
ctx.JSON(http.StatusOK, response.OuterResponseError(err))
return
}
for _, id := range ids {
ai_model.UpdateModelMeta(id)
count++
}
if len(ids) < pageSize {
break
}
pageNum++
}

} else {
for _, id := range ids {
ai_model.UpdateModelMeta(id)
count++
}
}
ctx.JSON(http.StatusOK, response.OuterSuccessWithData(map[string]int{"count": count}))
}

func QueryModelMetaById(ctx *context.Context) {
log.Info("Start to query model meta")
id := ctx.Query("model_id")
data, err := ai_model.QueryModelMeta(id)
if err != nil {
ctx.JSON(http.StatusOK, response.OuterResponseError(err))
return
}
ctx.JSON(http.StatusOK, response.OuterSuccessWithData(map[string]interface{}{"meta_data": data}))
}

func deleteModelByID(ctx *context.Context, id string) error {
log.Info("delete model start. id=" + id)
model, err := models.QueryModelById(id)


+ 32
- 23
routers/repo/ai_model_square.go View File

@@ -39,6 +39,7 @@ type ModelMap struct {
RepoDisplayName string
RepoId int64
Model *models.AiModelManage
Models4Parent []*models.AiModelManage
Next []*ModelMap
}

@@ -317,36 +318,44 @@ func findParent(model *models.AiModelManage) *ModelMap {
} else {
log.Info("find parent model name." + task.ModelName)
if task.ModelName != "" {
result := &ModelMap{
Type: 1,
IsParent: true,
}
modelsArray := make([]*models.AiModelManage, 0)
if task.ModelId != "" {
parentModel, err := models.QueryModelById(task.ModelId)
setModelRepo(parentModel)
setModelUser(parentModel)
setModelDataSet(parentModel)
if err == nil {
re := &ModelMap{
Type: 1,
IsParent: true,
Model: parentModel,
modelIdArray := task.GetModelIdArray()
for i := 0; i < len(modelIdArray); i++ {
modelId := modelIdArray[i]
parentModel, err := models.QueryModelById(modelId)
setModelRepo(parentModel)
setModelUser(parentModel)
setModelDataSet(parentModel)
if err != nil {
return nil
}
return re
modelsArray = append(modelsArray, parentModel)
}
result.Models4Parent = modelsArray
return result
} else {
modelList := models.QueryModelByName(task.ModelName, task.RepoID)
if modelList != nil && len(modelList) > 0 {
for _, parentModel := range modelList {
setModelUser(parentModel)
setModelRepo(parentModel)
setModelDataSet(parentModel)
if parentModel.Version == task.ModelVersion {
re := &ModelMap{
Type: 1,
IsParent: true,
Model: parentModel,
modelNameArray := task.GetModelNameArray()
for i := 0; i < len(modelNameArray); i++ {
modelName := modelNameArray[i]
modelList := models.QueryModelByName(modelName, task.RepoID)
if modelList != nil && len(modelList) > 0 {
for _, parentModel := range modelList {
setModelUser(parentModel)
setModelRepo(parentModel)
setModelDataSet(parentModel)
if parentModel.Version == task.ModelVersion {
modelsArray = append(modelsArray, parentModel)
}
return re
}
}
}
result.Models4Parent = modelsArray
return result
}
}
}
@@ -383,7 +392,7 @@ func findChild(currentNode *ModelMap) {
log.Info("error=" + err.Error())
} else {
log.Info("task.ModelId=%v,currentModel.ID=%v", task.ModelId, currentModel.ID)
if task.ModelId != "" && task.ModelId == currentModel.ID {
if task.ModelId != "" && task.HasUseModel(currentModel.ID) {
setModelUser(childModel)
setModelDataSet(childModel)
modelMap := &ModelMap{


+ 5
- 2
routers/repo/attachment_model.go View File

@@ -1,6 +1,7 @@
package repo

import (
"code.gitea.io/gitea/services/ai_model"
"errors"
"fmt"
"path"
@@ -20,7 +21,7 @@ import (
func GetModelChunks(ctx *context.Context) {
fileMD5 := ctx.Query("md5")
typeCloudBrain := ctx.QueryInt("type")
fileName := ctx.Query("file_name")
//fileName := ctx.Query("file_name")
//scene := ctx.Query("scene")
modeluuid := ctx.Query("modeluuid")
log.Info(" typeCloudBrain=" + fmt.Sprint(typeCloudBrain))
@@ -85,6 +86,7 @@ func GetModelChunks(ctx *context.Context) {
if err == nil && model != nil {
modelname = model.Name
}
fileNameUploaded := strings.Split(fileChunk.ObjectName, modeluuid+"/")[1]
ctx.JSON(200, map[string]string{
"uuid": fileChunk.UUID,
"uploaded": strconv.Itoa(fileChunk.IsUploaded),
@@ -92,7 +94,7 @@ func GetModelChunks(ctx *context.Context) {
"chunks": string(chunks),
"attachID": "0",
"modeluuid": modeluuid,
"fileName": fileName,
"fileName": fileNameUploaded,
"modelName": modelname,
})
} else {
@@ -323,6 +325,7 @@ func CompleteModelMultipart(ctx *context.Context) {
}
//更新模型大小信息
UpdateModelSize(modeluuid, fileChunk.ObjectName)
ai_model.UpdateModelMeta(modeluuid)

ctx.JSON(200, map[string]string{
"result_code": "0",


+ 39
- 40
routers/repo/modelarts.go View File

@@ -68,53 +68,52 @@ func DebugJobIndex(ctx *context.Context) {
if listType == "" {
listType = models.AllResource
}
ctx.Data["ListType"] = listType
MustEnableCloudbrain(ctx)
repo := ctx.Repo.Repository
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
}

jobTypeNot := false
var computeResource string
if listType != models.AllResource {
computeResource = listType
}

var jobTypes []string
jobTypes = append(jobTypes, string(models.JobTypeDebug))
ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{
Page: page,
PageSize: setting.UI.IssuePagingNum,
},
RepoID: repo.ID,
ComputeResource: computeResource,
Type: models.TypeCloudBrainAll,
JobTypeNot: jobTypeNot,
JobTypes: jobTypes,
})
if err != nil {
ctx.ServerError("Get debugjob faild:", err)
return
}

for i, task := range ciTasks {
ciTasks[i].CanDebug = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain)
ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
}

pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
pager.AddParam(ctx, "debugListType", "ListType")
ctx.Data["Page"] = pager
ctx.Data["ListType"] = listType
ctx.Data["PageIsCloudBrain"] = true
ctx.Data["Tasks"] = ciTasks
ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx)
ctx.Data["RepoIsEmpty"] = repo.IsEmpty
ctx.Data["debugListType"] = listType
ctx.HTML(200, tplDebugJobIndex)

// page := ctx.QueryInt("page")
// if page <= 0 {
// page = 1
// }

// jobTypeNot := false
// var computeResource string
// if listType != models.AllResource {
// computeResource = listType
// }

// var jobTypes []string
// jobTypes = append(jobTypes, string(models.JobTypeDebug))
// ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
// ListOptions: models.ListOptions{
// Page: page,
// PageSize: setting.UI.IssuePagingNum,
// },
// RepoID: repo.ID,
// ComputeResource: computeResource,
// Type: models.TypeCloudBrainAll,
// JobTypeNot: jobTypeNot,
// JobTypes: jobTypes,
// })
// if err != nil {
// ctx.ServerError("Get debugjob faild:", err)
// return
// }

// for i, task := range ciTasks {
// ciTasks[i].CanDebug = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain)
// ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
// ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
// }

// pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
// pager.AddParam(ctx, "debugListType", "ListType")
}

// MustEnableDataset check if repository enable internal cb


+ 3
- 1
routers/response/response_list.go View File

@@ -38,4 +38,6 @@ var MODEL_NUM_OVER_LIMIT = &BizError{Code: 2021, DefaultMsg: "The number of mode
var DATASET_NUMBER_OVER_LIMIT = &BizError{Code: 2022, DefaultMsg: "The dataset count exceed the limit", TrCode: "ai_task.dataset_number_over_limit"}
var NOTEBOOK_EXCEED_MAX_NUM = &BizError{Code: 2023, DefaultMsg: "You can have up to 5 Debug Tasks, please try again after delete some tasks. ", TrCode: "ai_task.too_many_notebook"}
var CAN_NOT_STOP_CREATING_JOB = &BizError{Code: 2024, DefaultMsg: "AI task is creating, can not be stopped", TrCode: "ai_task.can_not_stop_creating_job"}
var NO_CENTER_MATCH = &BizError{Code: 2024, DefaultMsg: "", TrCode: "ai_task.no_center_match"}
var NO_CENTER_MATCH = &BizError{Code: 2024, DefaultMsg: "Can not match an AI center, please select other specification.", TrCode: "ai_task.no_center_match"}
var MODEL_NUMBER_OVER_LIMIT = &BizError{Code: 2025, DefaultMsg: "The model count exceed the limit", TrCode: "ai_task.model_number_over_limit"}
var MODEL_SIZE_OVER_LIMIT = &BizError{Code: 2026, DefaultMsg: "The size of model exceeds limitation", TrCode: "ai_task.model_size_over_limit"}

+ 4
- 0
routers/routes/routes.go View File

@@ -741,6 +741,10 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/update/:id", binding.BindIgnErr(models.ResourceSceneReq{}), admin.UpdateResourceScene)
})
})
m.Group("/ai_model", func() {
m.Post("/update_version", repo.UpdateAllModelMeta)
m.Get("/query_meta", repo.QueryModelMetaById)
})
}, adminReq)
// ***** END: Admin *****



+ 74
- 0
services/ai_model/model_version.go View File

@@ -0,0 +1,74 @@
package ai_model

import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/obs"
"code.gitea.io/gitea/modules/redis/redis_key"
"code.gitea.io/gitea/modules/redis/redis_lock"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"encoding/json"
"errors"
"io"
"strings"
"time"
)

const versionKey = "data_version"
const versionFileName = "openi_resource.version"

//UpdateModelMeta 更新模型的版本信息
func UpdateModelMeta(modelId string) error {
lock := redis_lock.NewDistributeLock(redis_key.AIModelMetaUpdateLock(modelId))
success, err := lock.LockWithWait(3*time.Second, 3*time.Second)
if err != nil {
return err
}
if !success {
return errors.New("InitModelMeta err")
}
defer lock.UnLock()
m, err := models.QueryModelById(modelId)
if err != nil {
return err
}
info := map[string]interface{}{versionKey: time.Now().Unix()}
metaInfo, _ := json.Marshal(info)
input := &obs.PutObjectInput{}
input.Bucket = setting.Bucket
input.Key = m.Path[len(setting.Bucket)+1:] + versionFileName
input.Body = strings.NewReader(string(metaInfo))
_, err = storage.ObsCli.PutObject(input)
return err
}

func QueryModelMeta(modelId string) (string, error) {
m, err := models.QueryModelById(modelId)
if err != nil {
log.Error("QueryModelMeta QueryModelById err.%v")
return "", err
}
input := &obs.GetObjectInput{}
input.Bucket = setting.Bucket
input.Key = m.Path[len(setting.Bucket)+1:] + versionFileName
data, err := storage.ObsCli.GetObject(input)

if err != nil {
log.Error("QueryModelMeta GetObject err.%v")
return "", err
}
if data == nil {
return "", nil
}
s, _ := io.ReadAll(data.Body)
return string(s), nil
}

func InitModelMeta(modelId string) {
data, _ := QueryModelMeta(modelId)
if data != "" {
return
}
UpdateModelMeta(modelId)
}

+ 65
- 232
services/ai_task_service/cluster/c2net.go View File

@@ -11,10 +11,8 @@ import (
"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/manager/client/grampus"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain"
model_grampus "code.gitea.io/gitea/modules/grampus"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/timeutil"
@@ -57,7 +55,7 @@ func (c C2NetClusterAdapter) CreateOnlineInfer(req entity.CreateNoteBookTaskRequ
log.Error("CreateOnlineInfer err.req=%+v err=%v", req, err)
return nil, err
}
jobResult, err := grampus.CreateNotebookJob(newReq)
jobResult, err := grampus.CreateInferenceJob(newReq)
if err != nil {
log.Error("CreateNoteBook failed: %v", err.Error())
return nil, err
@@ -127,7 +125,8 @@ func convertNoteBookReq2Grampus(req entity.CreateNoteBookTaskRequest) (models.Cr
codePath = codePath[0:strings.LastIndex(codePath, "/")]
}
}
var commandGpuDebug = "mkdir -p /dataset;! [ -x \"$(command -v jupyter)\" ] && pip install jupyterlab==3 -i https://pypi.tuna.tsinghua.edu.cn/simple;jupyter lab --ServerApp.shutdown_no_activity_timeout=%s --TerminalManager.cull_inactive_timeout=%s --TerminalManager.cull_interval=%s --MappingKernelManager.cull_idle_timeout=%s --MappingKernelManager.cull_interval=%s --MappingKernelManager.cull_connected=True --MappingKernelManager.cull_busy=True --no-browser --ip=0.0.0.0 --allow-root --notebook-dir='%s' --port=$OCTOPUS_NOTEBOOK_PORT --LabApp.token='' --LabApp.allow_origin='*' --LabApp.base_url=$OCTOPUS_NOTEBOOK_BASE_URL;"

var commandGpuDebug = "mkdir -p /tmp/dataset;jupyter lab --ServerApp.shutdown_no_activity_timeout=%s --TerminalManager.cull_inactive_timeout=%s --TerminalManager.cull_interval=%s --MappingKernelManager.cull_idle_timeout=%s --MappingKernelManager.cull_interval=%s --MappingKernelManager.cull_connected=True --MappingKernelManager.cull_busy=True --no-browser --ip=0.0.0.0 --allow-root --notebook-dir='%s' --port=$OCTOPUS_NOTEBOOK_PORT --LabApp.token='' --LabApp.allow_origin='*' --LabApp.base_url=$OCTOPUS_NOTEBOOK_BASE_URL;"
command := fmt.Sprintf(commandGpuDebug, setting.CullIdleTimeout, setting.CullIdleTimeout, setting.CullInterval, setting.CullIdleTimeout, setting.CullInterval, codePath)
if models.DCU == req.Tasks[0].Spec.ComputeResource {
command = "cp -r /code /tmp;cp -r /dataset /tmp;cp -r /pretrainmodel /tmp;"
@@ -149,21 +148,22 @@ func convertNoteBookReq2Grampus(req entity.CreateNoteBookTaskRequest) (models.Cr
return models.CreateGrampusNotebookRequest{Name: req.Name, Tasks: tasks}, nil
}

func convertOnlineInfer2Grampus(req entity.CreateNoteBookTaskRequest) (models.CreateGrampusNotebookRequest, error) {
func convertOnlineInfer2Grampus(req entity.CreateNoteBookTaskRequest) (models.CreateGrampusInferenceRequest, error) {

command := generateCommand(req.RepoName, req.Tasks[0].BootFile, req.PrimitiveDatasetName)
command := ""
//:= generateCommand(req.RepoName, req.Tasks[0].BootFile, req.PrimitiveDatasetName)

tasks := make([]models.GrampusNotebookTask, len(req.Tasks))
tasks := make([]models.GrampusInferenceTask, len(req.Tasks))
for i := 0; i < len(req.Tasks); i++ {
t := req.Tasks[i]
task, err := convertNoteBookTask2Grampus(t, command)
task, err := convertOnlineInference2Grampus(t, command)
if err != nil {
return models.CreateGrampusNotebookRequest{}, nil
return models.CreateGrampusInferenceRequest{}, nil
}
tasks[i] = task
}

return models.CreateGrampusNotebookRequest{Name: req.Name, Tasks: tasks}, nil
return models.CreateGrampusInferenceRequest{Name: req.Name, Tasks: tasks}, nil
}

func generateCommand(repoName, bootFile, datasetName string) string {
@@ -191,7 +191,8 @@ func generateCommand(repoName, bootFile, datasetName string) string {
copyDatasetPath := "/code/" + strings.ToLower(repoName) + "/" + bootfilepath
commandUnzip := "export OPENI_GRADIO_URL=$OCTOPUS_NOTEBOOK_BASE_URL;" + "cd " + workDir + "code;echo \"start unzip code\";unzip -q master.zip; " + copyDatasetCmd + " echo \"start to unzip dataset\";cd " + copyDatasetPath + "; " + unZipDatasetCommand
//commandUnzip := "cd " + workDir + "code;echo \"start unzip code\";unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand
command += commandUnzip

command = command + commandUnzip
command += "echo \"unzip finished;start to exec code;\";"
if strings.HasSuffix(bootonlyfile, ".py") {
bootonlyfile = bootonlyfile[0 : len(bootonlyfile)-3]
@@ -212,6 +213,41 @@ func getCopyCmd(datasetName, repoName, bootfilepath string) string {
return cmd
}

func convertOnlineInference2Grampus(t entity.NoteBookTask, command string) (models.GrampusInferenceTask, error) {
code := models.GrampusDataset{}
codeArray := convertContainerArray2GrampusArray(t.Code)
if codeArray != nil && len(codeArray) > 0 {
code = codeArray[0]
}
output := models.GrampusDataset{}
outputArray := convertContainerArray2GrampusArray(t.OutPut)
if outputArray != nil && len(outputArray) > 0 {
output = outputArray[0]
}
centerIds, err := getGrampusAvailableCenterIds(t.Queues, t.ImageId, *models.GetComputeSourceInstance(t.Spec.ComputeResource), models.JobTypeDebug)
log.Info("check centerIds getGrampusAvailableCenterIds ImageId=%s queues=%v centerIds=%v", t.ImageId, t.Queues, centerIds)
if err != nil {
log.Error("check centerIds getGrampusAvailableCenterIds err.%v", err)
return models.GrampusInferenceTask{}, err
}
return models.GrampusInferenceTask{
Name: t.Name,
ResourceSpecId: t.Spec.SourceSpecId,
ImageId: t.ImageId,
ImageUrl: t.ImageUrl,
Datasets: convertContainerArray2GrampusArray(t.Datasets),
PreTrainModel: convertContainerArray2GrampusArray(t.PreTrainModel),
Code: code,
OutPut: output,
EnvVariables: t.EnvVariables,
AutoStopDuration: t.AutoStopDuration,
Capacity: t.Capacity,
Command: command,
CenterID: centerIds,
BootFile: t.BootFile,
}, nil
}

func convertNoteBookTask2Grampus(t entity.NoteBookTask, command string) (models.GrampusNotebookTask, error) {
code := models.GrampusDataset{}
codeArray := convertContainerArray2GrampusArray(t.Code)
@@ -233,8 +269,10 @@ func convertNoteBookTask2Grampus(t entity.NoteBookTask, command string) (models.
ImageId: t.ImageId,
ImageUrl: t.ImageUrl,
Datasets: convertContainerArray2GrampusArray(t.Datasets),
PreTrainModel: convertContainerArray2GrampusArray(t.PreTrainModel),
Code: code,
OutPut: output,
EnvVariables: t.EnvVariables,
AutoStopDuration: t.AutoStopDuration,
Capacity: t.Capacity,
Command: command,
@@ -307,6 +345,15 @@ func convertContainerArray2Grampus(containerDatas []entity.ContainerData) models
return res
}

func convertParameters2Grampus(parameters models.Parameters) map[string]interface{} {
req := make(map[string]interface{})
for _, param := range parameters.Parameter {
req[param.Label] = param.Value
}

return req
}

func convertContainer2Grampus(d entity.ContainerData) models.GrampusDataset {
return models.GrampusDataset{
Name: d.Name,
@@ -318,6 +365,7 @@ func convertContainer2Grampus(d entity.ContainerData) models.GrampusDataset {
GetBackEndpoint: d.GetBackEndpoint,
Size: d.Size,
IsOverwrite: d.IsOverwrite,
IsNeedUnzip: d.IsNeedUnzip,
}
}

@@ -498,7 +546,7 @@ func (c C2NetClusterAdapter) CreateTrainJob(req entity.CreateTrainTaskRequest) (
}

func convertTrainReq2Grampus(req entity.CreateTrainTaskRequest) (models.CreateGrampusJobRequest, error) {
command := generateGrampusTrainCommand(req)
command := ""

tasks := make([]models.GrampusTasks, len(req.Tasks))
for i := 0; i < len(req.Tasks); i++ {
@@ -513,222 +561,6 @@ func convertTrainReq2Grampus(req entity.CreateTrainTaskRequest) (models.CreateGr
return models.CreateGrampusJobRequest{Name: req.Name, Tasks: tasks}, nil
}

func generateGrampusTrainCommand(req entity.CreateTrainTaskRequest) string {
t := req.Tasks[0]
containerConfig := req.TaskConfig
computeResource := t.Spec.ComputeResource
var codePath = containerConfig.GetContainerPath(entity.ContainerCode)
var modelPath = containerConfig.GetContainerPath(entity.ContainerPreTrainModel)
var datasetPath = containerConfig.GetContainerPath(entity.ContainerDataset)
var outputPath = containerConfig.GetContainerPath(entity.ContainerOutPutPath)

var modelFilePath = ""
if t.PreTrainModel != nil && len(t.PreTrainModel) > 0 {
modelFilePath = t.PreTrainModel[0].ContainerPath
}
builder := &entity.CommandBuilder{}
builder.
//mkdir dirs
Add(buildMkdirCommand(codePath, modelPath, datasetPath, outputPath)).
//unzip code
Add(buildUnzipCodeCommand(codePath, t.Code[0].ContainerPath, computeResource)).
//unzip dataset
Add(buildUnzipDatasetCommand(t.Datasets, datasetPath, computeResource)).
//export
Add(buildExportCommand(req.Name, computeResource)).
//exec code
Add(buildExecCodeCommand(path.Join(codePath, strings.ToLower(t.RepoName)), modelFilePath, t.BootFile, computeResource, req.Name, t.Params, t.Datasets, datasetPath))

return builder.ToString()
}

func buildMkdirCommand(dirs ...string) *entity.CommandBuilder {
builder := &entity.CommandBuilder{}
for _, dir := range dirs {
builder.Next(entity.NewCommand("mkdir", "-p", dir))
}
return builder
}

func buildUnzipCodeCommand(codeConfigPath, codeFilePath, computeSource string) *entity.CommandBuilder {
builder := &entity.CommandBuilder{}
if computeSource == models.NPU {
return builder
}
builder.
Next(entity.NewCommand("echo", "'start to unzip code'")).
Next(entity.NewCommand("cd", codeConfigPath)).
Next(entity.NewCommand("unzip", "-q", codeFilePath)).
Next(entity.NewCommand("echo", "'unzip code finished'")).
Next(entity.NewCommand("ls", "-l")).
Next(entity.NewCommand("ls", "-l", "mnist_pytorchexample_gpu"))
return builder
}
func buildUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath, computeSource string) *entity.CommandBuilder {
builder := &entity.CommandBuilder{}
if computeSource == models.NPU {
return builder
}
if len(datasets) == 0 {
return nil
}
builder.Next(entity.NewCommand("cd", datasetPath)).
Next(entity.NewCommand("echo", "'start to unzip datasets'"))

fileDatasets := make([]entity.ContainerData, 0)
for _, dataset := range datasets {
if !dataset.IsDir {
fileDatasets = append(fileDatasets, dataset)
}
}
//单数据集
if len(fileDatasets) == 1 {
if strings.HasSuffix(fileDatasets[0].Name, ".tar.gz") {
builder.Next(entity.NewCommand("tar", "--strip-components=1", "-zxvf", "'"+fileDatasets[0].Name+"'"))
} else {
builder.Next(entity.NewCommand("unzip", "-q", "'"+fileDatasets[0].Name+"'"))
}
builder.Next(entity.NewCommand("ls", "-l"))
builder.Next(entity.NewCommand("echo", "'unzip datasets finished'"))
return builder
}
//多数据集
for i := 0; i < len(fileDatasets); i++ {
name := fileDatasets[i].Name
if strings.HasSuffix(name, ".tar.gz") {
builder.Next(entity.NewCommand("tar", "-zxvf", name))
} else {
builder.Next(entity.NewCommand("unzip", "-q", "'"+name+"'", "-d", "'./"+strings.TrimSuffix(name, ".zip")+"'"))
}
}
builder.Next(entity.NewCommand("ls", "-l"))
builder.Next(entity.NewCommand("echo", "'unzip datasets finished'"))
return builder
}

func buildDeleteUnzipDatasetCommand(builder *entity.CommandBuilder, datasets []entity.ContainerData, datasetPath, computeSource string) {
if computeSource == models.NPU {
return
}
if len(datasets) == 0 {
return
}
builder.Next(entity.NewCommand("cd", datasetPath)).
Next(entity.NewCommand("echo", "'start to delete unzip datasets'"))

fileDatasets := make([]entity.ContainerData, 0)
for _, dataset := range datasets {
if !dataset.IsDir {
fileDatasets = append(fileDatasets, dataset)
}
}
//单数据集
if len(fileDatasets) == 1 {

builder.Next(entity.NewCommand("find . ! -name", "'"+fileDatasets[0].Name+"'", "-type f -exec rm -f {} +"))
builder.Next(entity.NewCommand("find . -type d ! -name", "'"+fileDatasets[0].Name+"'", "-and ! -name . -and ! -name .. -exec rm -rf {} +"))

} else {
//多数据集
for i := 0; i < len(fileDatasets); i++ {

builder.Next(entity.NewCommand("rm", "-rf", "'"+getZipFileNameExcludeExt(fileDatasets[i].Name)+"'"))

}
}
builder.Next(entity.NewCommand("ls", "-l"))
builder.Next(entity.NewCommand("echo", "'delete unzip datasets finished'"))
}

func getZipFileNameExcludeExt(fileName string) string {
if strings.HasSuffix(fileName, ".tar.gz") {
return fileName[0 : len(fileName)-7]
} else if strings.HasSuffix(fileName, ".zip") {
return fileName[0 : len(fileName)-4]
}
return fileName
}

func buildExportCommand(jobName, computeResource string) *entity.CommandBuilder {
builder := &entity.CommandBuilder{}

if computeResource == models.NPU {
outputRemotePath := setting.CodePathPrefix + jobName + modelarts.OutputPath
builder.Next(entity.NewCommand("export", "bucket="+setting.Grampus.Env, "&&", "export", "remote_path="+outputRemotePath))
} else {
outputRemotePath := setting.CBCodePathPrefix + jobName + cloudbrain.ModelMountPath + "/"
builder.Next(entity.NewCommand("export", "env="+setting.Grampus.Env, "&&", "export", "remote_path="+outputRemotePath))
}
return builder
}

func buildExecCodeCommand(codeDirPath, modelFilePath, bootFile, computeResource, jobName string, params models.Parameters, datasets []entity.ContainerData, datasetPath string) *entity.CommandBuilder {
builder := &entity.CommandBuilder{}
builder.Next(entity.NewCommand("echo", "'start to exec code'"))

var paramCode string
for _, param := range params.Parameter {
paramCode += " --'" + param.Label + "'='" + param.Value + "'"
}
if computeResource == models.NPU {
builder.Next(entity.NewCommand("source", "/home/ma-user/.bashrc")).
Next(entity.NewCommand("export", "GLOG_v=3")).
Next(entity.NewCommand("export", "ASCEND_GLOBAL_LOG_LEVEL=3")).
Next(entity.NewCommand("export", "ASCEND_SLOG_PRINT_TO_STDOUT=0 ")).
Next(entity.NewCommand("export", "HCCL_CONNECT_TIMEOUT=3600")).
Next(entity.NewCommand("export", "HCCL_EXEC_TIMEOUT=1800")).
Next(entity.NewCommand("export", "PIPELINE_SLICE_SKIP_REDISTRIBUTION=1")).
Next(entity.NewCommand("export", "MS_DEV_REDUNDANCY_TASK_NUM=4")).
Next(entity.NewCommand("export", "MS_DEV_CELL_REUSE=2")).
Next(entity.NewCommand("python", "/home/ma-user/davinci/train/davincirun.py", "python", "/home/ma-user/grampus.py", paramCode))
} else if computeResource == models.GCU {
builder.Next(entity.NewCommand("cd", codeDirPath))
if modelFilePath != "" {
builder.Next(entity.NewCommand("python3", bootFile, paramCode, "--ckpt_url='"+modelFilePath+"'"))
} else {
builder.Next(entity.NewCommand("python3", bootFile, paramCode))
}
} else {
builder.Next(entity.NewCommand("cd", codeDirPath))
if modelFilePath != "" {
builder.Next(entity.NewCommand("python", bootFile, paramCode, "--ckpt_url='"+modelFilePath+"'"))
} else {
builder.Next(entity.NewCommand("python", bootFile, paramCode))
}
}

builder.Next(entity.NewCommand("result=$?"))
//delete unzip dataset
buildDeleteUnzipDatasetCommand(builder, datasets, datasetPath, computeResource)

builder.Next(entity.NewCommand("bash", "-c", "\"[[ $result -eq 0 ]] && exit 0 || exit -1\""))
return builder
}

func buildParamCommand(outputRemotePath, computeResource string) *entity.CommandBuilder {
builder := &entity.CommandBuilder{}
builder.Next(entity.NewCommand("echo", "'start to exec code'"))

if computeResource == models.NPU {
builder.Next(entity.NewCommand("export", "bucket="+setting.Grampus.Env, "&&", "export", "remote_path="+outputRemotePath))
} else {
builder.Next(entity.NewCommand("export", "env="+setting.Grampus.Env, "&&", "export", "remote_path="+outputRemotePath))
}

return builder
}

var BucketRemote = "grampus"
var RemoteModelPath = "/output"

func getNpuModelRemoteObsUrl(jobName string) string {
return "s3:///" + BucketRemote + "/" + getNpuModelObjectKey(jobName)
}

func getNpuModelObjectKey(jobName string) string {
return setting.CodePathPrefix + jobName + RemoteModelPath + "/" + models.ModelSuffix
}

func convertTrainTask2Grampus(t entity.TrainTask, command string) (models.GrampusTasks, error) {
centerIds, err := getGrampusAvailableCenterIds(t.Queues, t.ImageId, *models.GetComputeSourceInstance(t.Spec.ComputeResource), models.JobTypeTrain)
if err != nil {
@@ -749,6 +581,7 @@ func convertTrainTask2Grampus(t entity.TrainTask, command string) (models.Grampu
BootFile: t.BootFile,
OutPut: convertContainerArray2Grampus(t.OutPut),
WorkServerNumber: t.WorkServerNumber,
RunParams: convertParameters2Grampus(t.Params),
}, nil
}

@@ -869,7 +702,7 @@ func (c C2NetClusterAdapter) GetLog(opts entity.ClusterLogOpts) (*entity.Cluster
}

func getOnlineInferenceLog(opts entity.ClusterLogOpts) string {
helper := storage_helper.SelectUploaderFromStorageType(entity.MINIO)
helper := storage_helper.SelectStorageHelperFromStorageType(entity.MINIO)
//查找日志文件
files := getLogFilesInStorage(helper, helper.GetJobDefaultObjectKeyPrefix(opts.JobName)+"/model", ".txt")
if len(files) == 0 {
@@ -924,7 +757,7 @@ func (c C2NetClusterAdapter) GetLogDownloadInfo(opts entity.ClusterLogDownloadIn
}

func (c C2NetClusterAdapter) GetSingleOutputDownloadInfo(opts entity.ClusterSingleOutputDownloadInfoOpts) (*entity.FileDownloadInfo, error) {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
url, err := helper.GetSignedDownloadUrl(opts.Path)
if err != nil {
log.Error("GetSignedDownloadUrl err.opts=%+v,err =%v", opts, err)
@@ -1040,7 +873,7 @@ func (c C2NetClusterAdapter) GetOutput(opts entity.ClusterOutputOpts) (*entity.C
}, nil
}

helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
fileList, err := helper.GetOneLevelObjectsUnderDir(path.Join(opts.ObjectKeyPrefix, opts.ParentDir))
if err != nil {
log.Error("GetOneLevelObjectsUnderDir err.objectKeyPrefix=%s,err=%v", opts.ObjectKeyPrefix, err)
@@ -1063,7 +896,7 @@ func (c C2NetClusterAdapter) GetAllOutput(opts entity.ClusterOutputOpts) (*entit
return &entity.AllAITaskOutput{FileList: []storage.FileInfo{}}, nil
}

helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
fileList, err := helper.GetAllObjectsUnderDir(path.Join(opts.ObjectKeyPrefix, opts.ParentDir))
if err != nil {
log.Error("GetOneLevelObjectsUnderDir err.objectKeyPrefix=%s,err=%v", opts.ObjectKeyPrefix, err)


+ 14
- 13
services/ai_task_service/cluster/cloudbrain_one.go View File

@@ -2,18 +2,19 @@ package cluster

import "C"
import (
"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/ai_task_service/storage_helper"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"path"
"strings"

"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/ai_task_service/storage_helper"
)

type CloudbrainOneClusterAdapter struct {
@@ -236,7 +237,7 @@ func parseDiagnosticsToOperationProfile(appExitDiagnostics string, exitDiagnosti
func (c CloudbrainOneClusterAdapter) CreateTrainJob(req entity.CreateTrainTaskRequest) (*entity.CreateTrainTaskResponse, error) {
jobResult, err := cloudbrain.CreateJob(req.Name, convertTrainJobReq2CloudbrainOne(req))
if err != nil {
log.Error("CreateNoteBook failed: %v", err.Error())
log.Error("CreateTrainJob failed: %v", err.Error())
return nil, err
}
return convertCloudbrainOne2TrainJobRes(jobResult), nil
@@ -348,7 +349,7 @@ func (c CloudbrainOneClusterAdapter) GetLog(opts entity.ClusterLogOpts) (*entity
//获取任务退出信息
existStr := getCloudbrainOneExitDiagnostics(opts.JobId)

helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)

//查找日志文件
files := getLogFilesInStorage(helper, opts.ObjectKeyPrefix, "log.txt")
@@ -389,7 +390,7 @@ func (c CloudbrainOneClusterAdapter) GetLogDownloadInfo(opts entity.ClusterLogDo
//获取任务退出信息
existStr := getCloudbrainOneExitDiagnostics(opts.JobId)

helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)

//查找日志文件
files := getLogFilesInStorage(helper, opts.ObjectKeyPrefix, "log.txt")
@@ -428,7 +429,7 @@ func (c CloudbrainOneClusterAdapter) GetLogDownloadInfo(opts entity.ClusterLogDo
}

func (c CloudbrainOneClusterAdapter) GetSingleOutputDownloadInfo(opts entity.ClusterSingleOutputDownloadInfoOpts) (*entity.FileDownloadInfo, error) {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
url, err := helper.GetSignedDownloadUrl(opts.Path)
if err != nil {
log.Error("GetSignedDownloadUrl err.opts=%+v,err =%v", opts, err)
@@ -492,7 +493,7 @@ func getCloudbrainOneExitDiagnostics(jobId string) string {
return ""
}

//findStartAndEnd 基于baseLine,根据方向向上或者向下计算
// findStartAndEnd 基于baseLine,根据方向向上或者向下计算
func findStartAndEnd(opts entity.ClusterLogOpts, filePath string, helper storage_helper.StorageHelper) (startLine int64, endLine int64) {
baseLine := opts.BaseLine
if opts.Direction == entity.UP {
@@ -517,7 +518,7 @@ func (c CloudbrainOneClusterAdapter) GetTrainJobOperationProfile(jobId string) (
}

func (c CloudbrainOneClusterAdapter) GetOutput(opts entity.ClusterOutputOpts) (*entity.ClusterAITaskOutput, error) {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
fileList, err := helper.GetOneLevelObjectsUnderDir(path.Join(opts.ObjectKeyPrefix, opts.ParentDir))
if err != nil {
log.Error("GetOneLevelObjectsUnderDir err.objectKeyPrefix=%s,err=%v", opts.ObjectKeyPrefix, err)
@@ -531,7 +532,7 @@ func (c CloudbrainOneClusterAdapter) GetOutput(opts entity.ClusterOutputOpts) (*
}

func (c CloudbrainOneClusterAdapter) GetAllOutput(opts entity.ClusterOutputOpts) (*entity.AllAITaskOutput, error) {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
fileList, err := helper.GetAllObjectsUnderDir(path.Join(opts.ObjectKeyPrefix, opts.ParentDir))
if err != nil {
log.Error("GetAllObjectsUnderDir err.objectKeyPrefix=%s,err=%v", opts.ObjectKeyPrefix, err)


+ 51
- 8
services/ai_task_service/cluster/cloudbrain_two.go View File

@@ -31,6 +31,17 @@ func init() {
func (c CloudbrainTwoClusterAdapter) CreateNoteBook(req entity.CreateNoteBookTaskRequest) (*entity.CreateNoteBookTaskResponse, error) {
t := req.Tasks[0]

appUrl := JointCloudbrainTwoReqUrl(t.Code)
if appUrl != "" {
appUrl = "s3:/" + appUrl
}
trainUrl := JointCloudbrainTwoReqUrl(t.OutPut)
if trainUrl != "" {
trainUrl = "s3:/" + trainUrl
}
datasetUrl := getCloudbrainTwoMultiDataUrl(t.Datasets)
multiModelUrl := getCloudbrainTwoModelUrl(t.PreTrainModel)

var jobResult *models.CreateNotebookResult
var err error
if setting.ModelartsCD.Enabled {
@@ -62,6 +73,20 @@ func (c CloudbrainTwoClusterAdapter) CreateNoteBook(req entity.CreateNoteBookTas
Category: models.EVSCategory,
Ownership: models.ManagedOwnership,
},
EnvVariables: models.CloudBrain2EnvVarReq{
CodeObsUrl: appUrl,
DatasetObsUrl: datasetUrl,
PretrainedModelObsUrl: multiModelUrl,
OutputObsUrl: trainUrl,
LocalCodePath: models.LocalCodePath,
LocalDatasetPath: models.LocalDatasetPath,
LocalPretrainModelPath: models.LocalPretrainModelPath,
LocalOutputPath: models.LocalOutputPath,
DataDownloadMethod: models.DataDownloadMethodMoxing,
CodeNeedUnzip: models.CodeNeedUnzipTrue,
DatasetNeedUnzip: models.DatasetNeedUnzipTrue,
PretrainModelNeedUnzip: models.PretrainModelNeedUnzipFalse,
},
WorkspaceID: "0",
})
}
@@ -516,7 +541,25 @@ func getCloudbrainTwoUserCommand(appUrl, bootFile, dataUrl, trainUrl string, par
if len(tmpCodeObsPaths) > 0 {
lastCodeDir = tmpCodeObsPaths[len(tmpCodeObsPaths)-1]
}
userCommand = "/bin/bash /home/work/run_train.sh 's3://" + appUrl + "' '" + lastCodeDir + "/" + bootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + dataUrl + "' --'train_url'='s3://" + trainUrl + "'"
var multi_data_url string
var pretrain_url string
for _, param := range params.Parameter {
if param.Label == "multi_data_url" {
multi_data_url = string(param.Value)
}
if param.Label == "pretrain_url" {
pretrain_url = string(param.Value)
}
}
//配置环境变量,适配c2net的sdk用于训练脚本获取代码,数据集,模型等
var envCodeCommand = "export CODE_URL=" + "s3://" + appUrl + ";" + "export LOCAL_CODE_PATH=/cache/code;"
var envDataCommand = "export DATASET_URL=" + "'" + multi_data_url + "'" + ";" + "export LOCAL_DATASET_PATH=/cache/dataset;"
var envPretrainCommand = "export PRETRAIN_MODEL_URL=" + "'" + pretrain_url + "'" + ";" + "export LOCAL_PRETRAIN_MODEL_PATH=/cache/pretrainmodel;"
var envOutputCommand = "export OUTPUT_URL=" + "s3://" + trainUrl + ";" + "export LOCAL_OUTPUT_PATH=/cache/output;"
var envMoxingCommand = "export DATA_DOWNLOAD_METHOD=MOXING;"
var envNeedUnzipCommand = "export CODE_NEED_UNZIP=false;export DATASET_NEED_UNZIP=false;export PRETRAIN_MODEL_NEED_UNZIP=false;"
var envCommand = envCodeCommand + envDataCommand + envPretrainCommand + envOutputCommand + envMoxingCommand + envNeedUnzipCommand
userCommand = envCommand + "/bin/bash /home/work/run_train.sh 's3://" + appUrl + "' '" + lastCodeDir + "/" + bootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + dataUrl + "' --'train_url'='s3://" + trainUrl + "'"
for _, param := range params.Parameter {
userCommand += " --'" + param.Label + "'='" + param.Value + "'"
}
@@ -538,12 +581,12 @@ func getCloudbrainTwoMultiDataUrl(datasets []entity.ContainerData) string {
return string(jsondata)
}

func getCloudbrainTwoModelUrl(datasets []entity.ContainerData) string {
if len(datasets) == 0 {
func getCloudbrainTwoModelUrl(pretrainModels []entity.ContainerData) string {
if len(pretrainModels) == 0 {
return ""
}
var modelUrlList []models.ModelUrls
for _, d := range datasets {
for _, d := range pretrainModels {
modelUrlList = append(modelUrlList, models.ModelUrls{
ModelUrl: d.S3DownloadUrl,
ModelName: d.Name,
@@ -725,7 +768,7 @@ func getModelartsTrainJob(jobID string, versionID int64, baseLine string, order

func (c CloudbrainTwoClusterAdapter) GetLogDownloadInfo(opts entity.ClusterLogDownloadInfoOpts) (*entity.FileDownloadInfo, error) {
var err error
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)

//查找日志文件
files := getLogFilesInStorage(helper, opts.ObjectKeyPrefix, ".log")
@@ -779,7 +822,7 @@ func (c CloudbrainTwoClusterAdapter) GetLogDownloadInfo(opts entity.ClusterLogDo
}

func (c CloudbrainTwoClusterAdapter) GetSingleOutputDownloadInfo(opts entity.ClusterSingleOutputDownloadInfoOpts) (*entity.FileDownloadInfo, error) {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
url, err := helper.GetSignedDownloadUrl(opts.Path)
if err != nil {
log.Error("GetSignedDownloadUrl err.opts=%+v,err =%v", opts, err)
@@ -849,7 +892,7 @@ func (c CloudbrainTwoClusterAdapter) GetNodeInfo(opts entity.ClusterNodeInfoOpts
}

func (c CloudbrainTwoClusterAdapter) GetOutput(opts entity.ClusterOutputOpts) (*entity.ClusterAITaskOutput, error) {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
fileList, err := helper.GetOneLevelObjectsUnderDir(path.Join(opts.ObjectKeyPrefix, opts.ParentDir))
if err != nil {
log.Error("GetOneLevelObjectsUnderDir err.objectKeyPrefix=%s,err=%v", opts.ObjectKeyPrefix, err)
@@ -863,7 +906,7 @@ func (c CloudbrainTwoClusterAdapter) GetOutput(opts entity.ClusterOutputOpts) (*
}

func (c CloudbrainTwoClusterAdapter) GetAllOutput(opts entity.ClusterOutputOpts) (*entity.AllAITaskOutput, error) {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
fileList, err := helper.GetAllObjectsUnderDir(path.Join(opts.ObjectKeyPrefix, opts.ParentDir))
if err != nil {
log.Error("GetAllObjectsUnderDir err.objectKeyPrefix=%s,err=%v", opts.ObjectKeyPrefix, err)


+ 1
- 1
services/ai_task_service/cluster/common.go View File

@@ -97,7 +97,7 @@ func getLogFilesInStorage(helper storage_helper.StorageHelper, objectKeyPrefix s
}

func DownloadAllOutput(opts entity.DownloadOutputOpts) error {
helper := storage_helper.SelectUploaderFromStorageType(opts.StorageType)
helper := storage_helper.SelectStorageHelperFromStorageType(opts.StorageType)
var err error
fileList, err := helper.GetAllObjectsUnderDir(opts.Path)
if err != nil {


+ 2
- 1
services/ai_task_service/container_builder/code_builder.go View File

@@ -45,7 +45,7 @@ func (b *CodeBuilder) Build(ctx *context.CreationContext) ([]entity.ContainerDat
jobName := ctx.Request.JobName
repo := ctx.Repository
codeLocalPath := setting.JobPath + jobName + cloudbrain.CodeMountPath + "/"
uploader := storage_helper.SelectUploaderFromStorageType(storageTypes[0])
uploader := storage_helper.SelectStorageHelperFromStorageType(storageTypes[0])

remoteDir := uploader.GetJobDefaultObjectKeyPrefix(jobName) + opts.GetLocalPath()
//再次调试和在线运行notebook不需要下载、上传代码
@@ -93,6 +93,7 @@ func (b *CodeBuilder) Build(ctx *context.CreationContext) ([]entity.ContainerDat
RealPath: uploader.GetRealPath(objectKey),
IsDir: b.Opts.Uncompressed,
S3DownloadUrl: uploader.GetS3DownloadUrl(objectKey),
IsNeedUnzip: true,
StorageType: storageTypes[0],
}
return []entity.ContainerData{codeData}, nil


+ 6
- 6
services/ai_task_service/container_builder/common.go View File

@@ -2,16 +2,17 @@ package container_builder

import (
"bufio"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/services/ai_task_service/context"
"code.gitea.io/gitea/services/ai_task_service/storage_helper"
"errors"
"io"
"io/ioutil"
"os"
"strings"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/services/ai_task_service/context"
"code.gitea.io/gitea/services/ai_task_service/storage_helper"
)

func DownloadCode(ctx *context.CreationContext, codeLocalPath string, uncompressed bool) error {
@@ -23,7 +24,6 @@ func DownloadCode(ctx *context.CreationContext, codeLocalPath string, uncompress
}
}
var commitId string

//目录为空时需要下载代码
if len(dir) == 0 {
if uncompressed {


+ 8
- 5
services/ai_task_service/container_builder/dataset_builder.go View File

@@ -49,7 +49,7 @@ func (b *DatasetBuilder) Build(ctx *context.CreationContext) ([]entity.Container
log.Info("mount dataset directory.")
jobName := ctx.Request.JobName
storageTypes := b.Opts.AcceptStorageType
uploader := storage_helper.SelectUploaderFromStorageType(storageTypes[0])
uploader := storage_helper.SelectStorageHelperFromStorageType(storageTypes[0])
remoteDir := path.Join(uploader.GetJobDefaultObjectKeyPrefix(jobName), b.Opts.GetLocalPath())
err := uploader.MKDIR(remoteDir)
if err != nil {
@@ -72,6 +72,7 @@ func (b *DatasetBuilder) Build(ctx *context.CreationContext) ([]entity.Container
GetBackEndpoint: uploader.GetEndpoint(),
IsDir: true,
StorageType: storageTypes[0],
IsNeedUnzip: true,
})
}

@@ -89,10 +90,10 @@ func (b *DatasetBuilder) Build(ctx *context.CreationContext) ([]entity.Container
}
//由于云脑一训练任务单数据集情况比较特殊,挂载时没有数据集名字的父文件夹,因此特殊处理
//todo AITask 解决此特殊处理
if ctx.Request.Cluster == entity.OpenICloudbrainOne &&
ctx.Request.JobType == models.JobTypeTrain && len(datasetInfos) == 1 {
name = ""
}
// if ctx.Request.Cluster == entity.OpenICloudbrainOne &&
// ctx.Request.JobType == models.JobTypeTrain && len(datasetInfos) == 1 {
// name = ""
// }
if datasetInfo.Type == models.TypeCloudBrainOne {
data = append(data, entity.ContainerData{
Name: name,
@@ -105,6 +106,7 @@ func (b *DatasetBuilder) Build(ctx *context.CreationContext) ([]entity.Container
IsDir: b.Opts.Uncompressed,
Size: datasetInfo.Size,
StorageType: entity.MINIO,
IsNeedUnzip: true,
})

} else {
@@ -119,6 +121,7 @@ func (b *DatasetBuilder) Build(ctx *context.CreationContext) ([]entity.Container
IsDir: b.Opts.Uncompressed,
Size: datasetInfo.Size,
StorageType: entity.OBS,
IsNeedUnzip: true,
})
}
}


+ 1
- 1
services/ai_task_service/container_builder/log_path_builder.go View File

@@ -33,7 +33,7 @@ func (b *LogPathBuilder) Build(ctx *context.CreationContext) ([]entity.Container

jobName := ctx.Request.JobName

uploader := storage_helper.SelectUploaderFromStorageType(storageTypes[0])
uploader := storage_helper.SelectStorageHelperFromStorageType(storageTypes[0])
remoteDir := path.Join(uploader.GetJobDefaultObjectKeyPrefix(jobName), b.Opts.GetLocalPath())
if b.Opts.MKDIR {
err := uploader.MKDIR(remoteDir)


+ 1
- 1
services/ai_task_service/container_builder/output_path_builder.go View File

@@ -35,7 +35,7 @@ func (b *OutputPathBuilder) Build(ctx *context.CreationContext) ([]entity.Contai

jobName := ctx.Request.JobName

uploader := storage_helper.SelectUploaderFromStorageType(storageTypes[0])
uploader := storage_helper.SelectStorageHelperFromStorageType(storageTypes[0])
remoteDir := path.Join(uploader.GetJobDefaultObjectKeyPrefix(jobName), b.Opts.GetLocalPath())
if b.Opts.MKDIR {
err := uploader.MKDIR(remoteDir)


+ 62
- 102
services/ai_task_service/container_builder/pre_model_builder.go View File

@@ -1,20 +1,18 @@
package container_builder

import (
"code.gitea.io/gitea/routers/response"
"fmt"
"path"
"strings"

"code.gitea.io/gitea/services/ai_model"

"code.gitea.io/gitea/routers/response"

"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/services/ai_task_service/context"
"code.gitea.io/gitea/services/ai_task_service/storage_helper"
"code.gitea.io/gitea/services/cloudbrain/cloudbrainTask"
)

type PretrainModelBuilder struct {
@@ -40,126 +38,88 @@ func (b *PretrainModelBuilder) Build(ctx *context.CreationContext) ([]entity.Con
return nil, response.SYSTEM_ERROR
}
//未选择预训练模型,跳过此步
if form.PretrainModelName == "" {
return nil, nil
}
if form.PretrainModelId == "" {
//异常数据,理论上应该都有modelId
return nil, response.RESULT_CLEARD
return nil, nil
}
//查出模型数据
m, err := models.QueryModelById(form.PretrainModelId)
if err != nil {
uuids := strings.Split(form.PretrainModelId, ";")
modelInfoMaps, err := models.QueryModelMapsByIds(uuids)
if err != nil || len(modelInfoMaps) == 0 {
log.Error("Can not find model", err)
return nil, response.MODEL_NOT_EXISTS
}
preTrainModelUrl := m.Path
if err != nil {
log.Error("Can not find model", err)
return nil, response.MODEL_NOT_EXISTS
var preTrainModelEntity []entity.ContainerData
for _, m := range modelInfoMaps {
ai_model.InitModelMeta(m.ID)
data, err := b.buildModelData(m, form.JobName)
if err != nil {
return nil, response.SYSTEM_ERROR
}
preTrainModelEntity = append(preTrainModelEntity, data)
}
//模型文件存储方式
return preTrainModelEntity, nil
}

func (b *PretrainModelBuilder) GetContainerType() entity.ContainerDataType {
return entity.ContainerPreTrainModel
}

const MODEL_MKDIR_README = "The model files have already been loaded into the container and are ready for use.\n"

func (b *PretrainModelBuilder) buildModelData(m *models.AiModelManage, jobName string) (entity.ContainerData, *response.BizError) {
oldStorageType := entity.GetStorageTypeFromCloudbrainType(m.Type)
if oldStorageType == "" {
log.Error("model storage type error.modelId=%d", m.ID)
return nil, response.SYSTEM_ERROR
return entity.ContainerData{}, response.SYSTEM_ERROR
}
oldStorageHelper := storage_helper.SelectStorageHelperFromStorageType(oldStorageType)

var preTrainModelPath string
var preTrainModelEntity []entity.ContainerData
preTrainModelPath := getPreTrainModelPath(m.Path)
storageType := oldStorageType
ckptNames := strings.Split(form.PretrainModelCkptName, ";")
for _, ckptName := range ckptNames {
isExists, size := cloudbrainTask.CheckAndGetFileSize(m, ckptName)
if !isExists {
log.Error("model file not exist.name = %s", ckptName)
return nil, response.MODEL_NOT_EXISTS
}
preTrainModelPath = getPreTrainModelPath(preTrainModelUrl, ckptName)
if !b.Opts.IsStorageTypeIn(oldStorageType) {
//意味着模型之前存储的位置不符合要求,需要转存到指定存储
newStorageType := b.Opts.AcceptStorageType[0]
//todo 可优化
if newStorageType == entity.MINIO && oldStorageType == entity.OBS {
//复用以前代码
minioPreModelURL, err := dealModelInfo(form.PretrainModelId, form.JobName, ckptName)
if err != nil {
log.Error("Can not find model,modelId=%d err=%v", form.PretrainModelId, err)
return nil, response.MODEL_NOT_EXISTS
}
preTrainModelUrl = minioPreModelURL
preTrainModelPath = getPreTrainModelPath(minioPreModelURL, ckptName)
storageType = entity.MINIO
if !b.Opts.IsStorageTypeIn(oldStorageType) {
//意味着模型之前存储的位置不符合要求,需要转存到指定存储
newStorageType := b.Opts.AcceptStorageType[0]
newStorageHelper := storage_helper.SelectStorageHelperFromStorageType(newStorageType)
files, err := oldStorageHelper.GetAllObjectsUnderDir(preTrainModelPath)
newObjectPrefix := path.Join(newStorageHelper.GetJobDefaultObjectKeyPrefix(jobName), b.Opts.GetLocalPath(), m.Name)
for _, file := range files {
newFilePath := path.Join(newObjectPrefix, file.FileName)
err = storage_helper.TransferFileBetweenStorage(oldStorageHelper, newStorageHelper, file.RelativePath, newFilePath)
if err != nil {
log.Error("transfer file between storage error.model=%+v file=%+v err=%v", m, file, err)
return entity.ContainerData{}, response.SYSTEM_ERROR
}
}
uploader := storage_helper.SelectUploaderFromStorageType(storageType)
modelData := entity.ContainerData{
Name: ckptName,
Bucket: uploader.GetBucket(),
EndPoint: uploader.GetEndpoint(),
ObjectKey: preTrainModelPath,
ReadOnly: b.Opts.ReadOnly,
ContainerPath: path.Join(b.Opts.ContainerPath, ckptName),
RealPath: uploader.GetRealPath(preTrainModelPath),
S3DownloadUrl: uploader.GetS3DownloadUrl(preTrainModelPath),
IsDir: false,
Size: size,
IsOverwrite: true,
}
preTrainModelEntity = append(preTrainModelEntity, modelData)
preTrainModelPath = newObjectPrefix
storageType = newStorageType
}
form.PreTrainModelUrl = preTrainModelUrl
return preTrainModelEntity, nil
}

func (b *PretrainModelBuilder) GetContainerType() entity.ContainerDataType {
return entity.ContainerPreTrainModel
uploader := storage_helper.SelectStorageHelperFromStorageType(storageType)
uploader.MKDIR(preTrainModelPath, MODEL_MKDIR_README)
modelData := entity.ContainerData{
Name: m.Name,
Bucket: uploader.GetBucket(),
EndPoint: uploader.GetEndpoint(),
ObjectKey: preTrainModelPath,
ReadOnly: b.Opts.ReadOnly,
ContainerPath: path.Join(b.Opts.ContainerPath, m.Name),
RealPath: uploader.GetRealPath(preTrainModelPath),
S3DownloadUrl: uploader.GetS3DownloadUrl(preTrainModelPath),
IsDir: true,
Size: m.Size,
IsOverwrite: true,
IsNeedUnzip: false,
}
return modelData, nil
}

func getPreTrainModelPath(pretrainModelDir string, fileName string) string {
func getPreTrainModelPath(pretrainModelDir string) string {
index := strings.Index(pretrainModelDir, "/")
if index > 0 {
filterBucket := pretrainModelDir[index+1:]
return filterBucket + fileName
return filterBucket
} else {
return ""
}

}

func dealModelInfo(modelId string, jobName string, ckptName string) (string, error) {
preModel, err := models.QueryModelById(modelId)
if err != nil || preModel == nil || preModel.ID == "" {
log.Error("Can not find model", err)
return "", fmt.Errorf("Can not find model: %v", ckptName)
}
minioPreModelURL, err := downloadModelFromObs(preModel, jobName, cloudbrain.PretrainModelMountPath, ckptName)
if err != nil {
log.Error("Can not find model", err)

return "", err
}
return minioPreModelURL, nil
}

func downloadModelFromObs(preModel *models.AiModelManage, jobName, suffixPath string, ckptFileName string) (string, error) {
destPath := setting.CBCodePathPrefix + jobName + suffixPath + "/"
destFile := destPath + ckptFileName
returnStr := setting.Attachment.Minio.Bucket + "/" + destPath
srcUrl := preModel.Path[len(setting.Bucket)+1:] + ckptFileName
log.Info("dest model Path=" + returnStr + " src path=" + preModel.Path + ckptFileName)
body, err := storage.ObsDownloadAFile(setting.Bucket, srcUrl)
if err == nil {
defer body.Close()
_, err = storage.Attachments.UploadContent(setting.Attachment.Minio.Bucket, destFile, body)
if err != nil {
log.Error("UploadObject(%s) failed: %s", preModel.Path+ckptFileName, err.Error())
return "", err
}
} else {
log.Info("download model failed. as " + err.Error())
return "", err
}
log.Info("download model from obs succeed")
return returnStr, nil
}

+ 4
- 2
services/ai_task_service/context/context.go View File

@@ -1,10 +1,11 @@
package context

import (
"encoding/json"

"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"encoding/json"
)

type CreationContext struct {
@@ -13,13 +14,13 @@ type CreationContext struct {
GitRepo *git.Repository
Repository *models.Repository
Spec *models.Specification
Queues []models.ResourceQueue
User *models.User
CommitID string
Response *entity.CreationResponse
SourceCloudbrain *models.Cloudbrain
NewCloudbrain *models.Cloudbrain
Config *entity.AITaskBaseConfig
Queues []models.ResourceQueue
}

func (ctx *CreationContext) AddContainerData(t entity.ContainerDataType, d []entity.ContainerData) {
@@ -63,6 +64,7 @@ func (ctx *CreationContext) BuildCloudbrainConfig() *models.CloudbrainConfig {
output := ctx.GetContainerData(entity.ContainerOutPutPath)
log := ctx.GetContainerData(entity.ContainerLogPath)
c := &models.CloudbrainConfig{

ConfigurationSnapshot: aiConfigStr,
OutputBucket: output.Bucket,
OutputObjectPrefix: output.ObjectKey,


+ 12
- 2
services/ai_task_service/storage_helper/client.go View File

@@ -14,11 +14,12 @@ type UploaderConfig struct {

type StorageHelper interface {
UploadDir(codePath, jobName string) error
UploadFile(objectKey string, r io.Reader) error
GetRealPath(objectKey string) string
GetBucket() string
GetEndpoint() string
GetJobDefaultObjectKeyPrefix(jobName string) string
MKDIR(path string) error
MKDIR(path string, description ...string) error
GetOneLevelObjectsUnderDir(dirPath string, maxKeyArray ...int) ([]storage.FileInfo, error)
GetAllObjectsUnderDir(prefix string, maxKeyArray ...int) ([]storage.FileInfo, error)
OpenFile(path string) (io.ReadCloser, error)
@@ -27,7 +28,7 @@ type StorageHelper interface {
CopyByPath(sourcePath, targetPath string, filterSuffix []string) error
}

func SelectUploaderFromStorageType(storageType entity.StorageType) StorageHelper {
func SelectStorageHelperFromStorageType(storageType entity.StorageType) StorageHelper {
switch storageType {
case entity.OBS:
return &OBSHelper{}
@@ -46,3 +47,12 @@ func isMatchSuffix(fileName string, filterSuffix []string) bool {
return false

}

func TransferFileBetweenStorage(old, new StorageHelper, oldFilePath, newFilePath string) error {
body, err := old.OpenFile(oldFilePath)
if err != nil {
return err
}
defer body.Close()
return new.UploadFile(newFilePath, body)
}

+ 13
- 1
services/ai_task_service/storage_helper/minio.go View File

@@ -21,6 +21,15 @@ type MinioHelper struct {
func (m *MinioHelper) UploadDir(codePath, objectKeyPrefix string) error {
return UploadDirToMinio(codePath, objectKeyPrefix, "")
}

func (m *MinioHelper) UploadFile(objectKey string, r io.Reader) error {
_, err := storage.Attachments.UploadContent(m.GetBucket(), objectKey, r)
if err != nil {
return err
}
return nil
}

func (m *MinioHelper) GetJobDefaultObjectKeyPrefix(jobName string) string {
return path.Join(setting.CBCodePathPrefix, jobName)
}
@@ -38,10 +47,13 @@ func (m *MinioHelper) GetEndpoint() string {

const README = "README"

func (m *MinioHelper) MKDIR(path string) error {
func (m *MinioHelper) MKDIR(path string, description ...string) error {
//无法直接创建空文件夹,上传一个readme文件模拟
path = strings.TrimSuffix(path, "/") + "/" + README
val := "read me."
if description != nil && len(description) > 0 {
val = description[0]
}
_, err := storage.Attachments.UploadContent(m.GetBucket(), path, bytes.NewReader([]byte(val)))
return err
}


+ 13
- 1
services/ai_task_service/storage_helper/obs.go View File

@@ -19,6 +19,18 @@ func (m *OBSHelper) UploadDir(codePath, objectKeyPrefix string) error {
return UploadDirToObs(codePath, objectKeyPrefix, "")
}

func (m *OBSHelper) UploadFile(objectKey string, r io.Reader) error {
input := &obs.PutObjectInput{}
input.Bucket = m.GetBucket()
input.Key = objectKey
input.Body = r
_, err := storage.ObsCli.PutObject(input)
if err != nil {
return err
}
return nil
}

func (m *OBSHelper) GetJobDefaultObjectKeyPrefix(jobName string) string {
return path.Join(setting.CodePathPrefix, jobName)
}
@@ -30,7 +42,7 @@ func (m *OBSHelper) GetRealPath(objectKey string) string {
func (m *OBSHelper) GetBucket() string {
return setting.Bucket
}
func (m *OBSHelper) MKDIR(path string) error {
func (m *OBSHelper) MKDIR(path string, description ...string) error {
path = strings.TrimSuffix(path, "/") + "/"
input := &obs.PutObjectInput{}
input.Bucket = setting.Bucket


+ 67
- 5
services/ai_task_service/storage_helper/repo.go View File

@@ -1,7 +1,15 @@
package storage_helper

import (
"archive/zip"
"bufio"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/grampus"
@@ -9,12 +17,7 @@ import (
"code.gitea.io/gitea/modules/obs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"fmt"
"github.com/unknwon/com"
"io"
"os"
"path"
"strings"
)

func DownloadZipCode(gitRepo *git.Repository, codePath, branchName string) (commitId string, err error) {
@@ -199,5 +202,64 @@ func DownloadCode(gitRepo *git.Repository, repo *models.Repository, codePath, br
pos += int64(len(line))
}
commitID, _ := gitRepo.GetBranchCommitID(branchName)

return commitID, nil
}

func createZipArchive(sourceDir, archivePath string) error {
archiveFile, err := os.Create(archivePath)
if err != nil {
return err
}
defer archiveFile.Close()

zipWriter := zip.NewWriter(archiveFile)
defer zipWriter.Close()

err = filepath.Walk(sourceDir, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}

relPath, err := filepath.Rel(sourceDir, filePath)
if err != nil {
return err
}

// 排除压缩包本身
if strings.TrimSuffix(relPath, "/") == filepath.Base(archivePath) {
return nil
}

if info.IsDir() {
// 创建目录项
zipEntry, err := zipWriter.Create(filepath.ToSlash(relPath) + "/") // 目录项以斜杠结尾
if err != nil {
return err
}
_ = zipEntry // 忽略目录项

return nil
}

zipEntry, err := zipWriter.Create(filepath.ToSlash(relPath)) // 使用正斜杠分隔符
if err != nil {
return err
}

file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()

_, err = io.Copy(zipEntry, file)
if err != nil {
return err
}

return nil
})

return err
}

+ 5
- 1
services/ai_task_service/task/cloudbrain_one_notebook_task.go View File

@@ -34,6 +34,8 @@ func GetCloudbrainOneNotebookConfig(opts entity.AITaskConfigKey) *entity.AITaskB
IsActionUseJobId: false,
DatasetsLimitSizeGB: setting.DebugAttachSize,
DatasetsMaxNum: setting.MaxDatasetNum,
ModelLimitSizeGB: setting.DEBUG_MODEL_SIZE_LIMIT_GB,
ModelMaxNum: setting.DEBUG_MODEL_NUM_LIMIT,
ContainerSteps: map[entity.ContainerDataType]*entity.ContainerBuildOpts{
entity.ContainerCode: {
ContainerPath: "/code",
@@ -88,6 +90,7 @@ func (t CloudbrainOneNotebookTaskTemplate) Create(ctx *context.CreationContext)
Next(t.LoadSpec).
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckModels).
Next(t.CheckBranchExists).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
@@ -104,8 +107,9 @@ func (t CloudbrainOneNotebookTaskTemplate) Restart(ctx *context.CreationContext)
c := &CreateOperator{}
err := c.Next(t.BuildRequest4Restart).
Next(t.CheckSourceTaskIsCleared).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.CheckDatasets).
Next(t.CheckModels).
Next(t.CheckParamFormat).
Next(t.CheckMultiRequest).
Next(t.LoadSpec).


+ 1
- 1
services/ai_task_service/task/cloudbrain_one_train_task.go View File

@@ -80,7 +80,7 @@ func (t CloudbrainOneTrainTaskTemplate) Create(ctx *context.CreationContext) (*e
Next(t.LoadSpec).
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)


+ 1
- 1
services/ai_task_service/task/cloudbrain_two_inference_task.go View File

@@ -81,7 +81,7 @@ func (t CloudbrainTwoInferenceTaskTemplate) Create(ctx *context.CreationContext)
Next(t.LoadSpec).
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)


+ 13
- 5
services/ai_task_service/task/cloudbrain_two_notebook_task.go View File

@@ -1,6 +1,8 @@
package task

import (
"strings"

"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/convert"
@@ -11,7 +13,6 @@ import (
"code.gitea.io/gitea/routers/response"
"code.gitea.io/gitea/services/ai_task_service/context"
"code.gitea.io/gitea/services/cloudbrain/resource"
"strings"
)

type CloudbrainTwoNotebookTaskTemplate struct {
@@ -36,17 +37,19 @@ func GetCloudbrainTwoNotebookConfig(opts entity.AITaskConfigKey) *entity.AITaskB
IsActionUseJobId: false,
DatasetsLimitSizeGB: setting.DebugAttachSize,
DatasetsMaxNum: setting.MaxDatasetNum,
ModelLimitSizeGB: setting.DEBUG_MODEL_SIZE_LIMIT_GB,
ModelMaxNum: setting.DEBUG_MODEL_NUM_LIMIT,
ContainerSteps: map[entity.ContainerDataType]*entity.ContainerBuildOpts{
entity.ContainerCode: {
Disable: true,
Disable: false,
AcceptStorageType: []entity.StorageType{entity.OBS},
},
entity.ContainerDataset: {
Disable: true,
Disable: false,
AcceptStorageType: []entity.StorageType{entity.OBS},
},
entity.ContainerPreTrainModel: {
Disable: true,
Disable: false,
AcceptStorageType: []entity.StorageType{entity.OBS},
},
},
@@ -88,7 +91,7 @@ func (t CloudbrainTwoNotebookTaskTemplate) Restart(ctx *context.CreationContext)
c := &CreateOperator{}
err := c.Next(t.BuildRequest4Restart).
Next(t.CheckSourceTaskIsCleared).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.CheckDatasets).
Next(t.CheckParamFormat).
Next(t.CheckMultiRequest).
@@ -111,6 +114,7 @@ func (t CloudbrainTwoNotebookTaskTemplate) Restart(ctx *context.CreationContext)
}

func (g CloudbrainTwoNotebookTaskTemplate) CallCreationAPI(ctx *context.CreationContext) *response.BizError {
log.Info("ctx is here:", ctx)
c := g.GetMyCluster()
if c == nil {
return response.SYSTEM_ERROR
@@ -127,6 +131,10 @@ func (g CloudbrainTwoNotebookTaskTemplate) CallCreationAPI(ctx *context.Creation
ImageUrl: strings.TrimSpace(form.ImageUrl),
AutoStopDuration: autoStopDurationMs,
Spec: ctx.Spec,
Datasets: ctx.GetContainerDataArray(entity.ContainerDataset),
Code: ctx.GetContainerDataArray(entity.ContainerCode),
PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel),
OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath),
},
},
}


+ 1
- 1
services/ai_task_service/task/cloudbrain_two_train_task.go View File

@@ -81,7 +81,7 @@ func (t CloudbrainTwoTrainTaskTemplate) Create(ctx *context.CreationContext) (*e
Next(t.LoadSpec).
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)


+ 1
- 1
services/ai_task_service/task/grampus_inference_task.go View File

@@ -85,7 +85,7 @@ func (t GrampusInferenceTaskTemplate) Create(ctx *context.CreationContext) (*ent
Next(t.LoadSpec).
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)


+ 13
- 9
services/ai_task_service/task/grampus_notebook_task.go View File

@@ -29,9 +29,9 @@ func init() {
}

func GetGrampusNoteBookConfig(opts entity.AITaskConfigKey) *entity.AITaskBaseConfig {
codePath := "/code"
datasetPath := "/dataset"
pretrainModelPath := "/pretrainmodel"
codePath := "/tmp/code"
datasetPath := "/tmp/dataset"
pretrainModelPath := "/tmp/pretrainmodel"

config := &entity.AITaskBaseConfig{
ContainerSteps: map[entity.ContainerDataType]*entity.ContainerBuildOpts{
@@ -101,17 +101,17 @@ func GetGrampusNoteBookConfig(opts entity.AITaskConfigKey) *entity.AITaskBaseCon
config = &entity.AITaskBaseConfig{
ContainerSteps: map[entity.ContainerDataType]*entity.ContainerBuildOpts{
entity.ContainerCode: {
ContainerPath: "/tmp" + codePath,
ContainerPath: codePath,
ReadOnly: false,
AcceptStorageType: []entity.StorageType{entity.MINIO, entity.OBS},
},
entity.ContainerDataset: {
ContainerPath: "/tmp" + datasetPath,
ContainerPath: datasetPath,
ReadOnly: true,
AcceptStorageType: []entity.StorageType{entity.MINIO, entity.OBS},
},
entity.ContainerPreTrainModel: {
ContainerPath: "/tmp" + pretrainModelPath,
ContainerPath: pretrainModelPath,
ReadOnly: true,
AcceptStorageType: []entity.StorageType{entity.MINIO, entity.OBS},
},
@@ -161,6 +161,8 @@ func GetGrampusNoteBookConfig(opts entity.AITaskConfigKey) *entity.AITaskBaseCon
config.IsActionUseJobId = false
config.DatasetsLimitSizeGB = setting.DebugAttachSize
config.DatasetsMaxNum = setting.MaxDatasetNum
config.ModelLimitSizeGB = setting.DEBUG_MODEL_SIZE_LIMIT_GB
config.ModelMaxNum = setting.DEBUG_MODEL_NUM_LIMIT
return config
}

@@ -174,7 +176,7 @@ func (t GrampusNoteBookTaskTemplate) Create(ctx *context.CreationContext) (*enti
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckBranchExists).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)
@@ -189,7 +191,7 @@ func (t GrampusNoteBookTaskTemplate) Restart(ctx *context.CreationContext) (*ent
c := &CreateOperator{}
err := c.Next(t.BuildRequest4Restart).
Next(t.CheckSourceTaskIsCleared).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.CheckDatasets).
Next(t.CheckParamFormat).
Next(t.CheckMultiRequest).
@@ -231,8 +233,10 @@ func (g GrampusNoteBookTaskTemplate) CallCreationAPI(ctx *context.CreationContex
ResourceSpecId: ctx.Spec.SourceSpecId,
ImageId: form.ImageID,
ImageUrl: imageUrl,
Datasets: append(ctx.GetContainerDataArray(entity.ContainerDataset), ctx.GetContainerDataArray(entity.ContainerPreTrainModel)...),
Datasets: ctx.GetContainerDataArray(entity.ContainerDataset),
PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel),
Code: ctx.GetContainerDataArray(entity.ContainerCode),
EnvVariables: models.GrampusEnvVarReq{},
AutoStopDuration: autoStopDurationMs,
Capacity: setting.Capacity,
Queues: ctx.Queues,


+ 18
- 6
services/ai_task_service/task/grampus_online_infer_task.go View File

@@ -30,16 +30,18 @@ func init() {
}

func GetGrampusOnlineInferConfig(opts entity.AITaskConfigKey) *entity.AITaskBaseConfig {
codePath := "/code"
datasetPath := "/dataset"
pretrainModelPath := "/pretrainmodel"
outputPath := "/output"
codePath := "/tmp/code"
datasetPath := "/tmp/dataset"
pretrainModelPath := "/tmp/pretrainmodel"
outputPath := "/tmp/output"

config := &entity.AITaskBaseConfig{
ActionType: models.ActionCreateGrampusGPUOnlineInferTask,
IsActionUseJobId: false,
DatasetsLimitSizeGB: setting.DebugAttachSize,
DatasetsMaxNum: setting.MaxDatasetNum,
ModelLimitSizeGB: setting.DEBUG_MODEL_SIZE_LIMIT_GB,
ModelMaxNum: setting.DEBUG_MODEL_NUM_LIMIT,
ContainerSteps: map[entity.ContainerDataType]*entity.ContainerBuildOpts{
entity.ContainerCode: {
ContainerPath: codePath,
@@ -78,7 +80,7 @@ func (t GrampusOnlineInferTaskTemplate) Create(ctx *context.CreationContext) (*e
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckBranchExists).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)
@@ -116,7 +118,8 @@ func (g GrampusOnlineInferTaskTemplate) CallCreationAPI(ctx *context.CreationCon
ResourceSpecId: ctx.Spec.SourceSpecId,
ImageId: form.ImageID,
ImageUrl: imageUrl,
Datasets: append(ctx.GetContainerDataArray(entity.ContainerDataset), ctx.GetContainerDataArray(entity.ContainerPreTrainModel)...),
Datasets: append(ctx.GetContainerDataArray(entity.ContainerDataset)),
PreTrainModel: append(ctx.GetContainerDataArray(entity.ContainerPreTrainModel)),
Code: ctx.GetContainerDataArray(entity.ContainerCode),
OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath),
AutoStopDuration: -1,
@@ -160,3 +163,12 @@ func (g GrampusOnlineInferTaskTemplate) LoadSpec(ctx *context.CreationContext) *
ctx.Spec = spec
return nil
}

func (GrampusOnlineInferTaskTemplate) GetAvailableQueues(ctx *context.CreationContext) *response.BizError {
ctx.Queues = ctx.Spec.GetAvailableQueues(models.GetAvailableCenterIdOpts{
UserId: ctx.User.ID,
JobType: models.JobTypeDebug,
HasInternet: ctx.Request.HasInternet,
})
return nil
}

+ 3
- 2
services/ai_task_service/task/grampus_train_task.go View File

@@ -1,6 +1,8 @@
package task

import (
"strings"

"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain"
@@ -9,7 +11,6 @@ import (
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/routers/response"
"code.gitea.io/gitea/services/ai_task_service/context"
"strings"
)

type GrampusTrainTaskTemplate struct {
@@ -114,7 +115,7 @@ func (t GrampusTrainTaskTemplate) Create(ctx *context.CreationContext) (*entity.
Next(t.LoadSpec).
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.GetAvailableQueues, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)


+ 49
- 31
services/ai_task_service/task/opt_handler.go View File

@@ -30,7 +30,6 @@ type CreationHandler interface {
CheckPointBalance(ctx *context.CreationContext) *response.BizError
CheckDatasets(ctx *context.CreationContext) *response.BizError
CheckBranchExists(ctx *context.CreationContext) *response.BizError
CheckModel(ctx *context.CreationContext) *response.BizError
CheckBootFile(ctx *context.CreationContext) *response.BizError
CheckSourceTaskIsCleared(ctx *context.CreationContext) *response.BizError
BuildContainerData(ctx *context.CreationContext) *response.BizError
@@ -46,7 +45,7 @@ type CreationHandler interface {
GetAvailableQueues(ctx *context.CreationContext) *response.BizError
}

//DefaultCreationHandler CreationHandler的默认实现,公共逻辑可以在此结构体中实现
// DefaultCreationHandler CreationHandler的默认实现,公共逻辑可以在此结构体中实现
type DefaultCreationHandler struct {
}

@@ -91,13 +90,10 @@ func (DefaultCreationHandler) BuildRequest4Restart(ctx *context.CreationContext)
Cluster: entity.GetClusterTypeFromCloudbrainType(task.Type),
WorkServerNumber: task.WorkServerNumber,
BranchName: task.BranchName,
PreTrainModelUrl: task.PreTrainModelUrl,
PretrainModelCkptName: task.CkptName,
ImageUrl: imageUrl,
ImageID: imageId,
ImageName: imageName,
PretrainModelName: task.ModelName,
PretrainModelVersion: task.ModelVersion,
Description: task.Description,
LabelName: task.LabelName,
DatasetUUIDStr: task.Uuid,
@@ -159,6 +155,51 @@ func (g DefaultCreationHandler) CheckDatasets(ctx *context.CreationContext) *res
return nil
}

func (g DefaultCreationHandler) CheckModels(ctx *context.CreationContext) *response.BizError {
log.Info("Start to CheckModels.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
modelIdStr := ctx.Request.PretrainModelId
if modelIdStr == "" {
return nil
}
//check model num
uuids := strings.Split(modelIdStr, ";")
if ctx.Config.ModelMaxNum > 0 && len(uuids) > ctx.Config.ModelMaxNum {
log.Error("the dataset count(%d) exceed the limit", len(uuids))
return response.MODEL_NUMBER_OVER_LIMIT
}

modelInfoMaps, err := models.QueryModelMapsByIds(uuids)
if err != nil {
log.Error("QueryModelsByIds failed: %v", err)
return response.SYSTEM_ERROR
}

if len(modelInfoMaps) < len(uuids) {
log.Info("CheckModels has model deleted.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
return response.MODEL_NOT_EXISTS
}

//check datasets size
var attachSize int64
for _, infos := range modelInfoMaps {
attachSize += infos.Size
}
limitSizeGB := ctx.Config.ModelLimitSizeGB
if limitSizeGB > 0 && attachSize > int64(limitSizeGB*1000*1000*1000) {
log.Error("The model size exceeds the limit (%dGB)", limitSizeGB) // GB
return response.MODEL_SIZE_OVER_LIMIT.WithParams(limitSizeGB)
}

var modelNames string
for i := 0; i < len(uuids); i++ {
m := modelInfoMaps[uuids[i]]
modelNames += m.Name + ";"
}
ctx.Request.ModelNames = strings.TrimSuffix(modelNames, ";")
log.Info("CheckModels success.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
return nil
}

func (DefaultCreationHandler) CheckBranchExists(ctx *context.CreationContext) *response.BizError {
log.Info("Start to CheckBranchExists.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
if ctx.GitRepo == nil || ctx.Request.BranchName == "" {
@@ -171,20 +212,6 @@ func (DefaultCreationHandler) CheckBranchExists(ctx *context.CreationContext) *r
return nil
}

func (DefaultCreationHandler) CheckModel(ctx *context.CreationContext) *response.BizError {
log.Info("Start to CheckModel.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
if hasModelNumOverLimit(ctx.Request.PretrainModelCkptName) { //检查模型数量是否超出限制
log.Info("CheckModel hasModelNumOverLimit displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
return response.MODEL_NUM_OVER_LIMIT
}
if hasModelFileDeleted(ctx.Request.PretrainModelId, ctx.Request.PretrainModelCkptName) { //检查模型文件是否存在
log.Info("CheckModel hasModelFileDeleted.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
return response.MODEL_NOT_EXISTS
}
log.Info("CheckModel success.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
return nil
}

func (DefaultCreationHandler) CheckSourceTaskIsCleared(ctx *context.CreationContext) *response.BizError {
log.Info("Start to CheckSourceTaskIsCleared.displayJobName=%s jobType=%s cluster=%s", ctx.Request.DisplayJobName, ctx.Request.JobType, ctx.Request.Cluster)
task := ctx.SourceCloudbrain
@@ -413,12 +440,9 @@ func (DefaultCreationHandler) InsertCloudbrainRecord4Async(ctx *context.Creation
WorkServerNumber: req.WorkServerNumber,
EngineName: imageUrl,
Spec: ctx.Spec,
ModelName: req.PretrainModelName,
ModelVersion: req.PretrainModelVersion,
ModelName: req.ModelNames,
LabelName: req.LabelName,
PreTrainModelUrl: req.PreTrainModelUrl,
ModelId: req.PretrainModelId,
CkptName: req.PretrainModelCkptName,
SubTaskName: models.SubTaskName,
CreatedUnix: timeutil.TimeStampNow(),
UpdatedUnix: timeutil.TimeStampNow(),
@@ -478,12 +502,9 @@ func (DefaultCreationHandler) AfterCallCreationAPI4Sync(ctx *context.CreationCon
WorkServerNumber: req.WorkServerNumber,
EngineName: imageUrl,
Spec: ctx.Spec,
ModelName: req.PretrainModelName,
ModelVersion: req.PretrainModelVersion,
ModelName: req.ModelNames,
LabelName: req.LabelName,
PreTrainModelUrl: req.PreTrainModelUrl,
ModelId: req.PretrainModelId,
CkptName: req.PretrainModelCkptName,
SubTaskName: models.SubTaskName,
JobID: res.JobID,
Status: TransAITaskStatus(res.Status),
@@ -590,11 +611,8 @@ func (DefaultCreationHandler) CreateCloudbrainRecord4Restart(ctx *context.Creati
CreatedUnix: res.CreateTime,
UpdatedUnix: res.CreateTime,
Spec: ctx.Spec,
ModelName: req.PretrainModelName,
ModelVersion: req.PretrainModelVersion,
ModelName: req.ModelNames,
LabelName: req.LabelName,
PreTrainModelUrl: req.PreTrainModelUrl,
CkptName: req.PretrainModelCkptName,
SubTaskName: models.SubTaskName,
ModelId: req.PretrainModelId,
GpuQueue: ctx.Spec.QueueCode,


+ 138
- 0
services/ai_task_service/task/sdk_util.go View File

@@ -0,0 +1,138 @@
package task

import (
"fmt"
"math/rand"
"regexp"
"strings"
"unicode"

"code.gitea.io/gitea/models"
)

func GenerateSDKCode(datasetNames, pretrainModelNames, parameterKeys []string, jobType models.JobType) string {
return generateAITaskSDKCode(datasetNames, pretrainModelNames, parameterKeys, jobType)
}

func generateAITaskSDKCode(datasetNames, pretrainModelNames, parameterKeys []string, jobType models.JobType) string {
var code string
if len(parameterKeys) > 0 {
//添加参数解析代码
code = code + "import argparse\n\nparser = argparse.ArgumentParser(description='忽略超参数不存在的报错问题')\n#添加自定义参数\n"
for i := 0; i < len(parameterKeys); i++ {
code = code + fmt.Sprintf("parser.add_argument(\"--%s\")\n", parameterKeys[i])
}
code = code + "args, unknown = parser.parse_known_args()\n\n"
}

//判断是否需要执行prepare方法
shouldPrepareCtx := true
if len(datasetNames) == 0 && len(pretrainModelNames) == 0 {
shouldPrepareCtx = false
}
//判断是否需要执行回传方法
shouldAddUploadCode := shouldAddUpload(jobType)
if !shouldPrepareCtx && !shouldAddUploadCode {
return code
}
// code = code + "import os\nos.system(\"pip install openi\")\n"

//添加import相关代码
if !shouldAddUploadCode {
code = code + "from c2net.context import prepare\n\n"
} else if !shouldPrepareCtx {
code = code + "from c2net.context import upload_output\n\n"
} else {
code = code + "from c2net.context import prepare,upload_output\n\n"
}

//添加prepare()方法
if shouldPrepareCtx {
code = code + "#初始化导入数据集和预训练模型到容器内\nc2net_context = prepare()\n"
}

//添加数据集相关代码
var datasetNameMap = make(map[string]string, 0)
for i := 0; i < len(datasetNames); i++ {
if i == 0 {
code = code + "\n#获取数据集路径\n"
}
datasetName := strings.TrimSuffix(datasetNames[i], ".zip")
datasetName = strings.TrimSuffix(datasetName, ".tar.gz")
validName := makeValidPythonVariableName(datasetName)
for {
if _, exists := datasetNameMap[datasetName]; exists {
datasetName = datasetName + "_" + generateRandomString(3)
} else {
break
}
}
datasetNameMap[datasetName] = ""
pathCode := fmt.Sprintf("%s_path = c2net_context.dataset_path+\"/\"+\"%s\"\n", validName, datasetName)
code = code + pathCode
}

//添加预训练模型相关代码
var pretrainModelNameMap = make(map[string]string, 0)
for i := 0; i < len(pretrainModelNames); i++ {
if i == 0 {
code = code + "\n#获取预训练模型路径\n"
}
modelName := makeValidPythonVariableName(pretrainModelNames[i])
for {
if _, exists := pretrainModelNameMap[modelName]; exists {
modelName = modelName + "_" + generateRandomString(3)
} else {
break
}
}
pretrainModelNameMap[modelName] = ""
pathCode := fmt.Sprintf("%s_path = c2net_context.pretrain_model_path+\"/\"+\"%s\"\n", modelName, pretrainModelNames[i])
code = code + pathCode
}

//添加回传相关提示代码
code = code + "\n#输出结果必须保存在该目录\nyou_should_save_here = c2net_context.output_path\n\n"
//添加回传函数
if shouldAddUploadCode {
code = code + "\n#回传结果到openi,只有训练任务才能回传\nupload_output()\n"
}
return code
}

// 判断是否需要添加回传代码
func shouldAddUpload(jobType models.JobType) bool {
if jobType == models.JobTypeTrain {
return true
}
return false
}

func makeValidPythonVariableName(input string) string {
validName := strings.ToLower(input)
// 使用正则表达式将非字母数字下划线的字符替换为下划线
re := regexp.MustCompile(`[^a-zA-Z0-9_]`)
validName = re.ReplaceAllString(validName, "_")
re = regexp.MustCompile(`_+`)
validName = re.ReplaceAllString(validName, "_")
// 如果变量名以数字开头,添加一个下划线
if len(validName) > 0 && unicode.IsDigit(rune(validName[0])) {
validName = "_" + validName
}
if validName == "" || validName == "_" {
validName = "pretrain_model_" + generateRandomString(3)
}
return validName
}

// generateRandomString 生成指定长度的随机字母字符串
func generateRandomString(length int) string {
const letters = "abcdefghijklmnopqrstuvwxyz"

result := make([]byte, length)
for i := range result {
result[i] = letters[rand.Intn(len(letters))]
}

return string(result)
}

+ 1
- 1
services/ai_task_service/task/super_compute_task.go View File

@@ -55,7 +55,7 @@ func (t SuperComputeTaskTemplate) Create(ctx *context.CreationContext) (*entity.
Next(t.CheckPointBalance).
Next(t.CheckDatasets).
Next(t.CheckBranchExists).
Next(t.CheckModel).
Next(t.CheckModels).
Next(t.InsertCloudbrainRecord4Async).
AsyncNextWithErrFun(t.BuildContainerData, t.CallCreationAPI, t.AfterCallCreationAPI4Async, t.NotifyCreation, t.HandleErr4Async).
Operate(ctx)


+ 1
- 1
services/ai_task_service/task/task_config.go View File

@@ -41,7 +41,7 @@ func GetContainerStorageObjectPrefix(c *entity.AITaskBaseConfig, jobName string,
if st == nil && len(st) == 0 {
return ""
}
uploader := storage_helper.SelectUploaderFromStorageType(st[0])
uploader := storage_helper.SelectStorageHelperFromStorageType(st[0])
//兼容历史任务所以加上了versionName,另外云脑二训练任务为了适配modelarts接口加上了默认版本,此时要剔除
localPath := config.GetLocalPath()
localPath = strings.TrimSuffix(localPath, models.CloudbrainTwoDefaultVersion)


+ 6
- 1
services/ai_task_service/task/task_creation_info.go View File

@@ -86,8 +86,13 @@ func GetAITaskCreationInfo(req entity.GetAITaskCreationInfoReq) (*entity.Creatio
result.CanUseAllImages = canUseAll
}

c := t.GetConfig(entity.AITaskConfigKey{ComputeSource: req.ComputeSource.GetCloudbrainFormat()})
result.Config = entity.AITaskCreationConfig{
DatasetMaxSize: setting.DebugAttachSize * 1024 * 1024 * 1024,
//DatasetMaxSize: setting.DebugAttachSize * 1000 * 1000 * 1000,
DatasetMaxSize: c.DatasetsLimitSizeGB * 1024 * 1024 * 1024,
DatasetsMaxNum: c.DatasetsMaxNum,
ModelMaxSize: c.ModelLimitSizeGB * 1024 * 1024 * 1024,
ModelMaxNum: c.ModelMaxNum,
}
//查询可用节点数
if workerNums, err := t.GetAllowedWorkerNum(req.User.ID, req.ComputeSource); err == nil {


+ 25
- 45
services/ai_task_service/task/task_extend.go View File

@@ -11,50 +11,41 @@ import (
"strings"
)

func GetModelDownload(task *models.Cloudbrain) []*models.ModelDownload {
var repositoryLink string
pretrainModelList := []*models.ModelDownload{}
ckptNames := strings.Split(task.CkptName, ";")
func GetModelDownload(task *models.Cloudbrain) []*models.Model4Show {
var pretrainModelList []*models.Model4Show
var model *models.AiModelManage
var err error
if task.ModelId == "" {
model, err = models.QueryModelByPath(task.PreTrainModelUrl)
} else {
model, err = models.QueryModelById(task.ModelId)
}
if err != nil || model == nil {
return pretrainModelList
return []*models.Model4Show{}
}

if r, err := models.QueryModelRepoByModelID(model.ID); err == nil {
repositoryLink = r.Link()
}
for _, ckptName := range ckptNames {
var url string
if task.Type == models.TypeC2Net {
url = getModelContainerLink(task.DataUrl, ckptName)
} else {
url = getModelLocalLink(model, ckptName)
for _, modelId := range task.GetModelIdArray() {
model, err = models.QueryModelById(modelId)
if err != nil || model == nil {
pretrainModelList = append(pretrainModelList, &models.Model4Show{
IsDelete: true,
Name: model.Name,
})
continue
}
modelDownload := models.ModelDownload{
Name: ckptName,
DownloadLink: url,
IsDelete: false,
ModelName: model.Name,
var repositoryLink string
if r, err := models.QueryModelRepoByModelID(modelId); err == nil {
repositoryLink = r.Link()
}
if hasModelFileDeleted(task.ModelId, ckptName) {
log.Warn("Can not get model by path:" + url)
modelDownload.IsDelete = true
}
modelDownload.RepositoryLink = repositoryLink
pretrainModelList = append(pretrainModelList, &modelDownload)
pretrainModelList = append(pretrainModelList, &models.Model4Show{
ID: modelId,
IsDelete: false,
Name: model.Name,
RepositoryLink: repositoryLink,
Size: model.Size,
})
}
return pretrainModelList
}

func getModelLocalLink(model *models.AiModelManage, ckptName string) string {
func getModelLocalLink(model *models.AiModelManage) string {
index := strings.Index(model.Path, "/")
key := model.Path[index+1:] + ckptName
key := model.Path[index+1:]
url, _ := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key)
return url
}
@@ -70,17 +61,6 @@ func GetCloudBrainDataSetInfo(task *models.Cloudbrain) []*models.DatasetDownload
datasetObsUrlList := make([]entity.NotebookDataset, 0)
_ = json.Unmarshal([]byte(task.DataUrl), &datasetObsUrlList)

for _, datasetInfo := range datasetDownload {
datasetInfo.DatasetDownloadLink = ""
for _, datasetObs := range datasetObsUrlList {
log.Info("datasetObsUrl:" + datasetObs.DatasetUrl + "datasetName:" + datasetInfo.DatasetName)
if strings.Contains(datasetObs.DatasetUrl, datasetInfo.DatasetName) {
datasetInfo.DatasetDownloadLink = datasetObs.DatasetUrl
break
}
}

}
return datasetDownload
}

@@ -198,14 +178,14 @@ func correctAITaskSpec(task *models.Cloudbrain) {
}
}

func getModelContainerLink(dataUrl string, ckptName string) string {
func getModelContainerLink(dataUrl string, modelName string) string {
if dataUrl == "" {
return ""
}
datasetObsUrlList := make([]entity.NotebookDataset, 0)
_ = json.Unmarshal([]byte(dataUrl), &datasetObsUrlList)
for _, datasetObs := range datasetObsUrlList {
if strings.Contains(datasetObs.DatasetUrl, ckptName) {
if datasetObs.DatasetName == modelName {
return datasetObs.DatasetUrl
}
}


+ 73
- 54
services/ai_task_service/task/task_service.go View File

@@ -1,15 +1,6 @@
package task

import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strconv"
"strings"

"code.gitea.io/gitea/entity"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/convert"
@@ -25,6 +16,14 @@ import (
"code.gitea.io/gitea/services/cloudbrain/cloudbrainTask"
"code.gitea.io/gitea/services/cloudbrain/resource"
"code.gitea.io/gitea/services/lock"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strconv"
"strings"
)

type QueryFunc func(opts entity.JobIdAndVersionId) (*entity.QueryTaskResponse, error)
@@ -68,13 +67,34 @@ func buildAITaskInfo(task *models.Cloudbrain, creator *models.User, config *enti
return nil, err
}
datasets := []*models.DatasetDownload{}
pretrainModelList := []*models.ModelDownload{}
pretrainModelList := []*models.Model4Show{}
var datasetNames []string
if task.Uuid != "" {
datasets = GetCloudBrainDataSetInfo(task)
for i := 0; i < len(datasets); i++ {
datasetNames = append(datasetNames, datasets[i].DatasetName)
}
}
if task.ModelName != "" {
var pretrainModelNames []string
if task.ModelId != "" {
pretrainModelList = GetModelDownload(task)
for i := 0; i < len(pretrainModelList); i++ {
pretrainModelNames = append(pretrainModelNames, pretrainModelList[i].Name)
}
}

paramKeys := make([]string, 0)
if task.Parameters != "" {
params := parseAITaskParameters(task.Parameters)
if params != nil {
for i := 0; i < len(params.Parameter); i++ {
paramKeys = append(paramKeys, params.Parameter[i].Label)
}
}

}
code := GenerateSDKCode(datasetNames, pretrainModelNames, paramKeys, models.JobType(task.JobType))

n := 1
if task.WorkServerNumber > 1 {
n = task.WorkServerNumber
@@ -97,49 +117,48 @@ func buildAITaskInfo(task *models.Cloudbrain, creator *models.User, config *enti
baseConfig = config.BaseConfig
}
return &entity.AITaskDetailInfo{
ID: task.ID,
JobID: task.JobID,
Status: task.Status,
DetailedStatus: task.DetailedStatus,
JobType: task.JobType,
DisplayJobName: task.DisplayJobName,
FormattedDuration: task.TrainJobDuration,
ComputeSource: task.GetStandardComputeSource(),
PreVersionName: task.PreVersionName,
CurrentVersionName: task.VersionName,
WorkServerNumber: n,
Spec: convert.ToSpecification(spec),
DatasetList: datasets,
PretrainModelList: pretrainModelList,
AICenter: task.AiCenter,
BootFile: task.BootFile,
Cluster: string(entity.GetClusterTypeFromCloudbrainType(task.Type)),
Parameters: parseAITaskParameters(task.Parameters),
CreatedUnix: task.CreatedUnix,
CodePath: baseConfig.GetContainerPath(entity.ContainerCode),
DatasetPath: baseConfig.GetContainerPath(entity.ContainerDataset),
PretrainModelPath: baseConfig.GetContainerPath(entity.ContainerPreTrainModel),
OutputPath: baseConfig.GetContainerPath(entity.ContainerOutPutPath),
CodeUrl: task.RemoteCodeUrl,
PretrainModelName: task.ModelName,
PretrainModelVersion: task.ModelVersion,
PretrainCkptName: task.CkptName,
PretrainModelUrl: task.PreTrainModelUrl,
PretrainModelId: task.ModelId,
StartTime: task.StartTime,
EndTime: task.EndTime,
Description: task.Description,
FailedReason: task.FailedReason,
CommitID: task.CommitID,
BranchName: task.BranchName,
ImageName: imageName,
ImageID: imageId,
ImageUrl: imageUrl,
CreatorName: creator.GetDisplayName(),
EngineName: task.EngineName,
UserId: task.UserID,
AppName: task.AppName,
HasInternet: task.HasInternet,
ID: task.ID,
JobID: task.JobID,
Status: task.Status,
DetailedStatus: task.DetailedStatus,
JobType: task.JobType,
DisplayJobName: task.DisplayJobName,
FormattedDuration: task.TrainJobDuration,
ComputeSource: task.GetStandardComputeSource(),
PreVersionName: task.PreVersionName,
CurrentVersionName: task.VersionName,
WorkServerNumber: n,
Spec: convert.ToSpecification(spec),
DatasetList: datasets,
PretrainModelList: pretrainModelList,
SDKCode: code,
AICenter: task.AiCenter,
BootFile: task.BootFile,
Cluster: string(entity.GetClusterTypeFromCloudbrainType(task.Type)),
Parameters: parseAITaskParameters(task.Parameters),
CreatedUnix: task.CreatedUnix,
CodePath: baseConfig.GetContainerPath(entity.ContainerCode),
DatasetPath: baseConfig.GetContainerPath(entity.ContainerDataset),
PretrainModelPath: baseConfig.GetContainerPath(entity.ContainerPreTrainModel),
OutputPath: baseConfig.GetContainerPath(entity.ContainerOutPutPath),
CodeUrl: task.RemoteCodeUrl,
PretrainModelName: task.ModelName,
PretrainModelUrl: task.PreTrainModelUrl,
PretrainModelId: task.ModelId,
StartTime: task.StartTime,
EndTime: task.EndTime,
Description: task.Description,
FailedReason: task.FailedReason,
CommitID: task.CommitID,
BranchName: task.BranchName,
ImageName: imageName,
ImageID: imageId,
ImageUrl: imageUrl,
CreatorName: creator.GetDisplayName(),
EngineName: task.EngineName,
UserId: task.UserID,
AppName: task.AppName,
HasInternet: task.HasInternet,
}, nil
}



+ 1
- 1
templates/admin/cloudbrain/imagecommit.tmpl View File

@@ -67,7 +67,7 @@
</div>
<div class="inline required field">
<label class="label_color" for="">{{$.i18n.Tr "repo.images"}}</label>
<input type="text" name="place" required placeholder="{{$.i18n.Tr "cloudbrain.input_mirror"}}" style="width: 80%;" maxlength="100">
<input type="text" name="place" required placeholder="{{$.i18n.Tr "cloudbrain.input_mirror"}}" style="width: 80%;" maxlength="300">
</div>
<div class="inline required field">


+ 12
- 7
templates/repo/datasets/index.tmpl View File

@@ -145,6 +145,11 @@
</style>
<div class="repository">
{{template "repo/header" .}}
<div class="ui container" style="position:relative;">
<div style="display:inline-block;position:absolute;right:{{if.dataset}}0{{else}}120px{{end}};top:10px;z-index:2">
<div id="__dataset-code-dialog"></div>
</div>
</div>
{{if .dataset}}
<div id="dataset-range-value" data-num-stars="{{.dataset.NumStars}}" data-star-active="{{$.IsStaringDataset}}"
style="display: none;">
@@ -283,11 +288,9 @@
<div class="ui grid stackable item" id="{{.UUID}}">
<div class="row">
<!-- 数据集名称 -->

<div class="four wide column" style="width: 24% !important;display: flex;align-items: center;">
<div class="four wide column" style="width: 24% !important;display: flex;align-items: center;visibility:hidden">
<el-tooltip class="item" effect="dark" placement="top" popper-class="diy-popper">
<div slot="content"><span class="wrap">

{{if ne .DecompressState -1}}{{$.i18n.Tr "dataset.unzip_status"}}:{{if eq .DecompressState 1}}{{$.i18n.Tr "dataset.unzip_successed"}}{{else if eq .DecompressState 0 2}}{{$.i18n.Tr "dataset.unzip_stared"}}{{else}}{{$.i18n.Tr "dataset.unzip_failed"}}{{end}}
&nbsp;&nbsp;{{end}}<i
class="ri-download-line"></i>{{$.i18n.Tr "dataset.download"}}:{{.DownloadCount}}
@@ -338,14 +341,14 @@
{{.CreatedUnix | TimeSinceUnix1}}
</div>
<div class="four wide column text right">
<div class="ui compact buttons">

<div class="ui compact buttons" style="display:none">
<a class="ui basic blue button" href="{{.DownloadURL}}">{{$.i18n.Tr "dataset.download"}}</a>

{{if eq .DecompressState 1}}
<a class="ui basic blue button" href="datasets/dirs/{{.UUID}}?type={{$.Type}}"
data-tooltip='{{$.i18n.Tr "dataset.directory"}}'>{{$.i18n.Tr "preview"}}</a>
{{end}}
<span class="ui basic blue button" @click="copyUseCode('{{.Name}}')">
{{$.i18n.Tr "dataset.copy_code"}}</span>
{{if and (.CanDel) (not $.Repository.IsPrivate)}}
<span class="ui basic blue button" style="color: #13c28d !important;"
@click="setPrivate('{{.UUID}}',false,{{$k}})"
@@ -436,4 +439,6 @@
</div>
{{template "base/delete_modal_actions" .}}
</div>
{{template "base/footer" .}}
{{template "base/footer" .}}
<link rel="stylesheet" href="{{StaticUrlPrefix}}/css/vp-dataset-code-dialog.css?v={{MD5 AppVer}}" />
<script src="{{StaticUrlPrefix}}/js/vp-dataset-code-dialog.js?v={{MD5 AppVer}}"></script>

+ 0
- 1
templates/repo/debugjob/index.tmpl View File

@@ -1,6 +1,5 @@
{{template "base/head" .}}
<link rel="stylesheet" href="{{StaticUrlPrefix}}/css/vp-cloudbrain-list.css?v={{MD5 AppVer}}" />
<script>const Tasks = {{.Tasks}}; </script>
<div class="repository release dataset-list view">
{{template "repo/header" .}}
<div id="__vue-root"></div>


+ 4
- 0
templates/repo/home.tmpl View File

@@ -390,6 +390,10 @@
{{end}}
</div>
</div>
<div style="padding-top:16px;clear:both;">
<a class="use-dlg-btn" style="text-decoration:underline;"
href="https://openi.pcl.ac.cn/docs/index.html#/cloudbrain/codepath">{{.i18n.Tr "repo.code_use_resource"}}</a>
</div>
</div>
{{end}}
</div>


+ 1
- 1
templates/user/dashboard/dashboard.tmpl View File

@@ -101,7 +101,7 @@
.ui.placeholder.segment {
min-height: 15rem !important;
}
.line{
.bgtask-none.line{
border-top: 1px solid rgba(187, 187, 187, 0.5) !important;
margin-top: 20px !important;
}

+ 3
- 3
web_src/js/features/clipboard.js View File

@@ -1,9 +1,9 @@
export default async function initClipboard() {
const els = document.querySelectorAll(".clipboard");
export default async function initClipboard(elements) {
const els = elements || document.querySelectorAll(".clipboard");
if (!els || !els.length) return;

const { default: ClipboardJS } = await import(
/* webpackChunkName: "clipboard" */ "clipboard"
/* webpackChunkName: "clipboardjs" */ "clipboard"
);

const clipboard = new ClipboardJS(els);


+ 198
- 0
web_src/js/features/globalModalDlg.js View File

@@ -0,0 +1,198 @@
import SparkMD5 from "spark-md5";
import initClipboard from "./clipboard.js";
import highlight from "./highlight.js";

; (function () {
const WAIT_COUNT = 10;
const exceptPages = ['/home/term'];
const csrf = window.config.csrf;
const mdConfigReg = /^\---\n((.|\s)*?)\n---\n/;
let userID = '';
let curStorageKey = '';
const lang = document.querySelector('html').getAttribute('lang');

function init() {
userID = $('meta[name="_uid"]').attr('content');
if (!userID) { // 未登录,不显示
return;
}
curStorageKey = `g-models-${userID}`;
var pathName = window.location.pathname;
if (exceptPages.indexOf(pathName) > -1) return; // 排除页,不显示
$.ajax({
type: "GET",
url: "/dashboard/invitation",
dataType: "json",
data: { filename: `tips/global/change${lang == 'zh-CN' ? '' : '_en'}.md` },
success: function (res) {
try {
res && renderMdStr(res);
} catch (err) {
console.log(err);
}
},
error: function (err) {
console.log(err);
}
});
delete window.globalModalInit;
}

function renderMdStr(str) {
if (!str) return;
const hash = SparkMD5.hash(str);
const configs = parseMdConfigs(str);
str = str.replace(mdConfigReg, '');
const dataInfoStr = window.localStorage.getItem(curStorageKey) || '{}';
let dataInfoObj = JSON.parse(dataInfoStr);
var showOr = dataInfoObj[hash] === false ? false : true;
dataInfoObj[hash] = showOr;
configs.hash = hash;
if (!showOr) return;
$.ajax({
type: "POST",
url: `/api/v1/markdown?_csrf=${csrf}`,
data: {
mode: 'gfm',
text: str,
},
success: function (res) {
try {
if (res) {
createDialog(res, configs);
window.config.HighlightJS = true;
$('.__g-modal pre code').each((function () {
highlight(this);
}));
setTimeout(() => {
initClipboard('.__g-modal .clipboard');
$('.__g-modal .clipboard').popup('hide');
}, 300);
}
} catch (err) {
console.log(err);
}
},
error: function (err) {
console.log(err);
}
});
}

function sparkMD5Hash(str = '') {
return SparkMD5.hash(str) + Math.random().toString().replace('0.', '');
}

function parseMdConfigs(mdStr) {
const obj = {};
const regex = mdConfigReg;
const res = mdStr.match(regex);
if (res && res[1]) {
const str = res[1];
const rows = str.split('\n');
for (let i = 0, iLen = rows.length; i < iLen; i++) {
const row = rows[i];
const col = row.split(':');
if (col.length > 1) {
obj[col[0].trim()] = col[1].trim();
}
}
}
return obj
}

function insertCodeCopyBtn(htmlStr) {
const html = $(htmlStr);
const codeBlocks = html.find('.code-block');
for (let i = 0, iLen = codeBlocks.length; i < iLen; i++) {
const codeBlockI = codeBlocks.eq(i);
const txt = codeBlockI.text();
const copyBtn = $(`<div class="copy-btn"><a
href="javascript:;" class="ui poping up clipboard" id="clipboard-${sparkMD5Hash(txt)}"
data-position="top center" data-variation="inverted tiny" data-success="${window.i18n.cloudeBrainMirror.copy_succeeded}"
data-content="${window.i18n.cloudeBrainMirror.copy}"
data-original="${window.i18n.cloudeBrainMirror.copy}"
data-clipboard-text=""><i style="font-size:14px;" class="copy outline icon"></i></a></div>`);
copyBtn.find('a').attr('data-clipboard-text', txt);
codeBlockI[0].outerHTML = `<div class="code-content">${codeBlockI[0].outerHTML}${copyBtn[0].outerHTML}</div>`;
}
return html.html();
}

function createDialog(html, configs) {
const dataInfoStr = window.localStorage.getItem(curStorageKey) || '{}';
let dataInfoObj = JSON.parse(dataInfoStr);
const hash = configs.hash;
function startCount(modelEl, count) {
var timer = setInterval(function () {
count--;
modelEl.data('count', count);
if (count <= 0) {
modelEl.find('.button.positive').removeClass('disabled');
modelEl.find('.count-down-c').hide();
clearInterval(timer);
} else {
modelEl.find('.count-down-c .count-down').text(count);
}
}, 1000);
modelEl.data('timer', timer);
}
const renderHtml = insertCodeCopyBtn(html);
var showOr = dataInfoObj[hash] === false ? false : true;
dataInfoObj[hash] = showOr;
if (!showOr) return;
var el = $(`
<div class="ui longer large modal __g-modal" _id="g-modal-${hash}">
<div class="header" style="line-height:0.7em;font-size:1.2em;">${configs.title || window.i18n.warmPrompt}</div>
<style>
.__g-modal .code-content {position:relative;}
.__g-modal .code-block {position:relative;}
.__g-modal .copy-btn {position:absolute;right:4px;top:4px;}
</style>
<div class="content scrolling markdown">
${renderHtml}
</div>
<div class="actions" style="text-align:center;">
<div style="padding:0.78571429em 0.78571429em 0.78571429em;height:36px;display: inline-block;">
<div class="ui checkbox" >
<input type="checkbox" name="notRemindAgain">
<label>${window.i18n.notRemind}</label>
</div>
</div>
<div class="ui positive button ${WAIT_COUNT ? 'disabled' : ''}">
${window.i18n.close}${WAIT_COUNT ? `<span class="count-down-c"> (<span class="count-down">${WAIT_COUNT}</span>S)</span>` : ''}
</div>
</div>
</div>`);
$('body').append(el);
var modelEl = $(`.ui.modal[_id="g-modal-${hash}"]`);
modelEl.modal({
closable: false,
onDeny: function () { return false; },
onApprove: function (trigger) {
var modelEl = $(trigger).closest('.__g-modal');
var count = modelEl.data('count');
var indexNum = modelEl.data('index-num');
var notRemindAgain = modelEl.find('input[name="notRemindAgain"]').prop('checked');
if (Number(count) <= 0) {
if (notRemindAgain) {
const dataInfoStr = window.localStorage.getItem(curStorageKey) || '{}';
let dataInfoObj = JSON.parse(dataInfoStr);
dataInfoObj[indexNum] = false;
window.localStorage.setItem(curStorageKey, JSON.stringify(dataInfoObj));
}
}
}
}).modal('show');
modelEl.data('index-num', hash);
startCount(modelEl, WAIT_COUNT);
window.localStorage.setItem(curStorageKey, JSON.stringify(dataInfoObj));
}

window.globalModalInit = init;
setTimeout(function () {
if ($('.modal.network-security').hasClass('scale') || $('.modal.network-security').hasClass('active'))
return;
init();
}, 0);
})();

+ 1
- 1
web_src/js/features/highlight.js View File

@@ -3,7 +3,7 @@ export default async function highlight(elementOrNodeList) {
const nodes = 'length' in elementOrNodeList ? elementOrNodeList : [elementOrNodeList];
if (!nodes.length) return;

const {default: Worker} = await import(/* webpackChunkName: "highlight" */'./highlight.worker.js');
const {default: Worker} = await import(/* webpackChunkName: "highlight.worker" */'./highlight.worker.js');
const worker = new Worker();

worker.addEventListener('message', ({data}) => {


+ 10
- 0
web_src/js/features/i18nVue.js View File

@@ -144,6 +144,9 @@ export const i18nVue = {
scrolled_logs_bottom: '您已翻阅至日志底部',
scrolled_logs_bottom_pls_retry: '您已翻阅至日志底部,请稍后再试!',
computeNode: '计算节点',
notRemind: '不再提醒',
close: '关闭',
warmPrompt: '温馨提示',

cloudeBrainMirror: {
cloud_brain_mirror: '云脑镜像',
@@ -170,7 +173,9 @@ export const i18nVue = {
delete: '删除',
my_favorite_mirror: '我收藏的镜像',
more:'更多',
copy: '复制',
copy_succeeded: '复制成功!',
copy_failed: '复制失败!',
cancel_recommendation: '取消推荐',
set_as_recommended: '设为推荐',
create_cloud_brain_mirror: '创建云脑镜像',
@@ -369,6 +374,9 @@ export const i18nVue = {
scrolled_logs_bottom: 'You have scrolled to the bottom of the log',
scrolled_logs_bottom_pls_retry: 'You have scrolled to the bottom of the log, please try again later!',
computeNode: 'Compute Node',
notRemind: 'Don\'t remind again',
close: 'Close',
warmPrompt: 'Tips',

cloudeBrainMirror: {
cloud_brain_mirror: 'Cloud Brain Mirror',
@@ -395,7 +403,9 @@ export const i18nVue = {
delete: 'Delete',
my_favorite_mirror: 'My Favorite Mirror',
more:'More',
copy: 'Copy',
copy_succeeded: 'Copy succeeded!',
copy_failed: 'Copy failed',
cancel_recommendation: 'Cancel recommendation',
set_as_recommended: 'Set as recommended',
create_cloud_brain_mirror: 'Create cloud brain mirror',


+ 40
- 6
web_src/js/index.js View File

@@ -58,6 +58,7 @@ import { Message } from "element-ui";

import { i18nVue } from "./features/i18nVue.js";
import './features/ad.js';
import './features/globalModalDlg.js';
import { Fancybox } from "./vendor/fancybox.esm.js";
import "../../public/iconfonts1/iconfont";

@@ -4030,23 +4031,55 @@ function initVueDataset() {
this.descfile = dataset_file_desc;
this.repolink = repolink;
this.datasetType = datasetType;
$('.dataset .four.wide.column').css('visibility', 'visible');
$('.dataset .ui.compact.buttons').show();
},
methods: {
copyUrl(url) {
const cInput = document.createElement("input");
const cInput = document.createElement("textarea");
cInput.value = url;
document.body.appendChild(cInput);
cInput.select();
document.execCommand("Copy");
cInput.remove();
$("body").toast({
message: "复制成功!",
message: i18n['cloudeBrainMirror']['copy_succeeded'],
showProgress: "bottom",
showIcon: "check circle",
class: "info",
position: "top right",
});
},
copyUseCode(name) {
const self = this;
$.ajax({
url: `/api/v1/ai_task/generate_sdk_code?dataset_name=${name}&_csrf=${csrf}`,
type: 'get',
data: { },
success: function(res) {
if (res.code == 0 && res.data && res.data.code) {
self.copyUrl(res.data.code);
} else {
$("body").toast({
message: i18n['cloudeBrainMirror']['copy_failed'],
showProgress: "bottom",
showIcon: "times circle circle",
class: "error",
position: "top right",
});
}
},
error: function(err) {
$("body").toast({
message: i18n['cloudeBrainMirror']['copy_failed'],
showProgress: "bottom",
showIcon: "times circle circle",
class: "error",
position: "top right",
});
}
})
},
handleCurrentChange(val) {
this.page = val;
switch (this.activeName) {
@@ -5406,10 +5439,10 @@ function initAddUsageAgreement() {
centered: false,
onShow: function () {
$('.ui.dimmer').css({ "background-color": "rgb(136, 136, 136,0.7)" })
$('.network-security .approve').addClass('disabled').text(`${updateTips} (${cutDown}s)`)
$('.network-security .approve').addClass('disabled').text(`${confirmTips} (${cutDown}s)`)
var timer = setInterval(function () {
cutDown--
$('.network-security .approve').addClass('disabled').text(`${updateTips} (${cutDown}s)`)
$('.network-security .approve').addClass('disabled').text(`${confirmTips} (${cutDown}s)`)
if (cutDown <= 0) {
$('.network-security .approve').removeClass('disabled').text(confirmTips)
clearInterval(timer);
@@ -5422,6 +5455,7 @@ function initAddUsageAgreement() {
try {
const res = await postSaveProtocolInfo(userName)
if (res?.data?.result_code === "0") {
window.globalModalInit && window.globalModalInit()
return true
} else {
return false
@@ -5441,8 +5475,8 @@ function initAddUsageAgreement() {
let params = {userName:userName}
return axios.post(`${AppSubUrl}/user/saveOtherInfo`,qs.stringify(params))
}
if ($('meta[name="_uid"]').length) {
if ($('meta[name="_uid"]').length && ['/home/term'].indexOf(window.location.pathname) < 0) {
showLoginProtocolDialog($('meta[name="_uid"]').attr('content-ext'))
}
}
initAddUsageAgreement()
initAddUsageAgreement()

+ 44
- 0
web_src/vuepages/apis/modules/common.js View File

@@ -11,6 +11,19 @@ export const getPromoteData = (filePathName) => {
});
}

// 获取markdown渲染结果
export const getMarkdownHtml = (str, mode) => {
return service({
url: '/api/v1/markdown',
method: 'post',
parmas: {},
data: {
mode: mode || 'gfm',
text: str,
},
});
}

// 获取个人积分信息
// return {pointAccount:{id,account_code,balance,total_earned,total_consumed,status,version,created_unix,updated_unix },cloudBrainPaySwitch}
export const getPointAccountInfo = () => {
@@ -21,3 +34,34 @@ export const getPointAccountInfo = () => {
});
}

// 由模型名称列表或数据集名称列表或运行参数列表获取SDK的代码使用方式
// dataset_name-string[], datasetNameList
// pretrain_model_name -string[] modelNameList
// param_key -string[] param_key
// job_type,compute_source,cluster_type
export const getSDKCode = (params) => {
const datasetNames = params.dataset_name || [];
const modelNames = params.pretrain_model_name || [];
const parameterKeys = params.param_key || [];
const searchParams = new URLSearchParams();
datasetNames.forEach(name => {
searchParams.append('dataset_name', name);
});
modelNames.forEach(name => {
searchParams.append('pretrain_model_name', name);
});
parameterKeys.forEach(name => {
searchParams.append('param_key', name);
});
return service({
url: `/api/v1/ai_task/generate_sdk_code?${searchParams.toString()}`,
method: 'get',
params: {
job_type: params.job_type,
compute_source: params.compute_source,
cluster_type: params.cluster_type
},
data: {},
});
}


+ 161
- 0
web_src/vuepages/components/CommonTipsDialog.vue View File

@@ -0,0 +1,161 @@

<template>
<div class="common-tips-dlg">
<div class="trigger-container" @click="handlerOpen">
<slot></slot>
</div>
<BaseDialog :visible="visible" :title="title" :show-close="showClose" @closed="closeDialog"
:appendToBody="appendToBody">
<div class="markdown" v-html="content"></div>
<div slot="footer" class="dialog-footer">
<el-button class="close-btn" size="default" type="success" @click="closeDialog">{{
$t('cloudbrainObj.dialogTips.tips8') }}</el-button>
</div>
</BaseDialog>
</div>
</template>

<script>
import BaseDialog from '~/components/BaseDialog.vue';
import { getPromoteData, getMarkdownHtml, getSDKCode } from '~/apis/modules/common';
import SparkMD5 from "spark-md5";
import hljs from 'highlight.js';
import { initClipboard } from '~/utils';
export default {
name: "CommonTipsDialog",
components: { BaseDialog },
props: {
title: { type: String, default: 'Title' },
promotePath: { type: String, default: '' },
appendToBody: { type: Boolean, default: false },
type: { type: String, default: '' },
data: { type: Object, default: () => ({}) }
},
data() {
return {
showClose: true,
visible: false,
loading: false,
content: '',
}
},
methods: {
sparkMD5Hash(str = '') {
return SparkMD5.hash(str) + Math.random().toString().replace('0.', '');
},
hljsAndInsertCopyButton(htmlStr) {
const html = document.createElement('div');
html.innerHTML = htmlStr;
const codeBlocks = html.querySelectorAll('.code-block')
for (let i = 0, iLen = codeBlocks.length; i < iLen; i++) {
const codeBlockI = codeBlocks[i];
const txt = codeBlockI.textContent;
const codeEl = codeBlockI.querySelector('code');
const code = codeEl.textContent;
codeEl.innerHTML = hljs.highlight('python', code).value;
const copyBtn = document.createElement('div');
copyBtn.classList = ['copy-btn'];
copyBtn.innerHTML = `<a href="javascript:;" class="ui poping inline up clipboard" id="clipboard-${this.sparkMD5Hash(txt)}"
data-position="top center" data-variation="inverted tiny" data-success="${this.$t('copySuccess')}"
data-content="${this.$t('copy')}" data-original="${this.$t('copy')}"
data-clipboard-text=""><i style="font-size:14px;" class="copy outline icon"></i></a>`;
copyBtn.querySelector('a').setAttribute('data-clipboard-text', txt);
codeBlockI.outerHTML = `<div class="code-content">${codeBlockI.outerHTML}${copyBtn.outerHTML}</div>`;
}
return html.innerHTML;
},
getMarkdown(str) {
getMarkdownHtml(str).then(res => {
const html = res.data;
this.content = this.hljsAndInsertCopyButton(html);
this.$nextTick(() => {
initClipboard('.base-dlg .clipboard');
});
}).catch(err => {
console.log(err);
});
},
getContent() {
if (!this.promotePath) return;
this.loading = true;
getPromoteData(this.promotePath).then(res => {
this.loading = false;
let contentStr = res.data;
if (contentStr) {
if (this.type == 'model' && this.data.name) {
getSDKCode({
pretrain_model_name: [this.data.name],
}).then(res => {
res = res.data;
if (res.code == 0 && res.data && res.data.code) {
contentStr += `\n\`\`\`python\n${res.data.code}\`\`\`\n`;
}
this.getMarkdown(contentStr);
}).catch(err => {
console.log(err);
})
} else {
this.getMarkdown(contentStr);
}
}
}).catch(err => {
this.loading = false;
console.log(err);
});
},
handlerOpen() {
this.getContent();
this.visible = true;
},
closeDialog() {
this.visible = false;
setTimeout(() => {
this.content = '';
}, 300);
}
},
mounted() {
},
}
</script>

<style scoped lang="less">
.trigger-container {
display: inline-block;
}

.markdown {
font-size: 14px;
max-height: 65vh;
overflow: auto;
padding-left: 20px;
padding-right: 20px;

/deep/ .code-content {
position: relative;

.copy-btn {
position: absolute;
right: 4px;
top: 4px;
}
}
}

.dialog-footer {
text-align: center;

.close-btn {
color: #fff;
background-color: #21ba45;
border-color: #21ba45;
font-size: 1rem;
font-weight: 700;
}
}
</style>
<style>
.ui.popup {
z-index: 2003;
}
</style>

+ 11
- 8
web_src/vuepages/components/cloudbrain/AIEngineSelect.vue View File

@@ -1,14 +1,17 @@
<template>
<div class="form-row">
<div class="title align-items-center" v-if="showTitle"><span class="required">{{ 'AI引擎' }}</span></div>
<div class="content">
<el-select class="engine-type-sel field-input" v-model="engineType" @change="handleEngineTypeChange">
<el-option v-for="item in engineTypeList" :key="item.id" :value="item.id" :label="item.name"></el-option>
</el-select>
<el-select class="engine-sel field-input" v-model="engine" @change="handleEngineChange">
<el-option v-for="item in engineList" :key="item.id" :value="item.id" :label="item.name"></el-option>
</el-select>
<div class="left-area">
<div class="title align-items-center" v-if="showTitle"><span class="required">{{ 'AI引擎' }}</span></div>
<div class="content">
<el-select class="engine-type-sel field-input" v-model="engineType" @change="handleEngineTypeChange">
<el-option v-for="item in engineTypeList" :key="item.id" :value="item.id" :label="item.name"></el-option>
</el-select>
<el-select class="engine-sel field-input" v-model="engine" @change="handleEngineChange">
<el-option v-for="item in engineList" :key="item.id" :value="item.id" :label="item.name"></el-option>
</el-select>
</div>
</div>
<div class="right-area"></div>
</div>
</template>



+ 16
- 13
web_src/vuepages/components/cloudbrain/AlgBechmarkType.vue View File

@@ -1,21 +1,24 @@
<template>
<div class="form-row">
<div class="title align-items-center" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ '评测类型' }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<div class="content-l">
<el-select class="field-input" v-model="mainType" @change="handleInputChange">
<el-option v-for="item in mainTypeList" :key="item.key" :value="item.key" :label="item.value"></el-option>
</el-select>
<div class="left-area">
<div class="title align-items-center" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ '评测类型' }}</span>
</div>
<div class="content-r">
<div class="title align-items-center"><span class="required">{{ '子类型' }}</span></div>
<el-select class="field-input" v-model="childType" @change="handleInputChange">
<el-option v-for="item in childTypeList" :key="item.key" :value="item.key" :label="item.value"></el-option>
</el-select>
<div class="content" :class="errStatus ? 'error' : ''">
<div class="content-l">
<el-select class="field-input" v-model="mainType" @change="handleInputChange">
<el-option v-for="item in mainTypeList" :key="item.key" :value="item.key" :label="item.value"></el-option>
</el-select>
</div>
<div class="content-r">
<div class="title align-items-center"><span class="required">{{ '子类型' }}</span></div>
<el-select class="field-input" v-model="childType" @change="handleInputChange">
<el-option v-for="item in childTypeList" :key="item.key" :value="item.key" :label="item.value"></el-option>
</el-select>
</div>
</div>
</div>
<div class="right-area"></div>
</div>
</template>



+ 13
- 40
web_src/vuepages/components/cloudbrain/BootFile.vue View File

@@ -1,11 +1,14 @@
<template>
<div class="form-row">
<div class="title align-items-center" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ $t('modelManage.bootFile') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" v-model="currentValue" :placeholder="$t('bootPlaceholder')" @input="handleInput"
@change="handleInputChange" @keyup.native="$event.target.value=$event.target.value.replace(/^\s+|\s+$/gm,'')"></el-input>
<div class="left-area">
<div class="title align-items-center" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ $t('modelManage.bootFile') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" v-model="currentValue" :placeholder="$t('bootPlaceholder')" @input="handleInput"
@change="handleInputChange"
@keyup.native="$event.target.value = $event.target.value.replace(/^\s+|\s+$/gm, '')"></el-input>
</div>
</div>
<div class="right-area">
<div class="btn-select">
@@ -52,16 +55,16 @@ export default {
this.errStatus = false;
} else {
const reg = /.+\.py$/;
this.errStatus = !reg.test(this.currentValue.replace(/^\s+|\s+$/gm,''));
this.errStatus = !reg.test(this.currentValue.replace(/^\s+|\s+$/gm, ''));
}
return !this.errStatus;
},
handleInput(value) {
this.currentValue = value.replace(/^\s+|\s+$/gm,'');
this.$emit('input', value.replace(/^\s+|\s+$/gm,''));
this.currentValue = value.replace(/^\s+|\s+$/gm, '');
this.$emit('input', value.replace(/^\s+|\s+$/gm, ''));
},
handleInputChange(value) {
this.$emit('change', value.replace(/^\s+|\s+$/gm,''));
this.$emit('change', value.replace(/^\s+|\s+$/gm, ''));
this.check();
},
},
@@ -72,34 +75,4 @@ export default {

<style scoped lang="less">
@import 'cloudbrain.less';

.form-row {
.content {
flex: inherit;
width: 50%;
margin-right: 5px;
}

.right-area {
display: flex;
align-items: center;

.btn-select {
display: flex;
justify-content: center;
align-items: center;
cursor: pointer;
color: rgb(3, 102, 214);

i {
margin-right: 4px;
color: rgba(0, 0, 0, .87);
}
}

a {
color: #0366d6;
}
}
}
</style>

+ 10
- 7
web_src/vuepages/components/cloudbrain/BranchName.vue View File

@@ -1,13 +1,16 @@
<template>
<div class="form-row">
<div class="title align-items-center">
<span class="required">{{ $t('cloudbrainObj.codeBranch') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-select class="field-input" v-model="currentValue" @change="handleChange">
<el-option v-for="item in branches" :key="item" :value="item" :label="item"></el-option>
</el-select>
<div class="left-area">
<div class="title align-items-center">
<span class="required">{{ $t('cloudbrainObj.codeBranch') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-select class="field-input" v-model="currentValue" @change="handleChange">
<el-option v-for="item in branches" :key="item" :value="item" :label="item"></el-option>
</el-select>
</div>
</div>
<div class="right-area"></div>
</div>
</template>



+ 11
- 14
web_src/vuepages/components/cloudbrain/DatasetSelect.vue View File

@@ -1,14 +1,16 @@
<template>
<div class="form-row">
<div class="title align-items-center" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.dataset') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<div class="dataset-list-c">
<div class="dataset-item" v-for="(item) in selectList" :key="item.id" :title="item.name"> {{ item.name }};
</div>
<div v-if="selectList.length == 0" class="dataset-item-placeholder">
{{ $t('datasetObj.dataset_select_placeholder') }}
<div class="left-area">
<div class="title align-items-center" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.dataset') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<div class="dataset-list-c">
<div class="dataset-item" v-for="(item) in selectList" :key="item.id" :title="item.name"> {{ item.name }};
</div>
<div v-if="selectList.length == 0" class="dataset-item-placeholder">
{{ $t('datasetObj.dataset_select_placeholder') }}
</div>
</div>
</div>
</div>
@@ -333,10 +335,6 @@ export default {

.form-row {
.content {
flex: inherit;
width: 50%;
margin-right: 5px;

.dataset-list-c {
flex: 1;
min-height: 37.6px;
@@ -373,7 +371,6 @@ export default {
}

.right-area {
// width: 250px;
display: flex;
align-items: center;



+ 44
- 34
web_src/vuepages/components/cloudbrain/FormTop.vue View File

@@ -1,52 +1,61 @@
<template>
<div>
<div class="form-row" v-if="configs.showBenchmarkMode === true">
<div class="title align-items-center"><span class="required">评测场景</span></div>
<div class="content">
<div class="list">
<a class="item" :href="`/${repoOwnerName}/${repoName}/${item.url}`"
:class="item.key == configs.bechmarkMode ? 'focus' : ''" v-for="item in benchmarkModeList" :key="item.key">
<span>{{ item.label }}</span>
</a>
<div class="left-area">
<div class="title align-items-center"><span class="required">评测场景</span></div>
<div class="content">
<div class="list">
<a class="item" :href="`/${repoOwnerName}/${repoName}/${item.url}`"
:class="item.key == configs.bechmarkMode ? 'focus' : ''" v-for="item in benchmarkModeList" :key="item.key">
<span>{{ item.label }}</span>
</a>
</div>
</div>
</div>
</div>
<div class="form-row form-row-cluster" v-if="configs.hideCluster !== true">
<div class="title align-items-center"><span class="required">{{ $t('cloudbrainObj.cluster') }}</span></div>
<div class="content">
<div class="list">
<a class="item" :href="`/${repoOwnerName}/${repoName}/${item.url}`"
:class="item.key == configs.cluster ? 'focus' : ''" v-for="item in configs.clusters" :key="item.key">
<i class="icon ri-global-line"></i>
<span>{{ item.label }}</span>
</a>
<div class="left-area">
<div class="title align-items-center"><span class="required">{{ $t('cloudbrainObj.cluster') }}</span></div>
<div class="content">
<div class="list">
<a class="item" :href="`/${repoOwnerName}/${repoName}/${item.url}`"
:class="item.key == configs.cluster ? 'focus' : ''" v-for="item in configs.clusters" :key="item.key">
<i class="icon ri-global-line"></i>
<span>{{ item.label }}</span>
</a>
</div>
</div>
</div>
</div>
<div class="form-row form-row-computer-resource" v-if="configs.hideComputerResource !== true">
<div class="title"><span class="required">{{ $t('cloudbrainObj.computeResource') }}</span></div>
<div class="content">
<div class="list">
<a class="item" :href="`/${repoOwnerName}/${repoName}/${item.url}`"
:class="item.key == configs.computerResouce ? 'focus' : ''" v-for="item in configs.computerResouces" :key="item.key">
<i class="icon ri-archive-drawer-line"></i>
<span>{{ item.label }}</span>
</a>
<div class="left-area">
<div class="title"><span class="required">{{ $t('cloudbrainObj.computeResource') }}</span></div>
<div class="content">
<div class="list">
<a class="item" :href="`/${repoOwnerName}/${repoName}/${item.url}`"
:class="item.key == configs.computerResouce ? 'focus' : ''" v-for="item in configs.computerResouces"
:key="item.key">
<i class="icon ri-archive-drawer-line"></i>
<span>{{ item.label }}</span>
</a>
</div>
</div>
</div>
</div>
<div class="form-row tips-c">
<div class="title"></div>
<div class="content">
<div class="tips tips-1">
<i class="ri-error-warning-line"></i>
<span>
{{ $t('cloudbrainObj.waitCountStart') }} <span>{{ queueNum }}</span> {{ $t('cloudbrainObj.waitCountEnd') }}
</span>
</div>
<div class="tips tips-2" v-if="configs.hideTips2 !== true">
<i class="ri-error-warning-line"></i>
<span v-html="configs.tips2"></span>
<div class="left-area">
<div class="title"></div>
<div class="content">
<div class="tips tips-1">
<i class="ri-error-warning-line"></i>
<span>
{{ $t('cloudbrainObj.waitCountStart') }} <span>{{ queueNum }}</span> {{ $t('cloudbrainObj.waitCountEnd') }}
</span>
</div>
<div class="tips tips-2" v-if="configs.hideTips2 !== true">
<i class="ri-error-warning-line"></i>
<span v-html="configs.tips2"></span>
</div>
</div>
</div>
</div>
@@ -95,6 +104,7 @@ export default {
display: flex;
align-items: center;
flex-wrap: wrap;

.item {
display: flex;
align-items: center;


+ 8
- 12
web_src/vuepages/components/cloudbrain/ImageSelectV1.vue View File

@@ -1,11 +1,13 @@
<template>
<div class="form-row">
<div class="title" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.image') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" v-model="imageUrl" @input="imageChange"
:placeholder="$t('cloudbrainObj.selectImagePlaceholder')"></el-input>
<div class="left-area">
<div class="title" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.image') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" v-model="imageUrl" @input="imageChange"
:placeholder="$t('cloudbrainObj.selectImagePlaceholder')"></el-input>
</div>
</div>
<div class="right-area">
<div class="btn-select" @click="dlgShow = true">
@@ -199,12 +201,6 @@ export default {
@import 'cloudbrain.less';

.form-row {
.content {
flex: inherit;
width: 50%;
margin-right: 5px;
}

.right-area {
display: flex;
align-items: center;


+ 11
- 16
web_src/vuepages/components/cloudbrain/ImageSelectV2.vue View File

@@ -1,14 +1,17 @@
<template>
<div class="form-row">
<div class="title align-items-center">
<span class="required">{{ $t('cloudbrainObj.image') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-select class="field-input" v-model="currentValue" @change="handleChange">
<el-option v-for="item in images" :key="item.image_id" :value="item.image_id"
:label="item.image_name"></el-option>
</el-select>
<div class="left-area">
<div class="title align-items-center">
<span class="required">{{ $t('cloudbrainObj.image') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-select class="field-input" v-model="currentValue" @change="handleChange">
<el-option v-for="item in images" :key="item.image_id" :value="item.image_id"
:label="item.image_name"></el-option>
</el-select>
</div>
</div>
<div class="right-area"></div>
</div>
</template>

@@ -105,12 +108,4 @@ export default {

<style scoped lang="less">
@import 'cloudbrain.less';

.form-row {
.content {
flex: inherit;
width: 50%;
margin-right: 5px;
}
}
</style>

+ 12
- 14
web_src/vuepages/components/cloudbrain/ModelSelect.vue View File

@@ -1,14 +1,17 @@
<template>
<div class="form-row">
<div class="title" v-if="showTitle"><span :class="required ? 'required' : ''">{{ $t('modelObj.model_label') }}</span>
</div>
<div class="content">
<div class="model-list-c" :class="errStatus ? 'error' : ''">
<div class="model-item" v-for="(item) in selectList" :key="item.id" :title="item._modelName + '/' + item.name">
{{ item._modelName + '/' + item.name }}
</div>
<div v-if="selectList.length == 0" class="model-item-placeholder">
{{ $t('modelObj.model_select_placeholder') }}
<div class="left-area">
<div class="title" v-if="showTitle">
<span :class="required ? 'required' : ''">{{ $t('modelObj.model_label') }}</span>
</div>
<div class="content">
<div class="model-list-c" :class="errStatus ? 'error' : ''">
<div class="model-item" v-for="(item) in selectList" :key="item.id" :title="item._modelName + '/' + item.name">
{{ item._modelName + '/' + item.name }}
</div>
<div v-if="selectList.length == 0" class="model-item-placeholder">
{{ $t('modelObj.model_select_placeholder') }}
</div>
</div>
</div>
</div>
@@ -387,10 +390,6 @@ export default {

.form-row {
.content {
flex: inherit;
width: 50%;
margin-right: 5px;

.model-list-c {
min-height: 37.6px;
border-radius: 4px;
@@ -441,7 +440,6 @@ export default {
}

.right-area {
// width: 250px;
display: flex;
align-items: center;



+ 69
- 83
web_src/vuepages/components/cloudbrain/ModelSelectV2.vue View File

@@ -1,14 +1,17 @@
<template>
<div class="form-row">
<div class="title" v-if="showTitle"><span :class="required ? 'required' : ''">{{ $t('modelObj.model_label') }}</span>
</div>
<div class="content">
<div class="model-list-c" :class="errStatus ? 'error' : ''">
<div class="model-item" v-for="(item) in selectList" :key="item.id" :title="item.name">
{{ item.name }}
</div>
<div v-if="selectList.length == 0" class="model-item-placeholder">
{{ $t('modelObj.model_label') }}
<div class="left-area">
<div class="title" v-if="showTitle">
<span :class="required ? 'required' : ''"> {{ $t('modelObj.model_label') }}</span>
</div>
<div class="content">
<div class="model-list-c" :class="errStatus ? 'error' : ''">
<div class="model-item" v-for="(item) in selectList" :key="item.id" :title="item.name">
{{ item.name }}{{ multiple ? ';' : '' }}
</div>
<div v-if="selectList.length == 0" class="model-item-placeholder">
{{ $t('modelObj.model_label') }}
</div>
</div>
</div>
</div>
@@ -38,8 +41,7 @@
</el-input>
</div>
<el-tree :data="dlgModelTreeData" ref="dlgTreeRef" highlight-current show-checkbox node-key="id"
:default-expanded-keys="dlgInitTreeNode" :props="dlgTreeProps" :index="10" accordion :check-strictly="true"
@check="onTreeCheckChange">
:props="dlgTreeProps" :index="10" accordion :check-strictly="true" @check="onTreeCheckChange">
<span slot-scope="{ node, data }" class="slot-wrap">
<span v-if="data.parent" class="custom-tree-node">
<el-tooltip v-if="data.description" placement="top-start">
@@ -51,6 +53,9 @@
{{ data.version }} </span>
</span>
<img v-if="data.recommend == 1" style="margin-left: 0.4rem" src="/img/jian.svg" />
<span class="exceed-size-tips" v-if="useExceedSize && exceedSize && data.size > exceedSize">
{{ $t('modelObj.model_exceeds_failed') }}{{ exceedSize / (1024 * 1024 * 1024) }}G
</span>
</div>
</span>
</el-tooltip>
@@ -61,6 +66,9 @@
{{ data.version }} </span>
</span>
<img v-if="data.recommend == 1" style="margin-left: 0.4rem" src="/img/jian.svg" />
<span class="exceed-size-tips" v-if="useExceedSize && exceedSize && data.size > exceedSize">
{{ $t('modelObj.model_exceeds_failed') }}{{ exceedSize / (1024 * 1024 * 1024) }}G
</span>
</div>
</span>
<span class="model-repolink model-nowrap" @click.stop="return false;">
@@ -117,6 +125,8 @@ export default {
multiple: { type: Boolean, default: false },
title: { type: String, default: '' },
showTitle: { type: Boolean, default: true },
useExceedSize: { type: Boolean, default: false, },
exceedSize: { type: Number, default: 0 },
},
data() {
return {
@@ -200,6 +210,9 @@ export default {
dataI._modelName = dataI.name;
dataI._modelVersion = dataI.version;
dataI._preTrainModelUrl = dataI.path;
if (this.useExceedSize && dataI.size > this.exceedSize && this.exceedSize) {
dataI.disabled = true;
}
_children.forEach(item => {
item.disabled = true;
item.ModTimeNum = new Date(item.ModTime).getTime();
@@ -277,7 +290,12 @@ export default {
});
this.dlgSelectedModelList.splice(index, 1);
this.dlgSelectedModel = this.dlgSelectedModelList.map(item => item.id);
this.$refs.dlgTreeRef.setCheckedKeys([]);
const children = [];
this.dlgSelectedModelList.map(_data => {
const _children = (this.dlgModelTreeData.filter(itm => itm.id == _data.id)[0] || {}).children || [];
children.push(..._children.map(item => item.id));
})
this.$refs.dlgTreeRef.setCheckedKeys(this.dlgSelectedModel.concat(children));
},
dlgPageChange(page) {
this.dlgPage = page;
@@ -286,78 +304,45 @@ export default {
onTreeCheckChange(data) {
const children = [];
if (data.parent) {
if (this.dlgSelectedModel.indexOf(data.id) >= 0) {
this.dlgSelectedModelList = [];
this.dlgSelectedModel = [];
} else {
this.dlgSelectedModelList = [data];
this.dlgSelectedModel = this.dlgSelectedModelList.map(item => item.id);
children.push(...data.children.map(item => item.id));
}
this.$refs.dlgTreeRef.setCheckedKeys(this.dlgSelectedModel.concat(children));
}
},
onTreeCheckChange1(data, nodeStatus) {
if (this.multiple) {
this.maxCount = 30;
}
let checkedNodes = (nodeStatus.checkedNodes || []).filter(item => !item.parent);
if (this.maxCount == 1 && checkedNodes.length) {
this.dlgSelectedModelList = data.parent ? checkedNodes.slice(0, 1) : [data];
} else {
let diffModel = false;
checkedNodes.map((item1) => {
this.dlgSelectedModelList.map(item2 => {
if (item1._modelID != item2._modelID) {
diffModel = true;
}
})
})
if (diffModel) {
this.$message.warning(this.$t('modelObj.model_should_same_model'));
checkedNodes.map(item => {
this.$refs.dlgTreeRef.setChecked(item.id, false, false);
});
this.dlgSelectedModelList.map(item => {
this.$refs.dlgTreeRef.setChecked(item.id, true, true);
});
return;
}
if (checkedNodes.length > this.maxCount) {
this.$message.warning(this.$t('modelObj.model_most', { msg: this.maxCount }));
const newCheckedNodes = checkedNodes.slice(0, this.maxCount);
const keys = this.dlgSelectedModelList.map(item => item.id);
for (let i = 0, iLen = newCheckedNodes.length; i < iLen; i++) {
if (keys.indexOf(newCheckedNodes[i].id) < 0 && this.dlgSelectedModelList.length < this.maxCount) {
this.dlgSelectedModelList.push(newCheckedNodes[i]);
}
}
} else {
const list = [];
const key = [];
for (let i = 0, iLen = this.dlgSelectedModelList.length; i < iLen; i++) {
const node1 = this.dlgSelectedModelList[i];
for (let j = 0, jLen = checkedNodes.length; j < jLen; j++) {
const node2 = checkedNodes[j];
if (node1.id === node2.id) {
list.push(node2);
key.push(node2.id);
if (this.multiple) {
this.maxCount = 5;
if (this.dlgSelectedModel.indexOf(data.id) >= 0) {
const index = this.dlgSelectedModelList.findIndex(item => item.id == data.id);
this.dlgSelectedModelList.splice(index, 1);
} else {
if (this.dlgSelectedModel.length >= 5) {
this.$message.warning(this.$t('modelObj.model_most', { msg: this.maxCount }));
} else if (this.dlgSelectedModelList.find(item => item.name === data.name)) {
this.$message.warning(this.$t('modelObj.model_not_equal_file'));
} else {
const curSize = data.size;
const selectedSize = this.dlgSelectedModelList.reduce((pre, _data) => {
return pre + _data.size;
}, 0);
if (this.useExceedSize && this.exceedSize && (curSize + selectedSize) > this.exceedSize) {
this.$message.warning(this.$t('modelObj.model_exceeds_failed') + `${(this.exceedSize) / (1024 * 1024 * 1024)}G`);
} else {
this.dlgSelectedModelList.push(data);
}
}
}
for (let j = 0, jLen = checkedNodes.length; j < jLen; j++) {
const node2 = checkedNodes[j];
if (key.indexOf(node2.id) < 0) {
list.push(node2);
}
this.dlgSelectedModel = this.dlgSelectedModelList.map(item => item.id);
this.dlgSelectedModelList.map(_data => {
const _children = (this.dlgModelTreeData.filter(itm => itm.id == _data.id)[0] || {}).children || [];
children.push(..._children.map(item => item.id));
})
} else {
if (this.dlgSelectedModel.indexOf(data.id) >= 0) {
this.dlgSelectedModelList = [];
this.dlgSelectedModel = [];
} else {
this.dlgSelectedModelList = [data];
this.dlgSelectedModel = this.dlgSelectedModelList.map(item => item.id);
children.push(...data.children.map(item => item.id));
}
this.dlgSelectedModelList = list;
}
this.$refs.dlgTreeRef.setCheckedKeys(this.dlgSelectedModel.concat(children));
}
this.$refs.dlgTreeRef.setCheckedKeys(this.dlgSelectedModelList.map(item => item.id), true);
this.dlgSelectedModel = this.dlgSelectedModelList.map((item) => {
return item.id;
});
},
clearSelect() {
this.dlgSelectedModelList = [];
@@ -407,10 +392,6 @@ export default {

.form-row {
.content {
flex: inherit;
width: 50%;
margin-right: 5px;

.model-list-c {
min-height: 37.6px;
border-radius: 4px;
@@ -461,7 +442,6 @@ export default {
}

.right-area {
// width: 250px;
display: flex;
align-items: center;

@@ -689,4 +669,10 @@ export default {
background: rgba(220, 220, 220, 0.8);
padding: 1px 3px;
}

.exceed-size-tips {
margin-left: 1rem;
font-size: 12px;
color: red;
}
</style>

+ 18
- 15
web_src/vuepages/components/cloudbrain/NetworkType.vue View File

@@ -1,21 +1,24 @@
<template>
<div class="form-row">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.networkType') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-radio-group v-model="currentValue" @input="handleInput">
<!-- <el-radio label="all">{{ $t('cloudbrainObj.allNetworkType') }}</el-radio> -->
<el-radio label="no_internet">{{ $t('cloudbrainObj.noInternet') }}</el-radio>
<el-radio label="has_internet">{{ $t('cloudbrainObj.hasInternet') }}</el-radio>
</el-radio-group>
<el-tooltip class="tooltip" placement="top" effect="light">
<i class="question circle icon link" style="margin-top:-7px"></i>
<div slot="content">
<div style="width:200px;text-align:center;" v-html="$t('cloudbrainObj.networkTypeDesc')"></div>
</div>
</el-tooltip>
<div class="left-area">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.networkType') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-radio-group v-model="currentValue" @input="handleInput">
<!-- <el-radio label="all">{{ $t('cloudbrainObj.allNetworkType') }}</el-radio> -->
<el-radio label="no_internet">{{ $t('cloudbrainObj.noInternet') }}</el-radio>
<el-radio label="has_internet">{{ $t('cloudbrainObj.hasInternet') }}</el-radio>
</el-radio-group>
<el-tooltip class="tooltip" placement="top" effect="light">
<i class="question circle icon link" style="margin-top:-7px"></i>
<div slot="content">
<div style="width:200px;text-align:center;" v-html="$t('cloudbrainObj.networkTypeDesc')"></div>
</div>
</el-tooltip>
</div>
</div>
<div class="right-area"></div>
</div>
</template>



+ 28
- 22
web_src/vuepages/components/cloudbrain/RunParameters.vue View File

@@ -1,29 +1,34 @@
<template>
<div class="form-row">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.runParameter') }}</span>
</div>
<div class="content">
<div class="param-list field-input">
<div class="param-item" v-for="(item) in list" :key="item.id">
<div class="param-k-v">
<el-input class="param-k" :class="item.labelError ? 'error' : ''" :placeholder="$t('cloudbrainObj.parameterName')" @input="handleInput(item)"
v-model="item.label"></el-input>
<el-input class="param-v" :class="item.valueError ? 'error' : ''" :placeholder="$t('cloudbrainObj.parameterValue')" @input="handleInput(item)"
v-model="item.value"></el-input>
</div>
<div class="param-del-btn">
<i class="trash icon" @click="removeParameter(item)"></i>
<div class="left-area">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.runParameter') }}</span>
</div>
<div class="content">
<div class="param-list field-input">
<div class="param-item" v-for="(item) in list" :key="item.id">
<div class="param-k-v">
<el-input class="param-k" :class="item.labelError ? 'error' : ''"
:placeholder="$t('cloudbrainObj.parameterName')" @input="handleInput(item)"
v-model="item.label"></el-input>
<el-input class="param-v" :class="item.valueError ? 'error' : ''"
:placeholder="$t('cloudbrainObj.parameterValue')" @input="handleInput(item)"
v-model="item.value"></el-input>
</div>
<div class="param-del-btn">
<i class="trash icon" @click="removeParameter(item)"></i>
</div>
</div>
</div>
</div>
<div class="add-param-btn">
<a href="javascript:;" @click="addParameter">
<i class="plus square outline icon"></i>
<span>{{$t('cloudbrainObj.addRunParameter')}}</span>
</a>
<div class="add-param-btn">
<a href="javascript:;" @click="addParameter">
<i class="plus square outline icon"></i>
<span>{{ $t('cloudbrainObj.addRunParameter') }}</span>
</a>
</div>
</div>
</div>
<div class="right-area"></div>
</div>
</template>

@@ -136,6 +141,7 @@ export default {
display: flex;
align-items: center;
margin-bottom: 12px;
position: relative;

.param-k-v {
flex: 1;
@@ -167,8 +173,6 @@ export default {
}

.param-v {
margin-right: 10px;

&.error {
/deep/.el-input__inner {
color: #9f3a38;
@@ -193,6 +197,8 @@ export default {
}

.param-del-btn {
position: absolute;
right: -28px;
width: 20px;

i {


+ 129
- 0
web_src/vuepages/components/cloudbrain/SDKCode.vue View File

@@ -0,0 +1,129 @@
<template>
<div>
<div class="title">
<p><span v-html="$t('cloudbrainObj.sdkCodeTip1')"></span></p>
<p v-show="codeContent"><span>{{ $t('cloudbrainObj.sdkCodeTip2') }}</span></p>
</div>
<div class="content" v-show="codeContent">
<div class="code-c">
<div class="code-content markdown">
<pre><code class="python hljs" v-html="codeHtml"></code></pre>
</div>
<div class="copy-btn sdk-code-copy">
<a href="javascript:;" class="ui poping up clipboard" :id="`clipboard-${clipboardIDStr}`"
data-position="top center" data-variation="inverted tiny" :data-success="$t('copySuccess')"
:data-content="$t('copy')" :data-original="$t('copy')" :data-clipboard-text="codeContent">
<i style="font-size:14px;" class="copy outline icon"></i>
</a>
</div>
</div>
</div>
</div>
</template>

<script>
import hljs from 'highlight.js';
import { getSDKCode } from '~/apis/modules/common';
import { initClipboard } from '~/utils';

export default {
name: 'SDKCode',
props: {
pageConfigs: { type: Object, required: true, },
formConfigs: { type: Object, required: true, },
data: { type: Object, required: true },
},
data() {
return {
repoOwnerName: location.pathname.split('/')[1],
repoName: location.pathname.split('/')[2],
codeContent: '',
clipboardIDStr: Math.random().toString().replace('0.', ''),
delayTimer: null,
};
},
watch: {
data: {
immediate: true,
deep: true,
handler(newVal) {
this.delayTimer && clearTimeout(this.delayTimer);
this.delayTimer = setTimeout(() => {
this.getCode();
}, 1500);
}
},
},
computed: {
codeHtml() {
return hljs.highlight('python', this.codeContent).value;
},
},
methods: {
getCode() {
const job_type = this.pageConfigs.taskType;
const compute_source = this.pageConfigs.computerResouce;
const cluster_type = this.pageConfigs.clusterType;
const datasetList = this.data.dataset;
const modelList = this.data.model;
const runParameterList = this.data.runParameters;
const pretrain_model_name = modelList.map(itm => itm.name);
const dataset_name = datasetList.map(itm => itm.name);
const param_key = runParameterList.filter(itm => itm.label.trim()).map(itm => itm.label.trim());
// if (pretrain_model_name.length == 0 && dataset_name.length == 0 && param_key.length == 0) {
// this.codeContent = '';
// return;
// };
const params = {
pretrain_model_name,
dataset_name,
param_key,
job_type,
compute_source,
cluster_type,
};
const curParamsStr = JSON.stringify(params);
if (this.paramsStr == curParamsStr) return;
this.paramsStr = curParamsStr;
getSDKCode(params).then(res => {
res = res.data;
if (res.code == 0 && res.data) {
this.codeContent = res.data.code || '';
this.$nextTick(() => {
initClipboard('.sdk-code-copy.copy-btn .clipboard');
});
}
}).catch(err => {
console.log(err);
})
},
check() {
return true;
},
},
mounted() { },
};
</script>

<style scoped lang="less">
.title {
margin-bottom: 10px;
}

.code-c {
position: relative;
font-size: 14px;

.code-content {
position: relative;
overflow: auto;
font-size: 14px;
}

.copy-btn {
position: absolute;
top: 0.5em;
right: 0.8em;
}
}
</style>

+ 33
- 34
web_src/vuepages/components/cloudbrain/SpecSelect.vue View File

@@ -1,40 +1,42 @@
<template>
<div class="form-row">
<div class="title"><span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.resourceSpec') }}</span></div>
<div class="content" :class="errStatus ? 'error' : ''">
<div class="spec-list-c" v-if="!list.length">
<div class="spec-item-placeholder" style="color: red;">
{{ $t('specObj.no_use_resource') }}
</div>
</div>
<div v-else class="spec-info">
<el-select class="spec-sel field-input" :class="configs.showPoint ? 'spec-show-point' : ''" v-model="spec"
:placeholder="$t('cloudbrainObj.specPlaceholder')" @change="changeSpec">
<div slot="prefix" class="spec-sel-icon spec-op-icon">
<div :class="selIconType + '_icon _icon'">{{ selIconType[0] }}</div>
<div class="left-area">
<div class="title"><span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.resourceSpec') }}</span></div>
<div class="content" :class="errStatus ? 'error' : ''">
<div class="spec-list-c" v-if="!list.length">
<div class="spec-item-placeholder" style="color: red;">
{{ $t('specObj.no_use_resource') }}
</div>
<el-option v-for="(item) in list" :key="item.id" :label="item.specStr" :value="item.id">
<span style="float: left" class="spec-op-icon">
<div :class="item.type + '_icon _icon'">{{ item.type[0] }}</div>
</div>
<div v-else class="spec-info">
<el-select class="spec-sel field-input" :class="configs.showPoint ? 'spec-show-point' : ''" v-model="spec"
:placeholder="$t('cloudbrainObj.specPlaceholder')" @change="changeSpec">
<div slot="prefix" class="spec-sel-icon spec-op-icon">
<div :class="selIconType + '_icon _icon'">{{ selIconType[0] }}</div>
</div>
<el-option v-for="(item) in list" :key="item.id" :label="item.specStr" :value="item.id">
<span style="float: left" class="spec-op-icon">
<div :class="item.type + '_icon _icon'">{{ item.type[0] }}</div>
</span>
<span class="spec-op-spec" style="float: left">{{ item.specStr }}</span>
<span class="spec-op-point" style="float: right;" v-if="configs.showPoint">{{ item.pointStr }}</span>
</el-option>
<div slot="prefix" v-if="configs.showPoint" class="spec-sel-point"> {{ selPointStr }} </div>
</el-select>
<div class="self-point-info" v-if="configs.showPoint">
<span>{{ $t('cloudbrainObj.balanceOfPoints') }}:<span style="color:red"> {{ configs.blance }}</span>
{{ $t('cloudbrainObj.points') }}<span v-if="showUseTime">{{ $t('cloudbrainObj.canUseTime') }} <span
style="color:red">{{ canUseTime
}}</span> {{ $t('cloudbrainObj.hours') }}</span></span>
<span>
<i class="el-icon-question"></i>
<a target="_blank" href="/reward/point/rule">{{ $t('cloudbrainObj.PointGainDescr') }}</a>
</span>
<span class="spec-op-spec" style="float: left">{{ item.specStr }}</span>
<span class="spec-op-point" style="float: right;" v-if="configs.showPoint">{{ item.pointStr }}</span>
</el-option>
<div slot="prefix" v-if="configs.showPoint" class="spec-sel-point"> {{ selPointStr }} </div>
</el-select>
<div class="self-point-info" v-if="configs.showPoint">
<span>{{ $t('cloudbrainObj.balanceOfPoints') }}:<span style="color:red"> {{ configs.blance }}</span>
{{ $t('cloudbrainObj.points') }}<span v-if="showUseTime">{{ $t('cloudbrainObj.canUseTime') }} <span
style="color:red">{{ canUseTime
}}</span> {{ $t('cloudbrainObj.hours') }}</span></span>
<span>
<i class="el-icon-question"></i>
<a target="_blank" href="/reward/point/rule">{{ $t('cloudbrainObj.PointGainDescr') }}</a>
</span>
</div>
</div>
</div>
</div>
<div class="resource-descr-c">
<div class="right-area resource-descr-c">
<div class="resource-descr">
<el-tooltip placement="top" effect="light">
<i class="question circle icon link" style="margin-top:-7px"></i>
@@ -167,10 +169,6 @@ export default {

.form-row {
.content {
flex: inherit;
width: 50%;
margin-right: 5px;

.spec-list-c {
min-height: 37.6px;
border-radius: 4px;
@@ -241,6 +239,7 @@ export default {
}

.resource-descr-c {
align-items: flex-start;
margin-top: 8px;

.resource-descr {


+ 11
- 8
web_src/vuepages/components/cloudbrain/TaskDescr.vue View File

@@ -1,13 +1,16 @@
<template>
<div class="form-row">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.taskDescr') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" type="textarea" v-model="currentValue" :rows="3"
:placeholder="$t('cloudbrainObj.taskDescrPlaceholder')" :maxlength="255" @blur="handleBlur" @focus="handleFocus"
@input="handleInput" @change="handleInputChange"></el-input>
<div class="left-area">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.taskDescr') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" type="textarea" v-model="currentValue" :rows="3"
:placeholder="$t('cloudbrainObj.taskDescrPlaceholder')" :maxlength="255" @blur="handleBlur" @focus="handleFocus"
@input="handleInput" @change="handleInputChange"></el-input>
</div>
</div>
<div class="right-area"></div>
</div>
</template>

@@ -68,7 +71,7 @@ export default {
<style scoped lang="less">
@import 'cloudbrain.less';

.form-row .content .field-input /deep/ .el-textarea__inner {
.form-row .left-area .content .field-input /deep/ .el-textarea__inner {
line-height: 1.2857;
padding: 0.78571429em 1em;
}


+ 12
- 8
web_src/vuepages/components/cloudbrain/TaskName.vue View File

@@ -1,14 +1,18 @@
<template>
<div class="form-row">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.taskName') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" v-model="currentValue" @input="handleInput" @change="handleInputChange" :maxlength="type == 'supercompute' ? 26 : 36"
:placeholder="$t('cloudbrainObj.taskName')" :autofocus="autofocus"></el-input>
<div v-if="type === 'supercompute'" class="tips">{{ $t('cloudbrainObj.taskNameTips1') }}</div>
<div v-else class="tips">{{ $t('cloudbrainObj.taskNameTips') }}</div>
<div class="left-area">
<div class="title">
<span :class="required ? 'required' : ''">{{ $t('cloudbrainObj.taskName') }}</span>
</div>
<div class="content" :class="errStatus ? 'error' : ''">
<el-input class="field-input" v-model="currentValue" @input="handleInput" @change="handleInputChange"
:maxlength="type == 'supercompute' ? 26 : 36" :placeholder="$t('cloudbrainObj.taskName')"
:autofocus="autofocus"></el-input>
<div v-if="type === 'supercompute'" class="tips">{{ $t('cloudbrainObj.taskNameTips1') }}</div>
<div v-else class="tips">{{ $t('cloudbrainObj.taskNameTips') }}</div>
</div>
</div>
<div class="right-area"></div>
</div>
</template>



+ 14
- 9
web_src/vuepages/components/cloudbrain/WorkServerNum.vue View File

@@ -1,11 +1,15 @@
<template>
<div class="form-row">
<div class="title align-items-center" v-if="showTitle"><span class="required">{{ $t('cloudbrainObj.computeNodeCount') }}</span></div>
<div class="content">
<el-select class="work-server-num-sel field-input" v-model="workServerNum" @change="handleChange">
<el-option v-for="item in data" :key="item" :value="item" :label="item"></el-option>
</el-select>
<div class="left-area">
<div class="title align-items-center" v-if="showTitle"><span class="required">{{
$t('cloudbrainObj.computeNodeCount') }}</span></div>
<div class="content">
<el-select class="work-server-num-sel field-input" v-model="workServerNum" @change="handleChange">
<el-option v-for="item in data" :key="item" :value="item" :label="item"></el-option>
</el-select>
</div>
</div>
<div class="right-area"></div>
</div>
</template>

@@ -53,10 +57,11 @@ export default {
@import 'cloudbrain.less';

.form-row {
.content {

.work-server-num-sel {
width: 70px;
.left-area {
.content {
.work-server-num-sel {
width: 90px;
}
}
}
}


+ 92
- 61
web_src/vuepages/components/cloudbrain/cloudbrain.less View File

@@ -2,97 +2,128 @@
display: flex;
margin-bottom: 28px;

.title {
width: 200px;
text-align: right;
margin-right: 24px;
color: #101010;
font-size: 14px;
.left-area {
flex: 1;
width: 0;
display: flex;
justify-content: flex-end;
padding-top: 8px;
flex-shrink: 0;

&.align-items-center {
padding-top: 0;
align-items: center;
}
.title {
width: 200px;
text-align: right;
margin-right: 24px;
color: #101010;
font-size: 14px;
display: flex;
justify-content: flex-end;
padding-top: 8px;
flex-shrink: 0;

&.align-items-center {
padding-top: 0;
align-items: center;
}

.required {
position: relative;
.required {
position: relative;

&::after {
position: absolute;
content: "*";
top: -3px;
right: -10px;
color: red;
&::after {
position: absolute;
content: "*";
top: -3px;
right: -10px;
color: red;
}
}
}
}

.content {
flex: 1;
.content {
flex: 1;
width: 0;

.field-input {
width: 100%;
}

.tips {
font-size: 12px;
color: rgba(136, 136, 136, 1);
margin-top: 10px;
}

.field-input {
/deep/.el-input__inner {
height: 37.6px;
.field-input {
width: 100%;
}

/deep/.el-input__inner,
/deep/.el-textarea__inner {
color: rgba(0, 0, 0, .95);
font-size: 14px;
line-height: 37.6px;

&:visited {
border-color: #85b7d9;
}

&:focus {
border-color: #85b7d9;
}

&:active {
border-color: #85b7d9;
}
.tips {
font-size: 12px;
color: rgba(136, 136, 136, 1);
margin-top: 10px;
}
}

&.error {
.field-input {
/deep/.el-input__inner {
height: 37.6px;
}

/deep/.el-input__inner,
/deep/.el-textarea__inner {
color: #9f3a38;
background: #fff6f6;
border-color: #e0b4b4;
color: rgba(0, 0, 0, .95);
font-size: 14px;
line-height: 37.6px;

&:visited {
border-color: #e0b4b4;
border-color: #85b7d9;
}

&:focus {
border-color: #e0b4b4;
border-color: #85b7d9;
}

&:active {
border-color: #85b7d9;
}
}
}

&.error {
.field-input {

/deep/.el-input__inner,
/deep/.el-textarea__inner {
color: #9f3a38;
background: #fff6f6;
border-color: #e0b4b4;

&:visited {
border-color: #e0b4b4;
}

&:focus {
border-color: #e0b4b4;
}

&:active {
border-color: #e0b4b4;
}
}
}
}
}
}

.right-area {
width: 120px;
display: flex;
margin-left: 8px;
align-items: center;

.btn-select {
display: flex;
justify-content: center;
align-items: center;
cursor: pointer;
color: rgb(3, 102, 214);

i {
margin-right: 4px;
color: rgba(0, 0, 0, .87);
}
}

a {
color: #0366d6;
}
}
}

.submit-btn {


+ 56
- 5
web_src/vuepages/components/cloudbrain/details/ConfigInfo.vue View File

@@ -48,12 +48,12 @@
class="item-block item-model-list">
<div class="table-container">
<el-table ref="tableRef" :data="data.task.pretrain_model_list" style="width:100%">
<el-table-column prop="FileName" :label="$t('modelManage.modelFiles')"
<el-table-column prop="FileName" :label="$t('modelManage.modelList')"
:width="configs.showModelFileDownload ? 300 : ''" align="left" header-align="left">
<template slot-scope="scope">
<div class="tbl-file-name">
<a v-if="!scope.row.is_delete" target="_blank"
:href="`${scope.row.repository_link}/modelmanage/model_readme_tmpl?name=${scope.row.model_name}`">
:href="`${scope.row.repository_link}/modelmanage/model_readme_tmpl?name=${scope.row.name}`">
{{ scope.row.name }}
</a>
<span v-else>{{ scope.row.name }}({{ $t('cloudbrainObj.fileWasDeleted') }})</span>
@@ -79,6 +79,24 @@
</el-table>
</div>
</div>
<div class="item-block item-model-code" v-if="item == 'modelList' && data.task.sdk_code"
:key="item + '-code-' + index">
<div class="code-c">
<div class="code-title">{{ $t('cloudbrainObj.sdkUseWay') }}</div>
<div class="code-content-c">
<div class="code-content">
<pre><code class="python hljs" v-html="renderHljs(data.task.sdk_code)"></code></pre>
</div>
<div class="copy-btn">
<a href="javascript:;" class="ui poping up clipboard" :id="`clipboard-${sparkMD5Hash(data.task.sdk_code)}`"
data-position="top center" data-variation="inverted tiny" :data-success="$t('copySuccess')"
:data-content="$t('copy')" :data-original="$t('copy')" :data-clipboard-text="data.task.sdk_code">
<i style="font-size:14px;" class="copy outline icon"></i>
</a>
</div>
</div>
</div>
</div>
</template>
</div>
</template>
@@ -88,6 +106,7 @@ import { renderSpecObject, initClipboard } from '~/utils';
import { i18n } from '~/langs';
import { formatDate } from 'element-ui/lib/utils/date-util';
import SparkMD5 from "spark-md5";
import hljs from 'highlight.js';

export default {
name: 'ConfigInfo',
@@ -113,8 +132,11 @@ export default {
}
},
methods: {
sparkMD5Hash(str) {
return SparkMD5.hash(str);
sparkMD5Hash(str = '') {
return SparkMD5.hash(str) + Math.random().toString().replace('0.', '');
},
renderHljs(str = '') {
return hljs.highlight('python', str).value;
},
renderTitle(key) {
let result = key;
@@ -399,7 +421,8 @@ export default {
}

&.item-dataset,
&.item-model-list {
&.item-model-list,
&.item-model-code {
display: block;
width: 100%;
padding-right: 0;
@@ -444,6 +467,34 @@ export default {
white-space: break-spaces;
}
}

.code-c {
margin-top: -2px;

.code-title {
color: #8a8e99;
font-size: 12px;
font-weight: 700;
margin-bottom: 4px;
}

.code-content-c {
position: relative;

.code-content {
position: relative;
border: 1px solid rgba(34, 36, 38, .15);
padding: 0 0.8em;
overflow: auto;
}

.copy-btn {
position: absolute;
top: 0.5em;
right: 0.8em;
}
}
}
}

/deep/.clipboard {


+ 22
- 6
web_src/vuepages/langs/config/en-US.js View File

@@ -286,6 +286,7 @@ const en = {
modelBriefIntro: 'Model brief introduction',
trainingInfo: 'Training information',
modifyModelInfo: 'Modify model information',
modelList: 'Model list',
modelFiles: 'Model files',
addModelFiles: 'Add model files',
uploadModelFiles: 'Upload model files',
@@ -304,6 +305,7 @@ const en = {
modelModifyFailed: 'Model modify failed',
fileUpload: 'File upload',
upload: 'Upload',
uploadPath: 'Upload Path',
uploadStatus: 'Upload status',
modelFileUploadDefaultTips: 'Click to add files or drag files here directly',
modelFileUploadErrTips: 'Up to 10 files can be uploaded at a time, and the total file size of the model does not exceed {size}GB',
@@ -316,6 +318,7 @@ const en = {
uploading: 'Uploading...',
fileHasAlreadyInTheModel: 'This file has already in the model: ',
fileEqualInTheModel: 'This file content is the same as other file: ',
fileExistInTheModel: 'This file content is the same as other file: ',
basicInfo: 'Basic information',
modelSize: 'Model size',
descr: 'Description',
@@ -429,22 +432,27 @@ const en = {
},
modelObj: {
model_label: 'Select Model',
model_select_placeholder: 'Please select model file',
model_select_placeholder: 'Please select model',
model_select: 'Select Model',
model_current_repo: 'Current repository',
model_my: 'My models',
model_public: 'Public models',
model_collected: 'My collection',
model_search_placeholder: 'Search model name...',
model_selected: 'Selected model file',
model_selected: 'Selected model',
model_ok: 'OK',
model_most: 'Up to {msg} files.',
model_most: 'Up to {msg} models.',
model_should_same_model: 'Select the files should in the same model.',
model_search: 'Search model',
model_square_empty: 'No Models',
model_suport_file_tips: 'The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt]',
boot_file_helper: 'The startup file is the entry file that your program executes, and it must be a file ending in .py',
can_online_infer: 'Online',
model_exceeds_failed: "Model size exceeds ",
model_sdk_use_way: 'Model SDK use way',
codeUseDlgTriggerTxt: 'How to use models in OpenI',
codeUseDlgTitle: 'How to use models on the OpenI',
model_not_equal_file: "Cannot select models with the same name.",
},
datasetObj: {
dataset_label: "Dataset",
@@ -464,6 +472,9 @@ const en = {
dataset_not_equal_file: "Cannot select a data file with the same name.",
dataset_most: "Up to {msg} files.",
dataset_file_was_deleted: "The file has been deleted",
dataset_sdk_use_way: 'Dataset SDK use way',
codeUseDlgTriggerTxt: 'How to use datasets in OpenI',
codeUseDlgTitle: 'How to use datasets on the OpenI',
},
specObj: {
resSelectTips: 'The "resource specification" is the hardware you use to run the task. In order for more people to use the resources of this platform, please select according to your actual needs',
@@ -527,7 +538,7 @@ const en = {
license: 'License',
publick_dataset: 'Public Dataset',
my_dataset: 'My dataset',
favorite_dataset: 'My favorite dataset'
favorite_dataset: 'My favorite dataset',
},
cloudbrainObj: {
cloudbrain: 'Cloudbrain',
@@ -570,6 +581,7 @@ const en = {
dataset: 'Dataset',
networkType: 'Access Internet',
allNetworkType: 'All network',
codeGenerate: 'Code Generate',
noInternet: 'No',
hasInternet: 'Yes',
networkTypeDesc: '「Access Internet」Describe whether the computing resource center supports internet access. When \'No\' is selected, more computing resource centers can be allocated, and the dataset and models on the OpenI platform can still be used.',
@@ -684,6 +696,7 @@ const en = {
scrollToBottom: 'Scroll to bottom',
migratingData: 'Data migration in progress',
centerPending: 'Queuing in sub centers',
sdkUseWay: 'c2net library usage',

dialogTips: {
tips1: "The platform does not directly provide external ports such as 7860 for external services, and can only use FastAPI to forward to the specified URL exposed to the outside world (environment variable can be obtained: os. getenv ('OPENI_GRADIO_URL ') to provide services.",
@@ -697,8 +710,11 @@ const en = {
tips8: 'Close',
tips9: 'OpenI Online Inference Deployment Requirements',
tips10: 'Your original gradio webui code',
}

},
codeUseDlgTriggerTxt: 'How to access data resources in code',
codeUseDlgTitle: 'How to obtain models, datasets, and output paths in code via the c2net library',
sdkCodeTip1: 'Please use c2net library to access relevant resources in the container, which can be referred to as <a target="_blank" href="https://openi.pcl.ac.cn/docs/index.html#/cloudbrain/codepath">Help</a>.',
sdkCodeTip2: 'Based on the type of cloud brain task you choose, as well as the specified datasets, models, etc. The example code for accessing datasets and models and uploading training output are as follows:',
},
superComputeObj: {
mmlSparkDescr: `The full name of MMLSpark is Microsoft Machine Learning for Apache Spark, which enables users to run customized container images and grants them root accesses within the container. Users can directly use Microsoft's MMLSpark provided by the platform.\nNote: MMLSpark is a Spark version provided by Microsoft for machine learning environments( <a target="_blank" href="https://github.com/Azure/mmlspark">https://github.com/Azure/mmlspark</a> )Regarding mmlspark, please refer to the following paper: <a target="_blank" href="https://arxiv.org/pdf/1810.08744.pdf">https://arxiv.org/pdf/1810.08744.pdf</a>`,


+ 22
- 8
web_src/vuepages/langs/config/zh-CN.js View File

@@ -302,6 +302,7 @@ const zh = {
modelBriefIntro: '模型简介',
trainingInfo: '训练相关信息',
modifyModelInfo: '修改模型信息',
modelList: '模型列表',
modelFiles: '模型文件',
addModelFiles: '增加模型文件',
uploadModelFiles: '上传模型文件',
@@ -320,6 +321,7 @@ const zh = {
modelModifyFailed: '模型修改失败',
fileUpload: '文件上传',
upload: '上传',
uploadPath: '上传路径',
uploadStatus: '上传状态',
modelFileUploadDefaultTips: '点击添加文件或直接拖拽文件到此处',
modelFileUploadErrTips: '单次最多上传10个文件,模型总文件大小不超过{size}G',
@@ -332,6 +334,7 @@ const zh = {
uploading: '上传中...',
fileHasAlreadyInTheModel: '该文件已上传在模型:',
fileEqualInTheModel: '该文件与其它文件内容相同:',
fileExistInTheModel: '已存在相同内容文件:',
basicInfo: '基本信息',
modelSize: '模型大小',
descr: '描述',
@@ -445,22 +448,27 @@ const zh = {
},
modelObj: {
model_label: '选择模型',
model_select_placeholder: '选择模型文件',
model_select_placeholder: '选择模型',
model_select: '选择模型',
model_current_repo: '本项目',
model_my: '我的模型',
model_public: '公开模型',
model_collected: '我收藏的模型',
model_search_placeholder: '搜索模型名称...',
model_selected: '已选模型文件',
model_selected: '已选模型',
model_ok: '确定',
model_most: '最多不超过 {msg} 个文件',
model_most: '最多不超过 {msg} 个模型',
model_should_same_model: '请选择同一模型下的文件',
model_search: '搜索模型',
model_square_empty: '空荡荡的,什么都没有',
model_suport_file_tips: '模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt]',
boot_file_helper: '启动文件是您程序执行的入口文件,必须是以.py结尾的文件。比如train.py、main.py、example/train.py、case/main.py。',
can_online_infer: '可体验',
model_exceeds_failed: "模型大小超过",
model_sdk_use_way: '模型SDK使用方式',
codeUseDlgTriggerTxt: '在OpenI如何使用模型',
codeUseDlgTitle: '如何在OpenI协作平台使用模型',
model_not_equal_file: "不能选择相同名称的模型",
},
datasetObj: {
dataset_label: "数据集",
@@ -480,6 +488,9 @@ const zh = {
dataset_not_equal_file: "不能选择相同名称的数据文件",
dataset_most: "最多不超过 {msg} 个文件",
dataset_file_was_deleted: "文件已经被删除",
dataset_sdk_use_way: '数据集SDK使用方式',
codeUseDlgTriggerTxt: '在OpenI如何使用数据集',
codeUseDlgTitle: '如何在OpenI协作平台使用数据集',
},
specObj: {
resSelectTips: '「资源规格」是您运行该任务使用的硬件,为了更多人能够使用本平台的资源,请按照您的实际需求进行选择。',
@@ -542,8 +553,7 @@ const zh = {
license: '授权许可',
publick_dataset: '公开数据集',
my_dataset: '我的数据集',
favorite_dataset: '我收藏的数据集'

favorite_dataset: '我收藏的数据集',
},
cloudbrainObj: {
cloudbrain: '云脑',
@@ -586,6 +596,7 @@ const zh = {
dataset: '数据集',
networkType: '访问Internet',
allNetworkType: '全部网络',
codeGenerate: '代码生成',
noInternet: '否',
hasInternet: '是',
networkTypeDesc: '「访问Internet」是描述您的任务分配到的计算资源中心是否支持互联网访问的情况。选择“否”时可分配的计算资源中心更多,且依然可以使用启智平台上的数据集和模型。',
@@ -700,6 +711,7 @@ const zh = {
scrollToBottom: '滚动到底部',
migratingData: '数据迁移中',
centerPending: '分中心排队中',
sdkUseWay: '通过c2net库访问方式',

dialogTips: {
tips1: "平台没有对外直接提供端口如:7860 对外提供服务,只能使用fastapi转发到对外暴露的指定的URL(环境变量可获取:os.getenv('OPENI_GRADIO_URL'))提供服务。",
@@ -713,9 +725,11 @@ const zh = {
tips8: '关闭',
tips9: 'OpenI在线推理部署须知',
tips10: '您的原始gradio webui代码',
}


},
codeUseDlgTriggerTxt: '代码中如何访问数据资源',
codeUseDlgTitle: '如何在代码中通过c2net库方式获取模型、数据集和输出路径',
sdkCodeTip1: '请使用c2net库方式在容器中访问相关资源,可参考<a target="_blank" href="https://openi.pcl.ac.cn/docs/index.html#/cloudbrain/codepath">使用帮助</a>。',
sdkCodeTip2: '根据您选择的云脑任务类型,指定的数据集、模型等,访问数据集和模型,回传结果的示例代码如下:',
},
superComputeObj: {
mmlSparkDescr: `MMLSpark全称为Microsoft Machine Learning for Apache Spark,支持用户运行自制容器镜像,且赋予了用户容器内root权限。用户可直接使用平台提供的微软的MMLSpark。\n注:MMLSpark是微软提供针对机器学习环境的Spark版本(<a target="_blank" href="https://github.com/Azure/mmlspark">https://github.com/Azure/mmlspark</a>),关于mmlspark可参考论文:<a target="_blank" href="https://arxiv.org/pdf/1810.08744.pdf">https://arxiv.org/pdf/1810.08744.pdf</a>`,


+ 140
- 132
web_src/vuepages/pages/cloudbrain/configs.js View File

@@ -45,14 +45,14 @@ export const CreatePageConfigs = {
model: '/pretrainmodel',
output: '/model',
}),
// hideTips2: true,
hideTips2: true,
// hideCluster: true,
// hideComputerResource: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: true },
model: { required: false, multiple: true, useExceedSize: true },
imagev1: { required: true },
dataset: { required: false, type: 0, useExceedSize: true },
networkType: { required: true },
@@ -72,8 +72,8 @@ export const CreatePageConfigs = {
form: {
taskName: { required: true, },
taskDescr: { required: false, },
// branchName: { required: true, }, // 启智NPU需要手动下载代码
model: { required: false, multiple: true },
branchName: { required: true, },
model: { required: false, multiple: true, useExceedSize: true },
imagev2: { required: true },
dataset: { required: false, type: 1, useExceedSize: true },
networkType: { required: true },
@@ -92,12 +92,13 @@ export const CreatePageConfigs = {
dataset: '/dataset',
model: '/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
taskType: {},
branchName: {},
model: { required: false, multiple: true },
model: { required: false, multiple: true, useExceedSize: true },
imagev1: { required: true, type: 2 },
dataset: { required: false, type: 0, useExceedSize: true },
networkType: { required: true },
@@ -112,8 +113,8 @@ export const CreatePageConfigs = {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: true },
imagev2: { required: true, relatedSpec: true },
model: { required: false, multiple: true, useExceedSize: true },
dataset: { required: false, type: 1, useExceedSize: true },
networkType: { required: true },
spec: { required: true },
@@ -127,11 +128,12 @@ export const CreatePageConfigs = {
dataset: '/tmp/dataset',
model: '/tmp/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: true },
model: { required: false, multiple: true, useExceedSize: true },
imagev2: { required: true, relatedSpec: true },
dataset: { required: false, useExceedSize: true },
networkType: { required: true },
@@ -146,11 +148,12 @@ export const CreatePageConfigs = {
dataset: '/dataset',
model: '/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: true },
model: { required: false, multiple: true, useExceedSize: true },
imagev2: { required: true, relatedSpec: true },
dataset: { required: false, useExceedSize: true },
networkType: { required: true },
@@ -165,11 +168,12 @@ export const CreatePageConfigs = {
dataset: '/tmp/dataset',
model: '/tmp/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: true },
model: { required: false, multiple: true, useExceedSize: true },
imagev2: { required: true, relatedSpec: true },
dataset: { required: false, useExceedSize: true },
networkType: { required: true },
@@ -184,11 +188,12 @@ export const CreatePageConfigs = {
dataset: '/dataset',
model: '/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: true },
model: { required: false, multiple: true, useExceedSize: true },
imagev2: { required: true, relatedSpec: true },
dataset: { required: false, useExceedSize: true },
networkType: { required: true },
@@ -203,11 +208,12 @@ export const CreatePageConfigs = {
dataset: '/dataset',
model: '/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: true },
model: { required: false, multiple: true, useExceedSize: true },
imagev2: { required: true, relatedSpec: true },
dataset: { required: false, useExceedSize: true },
networkType: { required: true },
@@ -232,25 +238,18 @@ export const CreatePageConfigs = {
model: '/pretrainmodel',
output: '/model',
}),
// hideCluster: true,
// hideComputerResource: true,
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: false },
model: { required: false, multiple: true },
imagev1: { required: true },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_PytorchExample_GPU' },
dataset: { required: true, type: 0 },
networkType: { required: true },
runParameters: { required: false },
spec: { required: true },
/* just test */
// imagev2: { required: true },
// aiEngine: { required: true },
// bootFile: { required: true },
// runParameters: { required: false },
// workServerNum: { required: true },
},
modify: {
showIsContinue: true,
@@ -265,13 +264,14 @@ export const CreatePageConfigs = {
model: 'pretrain_url',
output: 'train_url',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: false },
model: { required: false, multiple: true },
imagev2: { required: true },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_Example' },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/npu_mnist_example/train.py' },
dataset: { required: true, type: 1 },
runParameters: { required: false },
networkType: { required: true },
@@ -280,7 +280,7 @@ export const CreatePageConfigs = {
},
modify: {
showIsContinue: true,
continueSampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_Example/src/branch/master/train_continue.py',
continueSampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/npu_mnist_example/train_continue.py',
},
}],
}],
@@ -296,14 +296,15 @@ export const CreatePageConfigs = {
model: '/tmp/pretrainmodel',
output: '/tmp/output',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
taskType: {},
branchName: {},
model: { multiple: false },
model: { multiple: true },
imagev1: { required: true, type: 2 },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_PytorchExample_GPU/src/branch/master/train_for_c2net.py' },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/gpu_mnist_example/train.py' },
dataset: { required: true, type: 0 },
runParameters: { required: false },
networkType: { required: true },
@@ -319,13 +320,14 @@ export const CreatePageConfigs = {
model: 'pretrain_url',
output: '/cache/output',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: false },
model: { required: false, multiple: true },
imagev2: { required: true, relatedSpec: true },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_Example/src/branch/master/train_for_c2net.py' },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/npu_mnist_example/train.py' },
dataset: { required: true, type: 1 },
runParameters: { required: false },
networkType: { required: true },
@@ -343,13 +345,14 @@ export const CreatePageConfigs = {
model: '/tmp/pretrainmodel',
output: '/tmp/output',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: false },
model: { required: false, multiple: true },
imagev2: { required: true, relatedSpec: true },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_PytorchExample_GCU/src/branch/master/train_for_c2net.py' },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/gcu_mnist_example/train.py' },
dataset: { required: true },
runParameters: { required: false },
networkType: { required: true },
@@ -366,11 +369,12 @@ export const CreatePageConfigs = {
dataset: '/dataset',
model: '/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: false, multiple: false },
model: { required: false, multiple: true },
imagev2: { required: true, relatedSpec: true },
bootFile: { required: true, sampleUrl: '' },
dataset: { required: true },
@@ -397,13 +401,14 @@ export const CreatePageConfigs = {
model: 'pretrain_url',
output: 'result_url',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: true, multiple: false },
imagev2: { required: true },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_Example/src/branch/master/inference.py' },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/npu_mnist_example/inference.py' },
dataset: { required: true, type: 1 },
runParameters: { required: false },
networkType: { required: true },
@@ -424,6 +429,7 @@ export const CreatePageConfigs = {
model: '/tmp/pretrainmodel',
output: '/tmp/output',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
@@ -431,7 +437,7 @@ export const CreatePageConfigs = {
branchName: {},
model: { required: true, multiple: false },
imagev1: { required: true, type: 2 },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_PytorchExample_GPU/src/branch/master/inference_for_c2net.py' },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/gpu_mnist_example/inference.py' },
dataset: { required: true, type: 0 },
runParameters: { required: false },
networkType: { required: true },
@@ -447,13 +453,14 @@ export const CreatePageConfigs = {
model: '/tmp/pretrainmodel',
output: '/tmp/output',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
branchName: { required: true, },
model: { required: true, multiple: false },
imagev2: { required: true, relatedSpec: true },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/MNIST_PytorchExample_GPU/src/branch/master/inference_for_c2net.py' },
bootFile: { required: true, sampleUrl: 'https://openi.pcl.ac.cn/OpenIOSSG/OpenI_Cloudbrain_Example/src/branch/master/gpgpu_mnist_example/inference.py' },
dataset: { required: true, type: 0 },
runParameters: { required: false },
networkType: { required: true },
@@ -544,6 +551,7 @@ export const CreatePageConfigs = {
dataset: '/dataset',
model: '/pretrainmodel',
}),
hideTips2: true,
form: {
taskName: { required: true, },
taskDescr: { required: false, },
@@ -593,7 +601,7 @@ export const ListPageConfigs = {
emptyTitle: i18n.t('cloudbrainObj.trainTaskEmptyTitle'),
emptyTip0: true,
emptyTip1: i18n.t('cloudbrainObj.trainTaskEmptyTip2'),
emptyTip2: i18n.t('cloudbrainObj.debugTaskEmptyTip3'),
emptyTip2: i18n.t('cloudbrainObj.debugTaskEmptyTip3', { url: 'https://openi.pcl.ac.cn/docs/index.html#/cloudbrain/train/train' }),
}, {
jobType: 'INFERENCE',
sortList: getSortList(['', 'GPU', 'NPU', 'ILUVATAR-GPGPU']),
@@ -605,7 +613,7 @@ export const ListPageConfigs = {
emptyTitle: i18n.t('cloudbrainObj.inferenceTaskEmptyTitle'),
emptyTip0: true,
emptyTip1: i18n.t('cloudbrainObj.inferenceTaskEmptyTip2'),
emptyTip2: i18n.t('cloudbrainObj.debugTaskEmptyTip3'),
emptyTip2: i18n.t('cloudbrainObj.debugTaskEmptyTip3', { url: 'https://openi.pcl.ac.cn/docs/index.html#/cloudbrain/infer/inference-job' }),
}, {
jobType: 'BENCHMARK',
jobTypeName: getListValueWithKey(JOB_TYPE, 'BENCHMARK'),
@@ -643,13 +651,13 @@ export const DetailPageConfigs = {
'taskName', 'imagev1',
'status', 'spec',
'creator', 'hasInternet',
'branch', 'modelName',
'computerRes', 'modelVersion',
'createTime', 'modelFiles',
'startTime', 'codePath',
'endTime', 'datasetPath',
'duration', 'modelPath',
'descr', 'outputPath',
'branch', 'codePath',
'computerRes', 'datasetPath',
'createTime', 'modelPath',
'startTime', 'outputPath',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
'modelList',
@@ -668,9 +676,9 @@ export const DetailPageConfigs = {
'taskName', 'imagev2',
'status', 'spec',
'creator', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'computerRes', '',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
@@ -678,8 +686,8 @@ export const DetailPageConfigs = {
'dataset',
'modelList',
],
showDatasetDownload: true,
showModelFileDownload: true,
showDatasetDownload: false,
showModelFileDownload: false,
}],
}],
}],
@@ -695,12 +703,12 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'endTime', 'codePath',
'duration', 'datasetPath',
'descr', 'modelPath',
'computerRes', 'codePath',
'createTime', 'datasetPath',
'startTime', 'modelPath',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
'modelList',
@@ -720,18 +728,18 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'endTime', 'codeObsPath',
'computerRes', '',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
'modelList',
],
showDatasetDownload: true,
showModelFileDownload: true,
showDatasetDownload: false,
showModelFileDownload: false,
}, {
name: 'operationProfile'
}],
@@ -747,9 +755,9 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'computerRes', '',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
@@ -772,9 +780,9 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'computerRes', '',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
@@ -797,9 +805,9 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'computerRes', '',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
@@ -822,9 +830,9 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'computerRes', '',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
@@ -847,9 +855,9 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'computerRes', '',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
@@ -878,11 +886,11 @@ export const DetailPageConfigs = {
'taskName', 'imagev1',
'status', 'spec',
'creator', 'hasInternet',
'branch', 'modelName',
'computerRes', 'modelVersion',
'createTime', 'modelFiles',
'startTime', 'bootFile',
'endTime', 'runParameters',
'branch', 'bootFile',
'computerRes', 'runParameters',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
@@ -907,12 +915,12 @@ export const DetailPageConfigs = {
'taskName', 'imagev2',
'status', 'spec',
'creator', 'hasInternet',
'branch', 'modelName',
'computerRes', 'modelVersion',
'runVersion', 'modelFiles',
'createTime', 'bootFile',
'startTime', 'runParameters',
'endTime', 'workServerNum',
'branch', 'bootFile',
'computerRes', 'runParameters',
'runVersion', 'workServerNum',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
@@ -942,12 +950,12 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'runVersion', 'modelVersion',
'createTime', 'modelFiles',
'startTime', 'bootFile',
'endTime', 'runParameters',
'duration', 'workServerNum',
'computerRes', 'bootFile',
'runVersion', 'runParameters',
'createTime', 'workServerNum',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
@@ -977,12 +985,12 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'runVersion', 'modelVersion',
'createTime', 'modelFiles',
'startTime', 'bootFile',
'endTime', 'runParameters',
'duration', 'workServerNum',
'computerRes', 'bootFile',
'runVersion', 'runParameters',
'createTime', 'workServerNum',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
@@ -1012,12 +1020,12 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'runVersion', 'modelVersion',
'createTime', 'modelFiles',
'startTime', 'bootFile',
'endTime', 'runParameters',
'duration', 'workServerNum',
'computerRes', 'bootFile',
'runVersion', 'runParameters',
'createTime', 'workServerNum',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
@@ -1047,12 +1055,12 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'runVersion', 'modelVersion',
'createTime', 'modelFiles',
'startTime', 'bootFile',
'endTime', 'runParameters',
'duration', 'workServerNum',
'computerRes', 'bootFile',
'runVersion', 'runParameters',
'createTime', 'workServerNum',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
@@ -1088,12 +1096,12 @@ export const DetailPageConfigs = {
'taskName', 'imagev2',
'status', 'spec',
'creator', 'hasInternet',
'branch', 'modelName',
'computerRes', 'modelVersion',
'runVersion', 'modelFiles',
'createTime', 'bootFile',
'startTime', 'runParameters',
'endTime', 'workServerNum',
'branch', 'bootFile',
'computerRes', 'runParameters',
'runVersion', 'workServerNum',
'createTime', '',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
@@ -1123,12 +1131,12 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'endTime', 'bootFile',
'duration', 'runParameters',
'descr', 'workServerNum',
'computerRes', 'bootFile',
'createTime', 'runParameters',
'startTime', 'workServerNum',
'endTime', '',
'duration', '',
'descr', '',
'failedReason', '',
'dataset',
'modelList',
@@ -1157,11 +1165,11 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'computerRes', 'modelName',
'createTime', 'modelVersion',
'startTime', 'modelFiles',
'endTime', 'bootFile',
'duration', 'runParameters',
'computerRes', 'bootFile',
'createTime', 'runParameters',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',
@@ -1260,12 +1268,12 @@ export const DetailPageConfigs = {
'status', 'spec',
'creator', 'aiCenter',
'branch', 'hasInternet',
'bootFile', 'modelName',
'computerRes', 'modelVersion',
'createTime', 'modelFiles',
'startTime', 'codePath',
'endTime', 'datasetPath',
'duration', 'modelPath',
'bootFile', 'codePath',
'computerRes', 'datasetPath',
'createTime', 'modelPath',
'startTime', '',
'endTime', '',
'duration', '',
'descr', '',
'failedReason',
'dataset',


+ 209
- 129
web_src/vuepages/pages/cloudbrain/create/index.vue View File

@@ -10,93 +10,112 @@
<div class="form-head">
<h4>{{ $t('cloudbrainObj.create') }}{{ pageCfg.taskTypeName }}</h4>
</div>
<div class="ui container" v-if="alreadyMsgBoxShow">
<div class="err-msg-box-already">
<div class="msg-content">
<i class="ri-information-line"></i>
<div class="msg-content-tip">
<div class="line-1" v-html="$t('cloudbrainObj.sameTaskTips1')"></div>
<div class="line-2" v-html="$t('cloudbrainObj.sameTaskTips2')"></div>
<div class="form-content" :class="showFormRight ? '' : 'hidden-right'">
<div class="form-left">
<div class="ui container" v-if="alreadyMsgBoxShow">
<div class="err-msg-box-already">
<div class="msg-content">
<i class="ri-information-line"></i>
<div class="msg-content-tip">
<div class="line-1" v-html="$t('cloudbrainObj.sameTaskTips1')"></div>
<div class="line-2" v-html="$t('cloudbrainObj.sameTaskTips2')"></div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="form-body">
<div class="form-body-content">
<div class="main-title">{{ $t('cloudbrainObj.basicInfo') }}:</div>
<FormTop ref="formTopRef" :repoOwnerName="repoOwnerName" :repoName="repoName" :configs="pageCfg"
:queueNum="queueNum"></FormTop>
<NetworkType ref="networkTypeRef" v-if="formCfg.networkType" v-model="state.networkType"></NetworkType>
<SpecSelect ref="specRef" v-if="formCfg.spec" v-model="state.spec" :required="formCfg.spec.required"
:configs="specConfigs" :workServerNum="state.workServerNum" :networkType="state.networkType"></SpecSelect>
<WorkServerNum ref="workServerNumRef" v-if="formCfg.workServerNum && workServerNumList.length > 1"
v-model="state.workServerNum" :required="formCfg.workServerNum.required" :data="workServerNumList">
</WorkServerNum>
<TaskName ref="taskNameRef" v-if="formCfg.taskName" v-model="state.taskName" autofocus
:required="formCfg.taskName.required" :userName="repoOwnerName">
</TaskName>
<TaskDescr ref="taskDescrRef" v-if="formCfg.taskDescr" v-model="state.taskDescr"
:required="formCfg.taskDescr.required"></TaskDescr>
</div>
<div class="line"></div>
<div class="form-body-content">
<div class="main-title params-setting">{{ $t('cloudbrainObj.paramsSetting') }}:</div>
<BranchName ref="branchNameRef" v-if="formCfg.branchName" v-model="state.branchName"
:required="formCfg.branchName.required" :branches="branchList">
</BranchName>
<AIEngineSelect ref="aiEngineRef" v-if="formCfg.aiEngine" v-model="state.engine"
:required="formCfg.aiEngine.required" :engines="engineList">
</AIEngineSelect>
<ModelSelect ref="modelRef" v-if="formCfg.model" v-model="state.model" :required="formCfg.model.required"
:multiple="formCfg.model.multiple" :repoOwnerName="repoOwnerName" :repoName="repoName"></ModelSelect>
<AlgBechmarkType ref="algBechmarkTypeRef" v-if="formCfg.algBechmarkType" v-model="state.algBechmarkType"
:required="formCfg.algBechmarkType.required"></AlgBechmarkType>
<ImageSelectV1 ref="imagev1Ref" v-if="formCfg.imagev1" v-model="state.image_url"
:required="formCfg.imagev1.required" :type="formCfg.imagev1.type != undefined ? formCfg.imagev1.type : 0">
</ImageSelectV1>
<ImageSelectV2 ref="imagev2Ref" v-if="formCfg.imagev2" v-model="state.image" :images="imageList"
:configs="pageCfg" :spec="state.spec" :networkType="state.networkType" @changeImages="changeImages"
:required="formCfg.imagev2.required">
</ImageSelectV2>
<BootFile ref="bootFileRef" v-if="formCfg.bootFile" v-model="state.bootFile"
:required="formCfg.bootFile.required" :sampleUrl="formCfg.bootFile.sampleUrl"></BootFile>
<DatasetSelect ref="datasetRef" v-if="formCfg.dataset" v-model="state.dataset"
:required="formCfg.dataset.required" :type="formCfg.dataset.type != undefined ? formCfg.dataset.type : -1"
:repoOwnerName="repoOwnerName" :repoName="repoName" :exceedSize="datasetSize"
:useExceedSize="formCfg.dataset.useExceedSize || false"></DatasetSelect>
<RunParameters ref="runParametersRef" v-if="formCfg.runParameters" v-model="state.runParameters"
:required="formCfg.runParameters.required">
</RunParameters>
<div class="form-row" v-if="this.isModifyTask && (pageCfg.modify && pageCfg.modify.showIsContinue)">
<div class="title"></div>
<div class="content">
<el-checkbox v-model="state.isContinue">{{ this.$t('cloudbrainObj.reuseLastResult') }}</el-checkbox>
<el-tooltip style="margin-left:12px" placement="top" effect="light">
<i class="question circle icon link" style="margin-top:-7px"></i>
<div slot="content">
<div style="width:200px;text-align:center;">{{ this.$t('cloudbrainObj.continue_helper') }}</div>
<div class="form-body">
<div class="form-body-content">
<div class="main-title">{{ $t('cloudbrainObj.basicInfo') }}:</div>
<FormTop ref="formTopRef" :repoOwnerName="repoOwnerName" :repoName="repoName" :configs="pageCfg"
:queueNum="queueNum"></FormTop>
<NetworkType ref="networkTypeRef" v-if="formCfg.networkType" v-model="state.networkType"></NetworkType>
<SpecSelect ref="specRef" v-if="formCfg.spec" v-model="state.spec" :required="formCfg.spec.required"
:configs="specConfigs" :workServerNum="state.workServerNum" :networkType="state.networkType">
</SpecSelect>
<WorkServerNum ref="workServerNumRef" v-if="formCfg.workServerNum && workServerNumList.length > 1"
v-model="state.workServerNum" :required="formCfg.workServerNum.required" :data="workServerNumList">
</WorkServerNum>
<TaskName ref="taskNameRef" v-if="formCfg.taskName" v-model="state.taskName" autofocus
:required="formCfg.taskName.required" :userName="repoOwnerName">
</TaskName>
<TaskDescr ref="taskDescrRef" v-if="formCfg.taskDescr" v-model="state.taskDescr"
:required="formCfg.taskDescr.required"></TaskDescr>
</div>
<div class="line"></div>
<div class="form-body-content">
<div class="main-title params-setting">{{ $t('cloudbrainObj.paramsSetting') }}:</div>
<ImageSelectV1 ref="imagev1Ref" v-if="formCfg.imagev1" v-model="state.image_url"
:required="formCfg.imagev1.required"
:type="formCfg.imagev1.type != undefined ? formCfg.imagev1.type : 0">
</ImageSelectV1>
<ImageSelectV2 ref="imagev2Ref" v-if="formCfg.imagev2" v-model="state.image" :images="imageList"
:configs="pageCfg" :spec="state.spec" :networkType="state.networkType" @changeImages="changeImages"
:required="formCfg.imagev2.required">
</ImageSelectV2>
<DatasetSelect ref="datasetRef" v-if="formCfg.dataset" v-model="state.dataset"
:required="formCfg.dataset.required"
:type="formCfg.dataset.type != undefined ? formCfg.dataset.type : -1" :repoOwnerName="repoOwnerName"
:repoName="repoName" :exceedSize="datasetSize" :useExceedSize="formCfg.dataset.useExceedSize || false">
</DatasetSelect>
<ModelSelect ref="modelRef" v-if="formCfg.model" v-model="state.model" :required="formCfg.model.required"
:multiple="formCfg.model.multiple" :repoOwnerName="repoOwnerName" :repoName="repoName"
:exceedSize="modelSize" :useExceedSize="formCfg.model.useExceedSize"></ModelSelect>
<BranchName ref="branchNameRef" v-if="formCfg.branchName" v-model="state.branchName"
:required="formCfg.branchName.required" :branches="branchList">
</BranchName>
<AIEngineSelect ref="aiEngineRef" v-if="formCfg.aiEngine" v-model="state.engine"
:required="formCfg.aiEngine.required" :engines="engineList">
</AIEngineSelect>
<AlgBechmarkType ref="algBechmarkTypeRef" v-if="formCfg.algBechmarkType" v-model="state.algBechmarkType"
:required="formCfg.algBechmarkType.required"></AlgBechmarkType>
<BootFile ref="bootFileRef" v-if="formCfg.bootFile" v-model="state.bootFile"
:required="formCfg.bootFile.required" :sampleUrl="formCfg.bootFile.sampleUrl"></BootFile>
<RunParameters ref="runParametersRef" v-if="formCfg.runParameters" v-model="state.runParameters"
:required="formCfg.runParameters.required">
</RunParameters>
<div class="form-row" v-if="this.isModifyTask && (pageCfg.modify && pageCfg.modify.showIsContinue)">
<div class="left-area">
<div class="title"></div>
<div class="content" style="margin-left:4px;">
<el-checkbox v-model="state.isContinue">{{ this.$t('cloudbrainObj.reuseLastResult') }}</el-checkbox>
<el-tooltip style="margin-left:12px" placement="top" effect="light">
<i class="question circle icon link" style="margin-top:-7px"></i>
<div slot="content">
<div style="width:200px;text-align:center;">{{ this.$t('cloudbrainObj.continue_helper') }}</div>
</div>
</el-tooltip>
<a :href="pageCfg.modify ? pageCfg.modify.continueSampleUrl : 'javascript:;'" target="_blank">{{
$t('modelManage.viewSamples') }}</a>
</div>
</div>
</div>
<div class="form-row">
<div class="left-area">
<div class="title"></div>
<div class="content">
<el-button type="primary"
:disabled="maskLoading || alreadyMsgBoxShow || !specConfigs.specs[state.networkType].length"
size="default" class="submit-btn" @click="submit">{{ $t('cloudbrainObj.createTask') }}</el-button>
<el-button class="cancel-btn" size="default" @click="cancel">{{ $t('cancel') }}</el-button>
</div>
</div>
</el-tooltip>
<a :href="pageCfg.modify ? pageCfg.modify.continueSampleUrl : 'javascript:;'" target="_blank">{{
$t('modelManage.viewSamples') }}</a>
</div>
</div>
</div>
<div class="form-row">
<div class="title"></div>
<div class="content">
<el-button type="primary"
:disabled="maskLoading || alreadyMsgBoxShow || !specConfigs.specs[state.networkType].length"
size="default" class="submit-btn" @click="submit">{{ $t('cloudbrainObj.createTask') }}</el-button>
<el-button class="cancel-btn" size="default" @click="cancel">{{ $t('cancel') }}</el-button>
</div>
</div>
<div class="form-right">
<div class="toggle-container" @click="showFormRight = !showFormRight">
<div class="icon ri-arrow-drop-right-line"></div>
</div>
<div class="form-right-content" v-show="showFormRight">
<SDKCode ref="sdkCodeRef" :data="state" :pageConfigs="pageCfg" :formConfigs="formCfg"></SDKCode>
</div>
</div>
</div>
</div>
</div>
<LoadingMask :loading="maskLoading" :content="maskLoadingContent"></LoadingMask>
<DialogTips :visible="visible" @closeDialog="visible=false"></DialogTips>
<DialogTips :visible="visible" @closeDialog="visible = false"></DialogTips>
</div>
</template>

@@ -106,7 +125,7 @@ import TaskName from '~/components/cloudbrain/TaskName.vue';
import TaskDescr from '~/components/cloudbrain/TaskDescr.vue';
import BranchName from '~/components/cloudbrain/BranchName.vue';
import AIEngineSelect from '~/components/cloudbrain/AIEngineSelect.vue';
import ModelSelect from '~/components/cloudbrain/ModelSelect.vue';
import ModelSelect from '~/components/cloudbrain/ModelSelectV2.vue';
import ImageSelectV1 from '~/components/cloudbrain/ImageSelectV1.vue';
import ImageSelectV2 from '~/components/cloudbrain/ImageSelectV2.vue';
import BootFile from '~/components/cloudbrain/BootFile.vue';
@@ -116,6 +135,7 @@ import NetworkType from '~/components/cloudbrain/NetworkType.vue';
import SpecSelect from '~/components/cloudbrain/SpecSelect.vue';
import WorkServerNum from '~/components/cloudbrain/WorkServerNum.vue';
import AlgBechmarkType from '~/components/cloudbrain/AlgBechmarkType.vue';
import SDKCode from '~/components/cloudbrain/SDKCode.vue';
import DialogTips from '~/components/cloudbrain/DialogTips.vue';
import LoadingMask from '~/components/cloudbrain/LoadingMask.vue';

@@ -170,17 +190,21 @@ export default {
maskLoading: false,
maskLoadingContent: '',
datasetSize: 0,
datasetCount: 0,
modelSize: 0,
modelCount: 0,
isModifyTask: false,
modeifyTaskId: '',
oldTask: {},
noSpecFlag: false,
visible:false
visible: false,
showFormRight: true,
};
},
components: {
FormTop, TaskName, TaskDescr, BranchName, BootFile, AIEngineSelect, ImageSelectV1, ImageSelectV2,
ModelSelect, DatasetSelect, RunParameters, NetworkType, SpecSelect, WorkServerNum,
AlgBechmarkType,
AlgBechmarkType, SDKCode,
LoadingMask, DialogTips
},
methods: {
@@ -214,13 +238,8 @@ export default {
subObj['branch_name'] = this.state.branchName;
break;
case 'model':
const modelInfo = this.state.model[0] || {};
const pretrain_model_ckpt_name = this.state.model.map(item => item.name).join(';');
subObj['pretrain_model_id'] = modelInfo._modelID || '';
subObj['pretrain_model_name'] = modelInfo._modelName || '';
subObj['pretrain_model_version'] = modelInfo._modelVersion || '';
subObj['pretrain_model_ckpt_name'] = pretrain_model_ckpt_name || '';
subObj['pretrain_model_url'] = modelInfo._preTrainModelUrl || '';
const modelIDStr = this.state.model.map(item => item.id).join(';');
subObj['pretrain_model_id_str'] = modelIDStr;
break;
case 'imagev1':
subObj['image_url'] = this.state.image_url;
@@ -408,13 +427,8 @@ export default {
const _model = models[i];
if (_model.is_delete) continue;
const model = {
_modelID: task.pretrain_model_id,
_modelName: task.pretrain_model_name,
_modelVersion: task.pretrain_model_version,
_preTrainModelUrl: task.pretrain_model_url,
..._model,
};
model.name = _model.name;
model.id = `${model._modelID}|${model._modelName}|${model._modelVersion}|${model._preTrainModelUrl}|${model.name}`;
modelList.push(model);
}
this.state.model = modelList;
@@ -446,35 +460,6 @@ export default {
this.state.image.image_name = '';
}
},
transformTreeData(data) {
for (let i = 0, iLen = data.length; i < iLen; i++) {
const dataI = data[i];
const _children = dataI.modelFileList || [];
const children = [];
dataI.parent = true;
dataI.disabled = true;
_children.forEach(item => {
item.ModTimeNum = new Date(item.ModTime).getTime();
})
_children.sort((a, b) => a.FileName.localeCompare(b.FileName));
_children.sort((a, b) => b.ModTimeNum - a.ModTimeNum);
for (let j = 0, jLen = _children.length; j < jLen; j++) {
const file = _children[j];
if (file.IsDir) continue;
// const arr = file.FileName.split('.');
// if (!supportCheckPointFileExt.includes(arr[arr.length - 1])) continue;
file._modelID = dataI.id;
file._modelName = dataI.name;
file._modelVersion = dataI.version;
file._preTrainModelUrl = dataI.path;
file.name = file.FileName;
file.id = `${file._modelID}|${file._modelName}|${file._modelVersion}|${file._preTrainModelUrl}|${file.name}`;
children.push(file);
}
dataI.children = children;
}
return data;
},
},
beforeMount() {
const urlParams = getUrlSearchParams();
@@ -484,7 +469,7 @@ export default {
this.cancelUrl = urlParams.backurl;
}
this.pageCfg = configs;
this.formCfg = this.pageCfg.form || {};
if (!configs.taskType || !configs.clusterType || !configs.computerResouce) {
console.log(`page configs error.`);
@@ -512,8 +497,14 @@ export default {
getModelList(params).then((res) => {
if (res.data.count === 1) {
const data = res.data?.data || [];
const transData = this.transformTreeData(data)
this.state.model = transData[0].modelFileList.splice(0, 30)
const model = data[0];
if (model) {
this.state.model = [{
...model,
id: model.id,
name: model.name
}]
}
}
}).catch((err) => {
console.log(err)
@@ -555,6 +546,11 @@ export default {
} : {};
if (data.config && data.config.dataset_max_size) {
this.datasetSize = data.config.dataset_max_size;
this.datasetCount = data.config.dataset_max_num;
}
if (data.config && data.config.model_max_size) {
this.modelSize = data.config.model_max_size;
this.modelCount = data.config.model_max_num;
}
this.workServerNumList = data.allowed_worker_num || [1];
if (this.workServerNumList.length > 1) {
@@ -572,7 +568,7 @@ export default {
},
mounted() {
const isCloseOnlineTips = localStorage.getItem("isCloseOnlineTips")
if(this.pageCfg.taskType==="ONLINEINFERENCE" && !JSON.parse(isCloseOnlineTips)){
if (this.pageCfg.taskType === "ONLINEINFERENCE" && !JSON.parse(isCloseOnlineTips)) {
this.visible = true
}
},
@@ -646,17 +642,87 @@ export default {
border: 1px solid #d4d4d5;
}

.form-body {
min-height: 400px;
.form-content {
display: flex;
border: 1px solid #d4d4d5;
background-color: #FFF;
margin: -1px -1px;

.form-left {
flex: 5;
width: 0;
transition: all .3s ease;
}

.form-right {
flex: 2;
min-width: 300px;
border-left: 1px solid #d4d4d5;
position: relative;
transition: all .3s ease;

.toggle-container {
position: absolute;
width: 12px;
height: 56px;
top: 50%;
left: -12px;
background: url(/img/holder.svg);
transform: translateY(-50%) rotate(180deg);
cursor: pointer;

.icon {
display: inline-block;
margin-top: 17px;
margin-left: -2px;
color: #b7bdc8;
transform: rotate(180deg);
transition: all .3s ease;
}
}

.form-right-content {
margin: 20px;
position: sticky;
top: 10px;

/deep/ code {
white-space: pre-wrap;
word-break: break-all;
}
}
}

&.hidden-right {
.form-left {
flex: 99999;
}

.form-right {
flex: 1;
min-width: 0px;
border-left: 1px solid #d4d4d5;
position: relative;

.toggle-container {
left: -11px;

.icon {
transform: rotate(0deg);
}
}
}
}
}

.form-body {
min-height: 400px;
padding: 3em;
background-color: #FFF;

.form-body-content {
max-width: 1200px;
margin: 0 auto;
padding-right: 120px;
padding-right: 20px;

.main-title {
color: #101010;
@@ -675,4 +741,18 @@ export default {
}
}
}

@media only screen and (min-width: 1600px) {
.form-container .form-content.hidden-right .form-body {
.form-body-content {
padding-right: 120px;
}
}
}

@media only screen and (max-width: 1200px) {
.form-container .form-content:not(.hidden-right) .form-body {
padding-right: 1em;
}
}
</style>

+ 97
- 2
web_src/vuepages/pages/cloudbrain/list/index.vue View File

@@ -9,6 +9,12 @@
</a>
</div>
<div class="right-c">
<div class="use-code-c" v-if="tableData.length > 0">
<CommonTipsDialog type="cloudbrain" :promotePath="codeUsePromotePath"
:title="$t('cloudbrainObj.codeUseDlgTitle')">
<a class="use-dlg-btn">{{ $t('cloudbrainObj.codeUseDlgTriggerTxt') }}</a>
</CommonTipsDialog>
</div>
<el-select class="sort-select" v-model="sort" @change="changeSort" v-if="pageCfg.sortList.length">
<el-option v-for="item in pageCfg.sortList" :key="item.k" :value="item.k" :label="item.v"></el-option>
<template v-slot:prefix><i class="icon server"
@@ -158,6 +164,7 @@
<div v-if="pageCfg.page.emptyTip2" v-html="pageCfg.page.emptyTip2"></div>
<div v-if="pageCfg.page.emptyTip3" v-html="pageCfg.page.emptyTip3"></div>
</div>
<div class="code-use-guide markdown" v-html="codeUseContent"></div>
</div>
</div>
<LoadingMask :loading="maskLoading" :content="maskLoadingContent"></LoadingMask>
@@ -177,17 +184,21 @@

<script>
import LoadingMask from '~/components/cloudbrain/LoadingMask.vue';
import CommonTipsDialog from '~/components/CommonTipsDialog.vue';
import { getListPageConfigs, getCreatePageConfigsByTask } from '../configs';
import { CloudBrainTools } from '../tools';
import { CLUSTERS } from '~/const';
import { getUrlSearchParams, timeSinceUnix, getListValueWithKey } from '~/utils';
import { getUrlSearchParams, timeSinceUnix, getListValueWithKey, initClipboard } from '~/utils';
import { getAiTaskList, getAiTaskDebugUrl, getAiTaskRestart, stopAiTask, deleteAiTask } from '~/apis/modules/cloudbrain';
import { getPromoteData, getMarkdownHtml } from '~/apis/modules/common';

import relativeTime from 'dayjs/plugin/relativeTime';
import localizedFormat from 'dayjs/plugin/localizedFormat';
import 'dayjs/locale/zh-cn';
import 'dayjs/locale/en';
import dayjs from 'dayjs';
import SparkMD5 from "spark-md5";
import hljs from 'highlight.js';
import { lang } from '~/langs';

dayjs.locale(lang == 'zh-CN' ? 'zh-cn' : 'en');
@@ -217,9 +228,11 @@ export default {
maskLoadingContent: '',
operating: false,
taskAlreadyDialogShow: false,
codeUsePromotePath: `tips/cloudbrain/sdkcode${lang == 'zh-CN' ? '' : '_en'}.md`,
codeUseContent: '',
};
},
components: { LoadingMask },
components: { LoadingMask, CommonTipsDialog },
methods: {
getTableData() {
this.loading = true;
@@ -258,6 +271,58 @@ export default {
this.tableData = data.tasks || [];
cloudBrainTools.initRefreshData(this.tableData);
}
this.$nextTick(() => {
if (!this.tableData.length) {
this.getCodeUseGuide();
}
})
}).catch(err => {
this.loading = false;
console.log(err);
});
},
sparkMD5Hash(str = '') {
return SparkMD5.hash(str) + Math.random().toString().replace('0.', '');
},
hljsAndInsertCopyButton(htmlStr) {
const html = document.createElement('div');
html.innerHTML = htmlStr;
const codeBlocks = html.querySelectorAll('.code-block')
for (let i = 0, iLen = codeBlocks.length; i < iLen; i++) {
const codeBlockI = codeBlocks[i];
const txt = codeBlockI.textContent;
const code = codeBlockI.querySelector('code').textContent;
codeBlockI.querySelector('code').innerHTML = hljs.highlight('python', code).value;
const copyBtn = document.createElement('div');
copyBtn.classList = ['copy-btn'];
copyBtn.innerHTML = `<a href="javascript:;" class="ui poping inline up clipboard" id="clipboard-${this.sparkMD5Hash(txt)}"
data-position="top center" data-variation="inverted tiny" data-success="${this.$t('copySuccess')}"
data-content="${this.$t('copy')}" data-original="${this.$t('copy')}"
data-clipboard-text=""><i style="font-size:14px;" class="copy outline icon"></i></a>`;
copyBtn.querySelector('a').setAttribute('data-clipboard-text', txt);
codeBlockI.appendChild(copyBtn);
}
return html.innerHTML;
},
getMarkdown(str) {
getMarkdownHtml(str).then(res => {
const html = res.data;
this.codeUseContent = this.hljsAndInsertCopyButton(html);
this.$nextTick(() => {
initClipboard('.code-use-guide .clipboard');
});
}).catch(err => {
console.log(err);
});
},
getCodeUseGuide() {
if (this.codeUseContent) return;
getPromoteData(this.codeUsePromotePath).then(res => {
this.loading = false;
let contentStr = res.data;
if (contentStr) {
this.getMarkdown(contentStr);
}
}).catch(err => {
this.loading = false;
console.log(err);
@@ -473,6 +538,15 @@ export default {
display: flex;
align-items: center;

.use-code-c {
display: inline-block;
margin-right: 16px;

.use-dlg-btn {
text-decoration: underline;
}
}

.sort-select {
margin-right: 12px;
height: 36px;
@@ -694,4 +768,25 @@ export default {
}
}
}

.code-use-guide.markdown {
font-size: 14px;
padding-left: 20px;
padding-right: 20px;
width: 80%;
margin-top: 20px;
border: 1px solid #d4d4d5;
padding: 20px;
border-radius: 4px;

/deep/ .code-block {
position: relative;

.copy-btn {
position: absolute;
right: 12px;
top: 12px;
}
}
}
</style>

+ 17
- 29
web_src/vuepages/pages/computingpower/domestic/index.vue View File

@@ -14,8 +14,7 @@
</div>
<div class="sort-type-c">
<el-select :size="useSmall ? 'small' : 'default'" v-model="sortType" @change="changeSort">
<el-option v-for="(item, index) in sortList" :key="item.key" :value="item.key"
:label="item.label"></el-option>
<el-option v-for="(item) in sortList" :key="item.key" :value="item.key" :label="item.label"></el-option>
</el-select>
</div>
</div>
@@ -157,7 +156,6 @@ export default {
],
features: [
{ title: '产品特点', list: ['- 自研华为达芬奇架构NPU', '- 640 TOPS@INT8,320TFLOPS@FP16', '- 最大功耗310W'], },

],
table: {
title: '关键特性',
@@ -276,23 +274,18 @@ export default {
name: '天垓100(BI -V100)',
icon: manufacturerIconMap.iluvatar,
descr: [
'天数智芯天垓100加速卡是一款基于天垓100芯片的通用GPU训练加速卡。',
'天垓100芯片采用通用GPU架构,支持FP32,FP16,INT32/16/8等多精度数据混合训练,并可提供147 TFLOPS@FP16/BF16的峰值算;兼容多种主流服务器和市场主流生态,兼容CUDA生态,可助力客户实现无痛系统迁移;可提供灵活的编程能力,超强的性能及富有吸引力的性价比。同时,天垓100芯片可提供灵活的编程能力和完善高效的软件栈工具,支持x86和ARM等架构,能够支持国内外主流软硬件生态和各种深度学习框架、算法模型和加速库,应用迁移成本低、耗时短、无需重新开发,用户几乎可以无感知地使用天垓100产品。'
'天垓100通用GPU训练加速卡,产品特点如下:'
],
features: [
{ title: '应用覆盖广', list: ['天垓100聚焦高性能、通用性和灵活性,支持200余种人工智能模型(数量持续增加),支持通用计算、科学计算、大模型、支持业界前沿新算法模型。模型适配速度快,从容面对未来的算法变迁,为人工智能及通用计算和相关垂直应用行业提供匹配行业高速发展的计算力。'], },
{ title: '性能可预期', list: ['天垓100基于通用GPU架构设计,拥有丰富的自研指令集全方位支持标量、矢量、张量运算,提供业界领先的高算力和高能效比,在百余个算法模型的测试平均性能可媲美主流产品。'], },
{ title: '开发易迁移', list: ['适配主流 CPU 芯片 / 服务器厂商,能够支持国内外主流软硬件生态和各种深度学习框架、算法模型和加速库,并通过标准化的软硬件生态接口为行业解决产品使用难、开发平台迁移成本大等痛点,应用迁移成本低、耗时短、无需重新开发,大幅缩短适配验证周期,使客户业务系统几乎无感知地使用天垓 100 产品。'], },
{ title: '全栈可定制', list: ['天垓100 的核心IP 、系统架构、指令集、核心算子、软件栈均由天数科学家团队开发完成,本地支持团队能够根据客户需求提供定制化开发服务。'], },
],
table: {
title: '天垓100产品规格',
fields: [
{ title: '算力性能', value: '37 TFLOPS@FP32<br>147 TFLOPS@FP16/BF16<br>295 TOPS@INT8<br>支持INT16、INT32计算' },
{ title: '板级功耗', value: '250W TDP' },
{ title: '内存理论数据', value: '32GB HBM2' },
{ title: '接口', value: 'PCIe Gen4.0 x16lane<br>共享64GB/s主控双向带宽,共享64GB/s片间互联带宽' },
{ title: '板卡规格', value: '全长全高双槽PCIe卡<br>被动散热' },
{ title: '虚拟化', value: '支持Hypervisor,Docker' },
]
},
useExample: {
title: '在启智AI协作平台使用案例:',
list: [
'https://openi.pcl.ac.cn/iluvatar/TianGai100',
'https://openi.pcl.ac.cn/iluvatar/bert_crf_sequence_labeling',
'https://openi.pcl.ac.cn/iluvatar/resnet50',
'https://openi.pcl.ac.cn/iluvatar/paddleyolo',
@@ -302,23 +295,18 @@ export default {
name: '智铠100(MR-V100)',
icon: manufacturerIconMap.iluvatar,
descr: [
'天数智芯智铠100加速卡是一款基于智铠100芯片的通用GPU推理加速卡。',
'智铠100芯片采用通用GPU架构,支持FP32,FP16,INT8等多精度推理。智铠100具有应用覆盖广,计算性能高,应用成本低及落地支持强的特性,兼容CUDA生态,提供灵活的编程能力,超强的性能及富有吸引力的性价比。智铠100芯片可提供完善高效的软件栈工具,支持X86和ARM等架构。天数智芯软件栈集成了多种主流的深度学习编程框架,并且提供了基于C/C++的编程接口拓展和高性能函数库,提供了一系列调试和调优工具,为高性能计算和人工智能应用的开发和部署提供了便利。'
'智铠100系列通用GPU推理加速卡,产品特点如下:'
],
features: [
{ title: '计算性能高', list: ['支持FP32、FP16、INT8等多精度推理混合计算,实现了指令集增强、算力密度提升、计算存储再平衡,相较于市场上现有主流产品,智铠100将提供2-3倍的实际使用性能。'], },
{ title: '应用覆盖广', list: ['智铠100系列加速卡基于通用GPU架构,支持多种视频规格解码、800+通用指令集、国内外主流深度学习开发框架,拥有丰富编程接口拓展和高性能函数库,可以灵活支持各种算法模型,便于客户自定义开发。'], },
{ title: '使用成本低', list: ['兼容CUDA生态,支持市场主流生态,高达128路视频接入,单路视频性价比高灵活的编程能力,超强的性能及富有吸引力的性价比,为高性能计算和人工智能应用的开发和部署提供了便利。平均迁移时间相较市场主流产品下降50%以上,生态应用迁移迅速。'], },
{ title: '落地支持强', list: ['智铠100全自研架构、核心及软件栈,支持算力开发与优化,可根据客户需求提供定制化开发服务。'], },
],
table: {
title: '智铠100产品规格',
fields: [
{ title: '峰值算力', value: '24 TFLOPS @ FP32<br>96 TFLOPS @ FP16<br>384 TOPS @ INT8' },
{ title: '内存', value: '32GB HBM2E' },
{ title: '视频图像处理', value: '128路FHD解码<br>HEVC/AVC/P9/AVS2<br>2000/500 FPS JPEG编解码' },
{ title: '主控及互联', value: 'PCIe 4.0x16,64GB/s双向宽带' },
{ title: '热设计功耗', value: '150W' },
{ title: '板卡规格', value: '全高全长,PCIe单槽' },
]
},
useExample: {
title: '在启智AI协作平台使用案例:',
list: [
'https://openi.pcl.ac.cn/iluvatar/iluva202401091747306',
'https://openi.pcl.ac.cn/iluvatar/bert_crf_sequence_labeling',
'https://openi.pcl.ac.cn/iluvatar/resnet50',
'https://openi.pcl.ac.cn/iluvatar/paddleyolo',


+ 27
- 0
web_src/vuepages/pages/dataset/codedialog/index.vue View File

@@ -0,0 +1,27 @@
<template>
<CommonTipsDialog type="dataset" :promotePath="promotePath" :appendToBody="true"
:title="$t('datasetObj.codeUseDlgTitle')">
<a class="use-dlg-btn">{{ $t('datasetObj.codeUseDlgTriggerTxt') }}</a>
</CommonTipsDialog>
</template>
<script>

import CommonTipsDialog from '~/components/CommonTipsDialog.vue';
import { lang } from '~/langs';

export default {
data() {
return {
promotePath: `tips/dataset/sdkcode${lang == 'zh-CN' ? '' : '_en'}.md`
}
},
components: { CommonTipsDialog },
methods: {},
mounted() { },
}
</script>
<style scoped>
.use-dlg-btn {
text-decoration: underline;
}
</style>

+ 17
- 0
web_src/vuepages/pages/dataset/codedialog/vp-dataset-code-dialog.js View File

@@ -0,0 +1,17 @@
import Vue from 'vue';
import ElementUI from 'element-ui';
import 'element-ui/lib/theme-chalk/index.css';
import localeEn from 'element-ui/lib/locale/lang/en';
import localeZh from 'element-ui/lib/locale/lang/zh-CN';
import { i18n, lang } from '~/langs';
import App from './index.vue';

Vue.use(ElementUI, {
locale: lang === 'zh-CN' ? localeZh : localeEn,
size: 'small',
});

new Vue({
i18n,
render: (h) => h(App),
}).$mount('#__dataset-code-dialog');

+ 47
- 37
web_src/vuepages/pages/modelmanage/components/ModelHeader.vue View File

@@ -7,13 +7,14 @@
<img src="/img/jian.svg" v-if="model.recommend == 1">
</div>
<div class="title-r">
<CommonTipsDialog type="model" :promotePath="promotePath" :title="$t('modelObj.codeUseDlgTitle')" :data="model">
<a class="use-dlg-btn">{{ $t('modelObj.codeUseDlgTriggerTxt') }}</a>
</CommonTipsDialog>
<div style="background: transparent;" v-if="!isCanDebug && !model.isPrivate">
<el-button class="create-btn" size="mini" type="primary" @click="createOnlineDebug"
v-loading.fullscreen.lock="fullscreenLoading"
element-loading-text="拼命加载中"
element-loading-background="rgba(0, 0, 0, 0.8)"
>
{{ $t('modelManage.debugModel') }}
<el-button class="create-btn" size="mini" type="primary" @click="createOnlineDebug"
v-loading.fullscreen.lock="fullscreenLoading" element-loading-text="拼命加载中"
element-loading-background="rgba(0, 0, 0, 0.8)">
{{ $t('modelManage.debugModel') }}
</el-button>
</div>
<div @click="changeFav" class="like-btn">
@@ -80,7 +81,9 @@
</template>

<script>
import { setModelFav,createModelNotebook } from '~/apis/modules/modelsquare';
import CommonTipsDialog from '~/components/CommonTipsDialog.vue';
import { setModelFav, createModelNotebook } from '~/apis/modules/modelsquare';
import { lang } from '~/langs';

export default {
name: "ModelHeader",
@@ -91,7 +94,7 @@ export default {
modelName: { type: String, default: '' },
tab: { type: String, default: 'intro' },
},
components: {},
components: { CommonTipsDialog },
data() {
return {
favCnt: '',
@@ -99,8 +102,9 @@ export default {
collectedCount: '',
isSetting: false,
fullscreenLoading: false,
isCanDebug:false,
loginName:'zhoupzh',
isCanDebug: false,
loginName: 'zhoupzh',
promotePath: `tips/model/sdkcode${lang == 'zh-CN' ? '' : '_en'}.md`
};
},
methods: {
@@ -127,44 +131,44 @@ export default {
this.isSetting = false;
});
},
createOnlineDebug(){
createOnlineDebug() {
this.fullscreenLoading = true;
createModelNotebook({repoId:this.model.repoId,modelId:this.model.id}).then((res)=>{
this.$message.success(this.$t('modelManage.forkModelSuccess'));
createModelNotebook({ repoId: this.model.repoId, modelId: this.model.id }).then((res) => {
this.$message.success(this.$t('modelManage.forkModelSuccess'));
this.fullscreenLoading = false;
const repoPath = res.data.repoPath
if(this.model.modelType===1){
if(this.model.computeResource==='CPU/GPU'){
if (this.model.modelType === 1) {
if (this.model.computeResource === 'CPU/GPU') {
window.open(`/${repoPath}/grampus/notebook/create?type=0&model_name=${this.model.name}`)
}else{
} else {
window.open(`/${repoPath}/grampus/notebook/create?type=1&model_name=${this.model.name}`)
}
}else{
} else {
const trainTaskInfo = this.model.trainTaskInfo ? JSON.parse(this.model.trainTaskInfo) : '';
const taskType = trainTaskInfo.Type;
const ComputeResource = trainTaskInfo.ComputeResource
const image = trainTaskInfo.Image
let taskUrl = ''
if (taskType == 0) {
taskUrl = `/${repoPath}/cloudbrain/create?model_name=${this.model.name}&model_image=${image}`
} else if (taskType == 1) {
taskUrl = `/${repoPath}/modelarts/notebook/create?model_name=${this.model.name}`
} else if (taskType == 2) {
if(ComputeResource==="NPU"){
taskUrl = `/${repoPath}/grampus/notebook/create?type=1&model_name=${this.model.name}`
}else if(ComputeResource==="GCU"){
taskUrl = `/${repoPath}/grampus/notebook/create?type=2&model_name=${this.model.name}`
}else{
taskUrl = `/${repoPath}/grampus/notebook/create?type=0&model_name=${this.model.name}&model_image=${image}`
}
if (taskType == 0) {
taskUrl = `/${repoPath}/cloudbrain/create?model_name=${this.model.name}&model_image=${image}`
} else if (taskType == 1) {
taskUrl = `/${repoPath}/modelarts/notebook/create?model_name=${this.model.name}`
} else if (taskType == 2) {
if (ComputeResource === "NPU") {
taskUrl = `/${repoPath}/grampus/notebook/create?type=1&model_name=${this.model.name}`
} else if (ComputeResource === "GCU") {
taskUrl = `/${repoPath}/grampus/notebook/create?type=2&model_name=${this.model.name}`
} else {
taskUrl = `/${repoPath}/grampus/notebook/create?type=0&model_name=${this.model.name}&model_image=${image}`
}

}
window.open(taskUrl)
}
}).catch((err)=>{
if(err.response.status===401){
}).catch((err) => {
if (err.response.status === 401) {
window.location.href = `/user/login?redirect_to=${encodeURIComponent(window.location.href)}`;
}else{
} else {
this.$message.error(err);
}
this.fullscreenLoading = false;
@@ -176,9 +180,9 @@ export default {
handler(newVal) {
this.isCollected = newVal.isCollected;
this.collectedCount = newVal.collectedCount;
if(newVal.userName===this.loginName){
if (newVal.userName === this.loginName) {
this.isCanDebug = true
}else{
} else {
this.isCanDebug = false
}
},
@@ -186,10 +190,10 @@ export default {
deep: true,
},
},
mounted() {},
mounted() { },
beforeMount() {
const isLogin = !!document.querySelector('meta[name="_uid"]');
if(isLogin){
if (isLogin) {
this.loginName = document.querySelector('meta[name="_uid"]').getAttribute('content-ext')
}
},
@@ -269,6 +273,7 @@ export default {
cursor: default;
}
}

.create-btn {
font-size: 14px;
margin-right: 1rem;
@@ -332,4 +337,9 @@ export default {
}
}
}

.use-dlg-btn {
margin-right: 1rem;
text-decoration: underline;
}
</style>

+ 34
- 9
web_src/vuepages/pages/modelmanage/files/index.vue View File

@@ -4,8 +4,8 @@
<NotFound></NotFound>
</div>
<div v-else>
<ModelHeader :tab="'files'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName" v-if="JSON.stringify(modelData)!=='{}'"
:model="modelData">
<ModelHeader :tab="'files'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName"
v-if="JSON.stringify(modelData) !== '{}'" :model="modelData">
</ModelHeader>
<div class="ui container">
<div class="header">
@@ -22,7 +22,7 @@
<div style="width:100%;margin-right:20px;">
<div class="title">{{ $t('modelManage.modelFilesList') }}:</div>
<div class="title files-path-c" style="margin-top:8px;margin-bottom:4px">
<div class="file-path" v-for="(item, index) in filePath">
<div class="file-path" v-for="(item, index) in filePath" :key="index">
<span v-if="index == filePath.length - 1" class="path-name">{{ item.label }}</span>
<a v-if="index != filePath.length - 1" class="path-name canback" @click="goBackDir(item)">{{
item.label
@@ -125,6 +125,7 @@ export default {
engineList: MODEL_ENGINES,

curVersion: '',
urlPath: '',
modelList: [],
filesList: [],
filePath: [],
@@ -132,7 +133,7 @@ export default {
},
components: { ModelHeader, NotFound },
methods: {
getDirFiles(dir) {
getDirFiles(dir, usePath, tempFilePath) {
dir = dir.length ? dir.slice(1) : '';
getModelFiles({
repo: this.repoUrl,
@@ -140,10 +141,16 @@ export default {
parentDir: dir,
}).then(res => {
const list = res.data || [];
if (usePath && !list.length) {
this.filePath = this.filePath.slice(0, 1);
this.getDirFiles('');
return;
}
this.filePath = usePath ? tempFilePath : this.filePath;
list.forEach(item => {
item.SizeShow = item.IsDir ? '' : transFileSize(item.Size);
item.ModTimeNum = new Date(item.ModTime).getTime();
});
});
list.sort((a, b) => a.FileName.localeCompare(b.FileName));
list.sort((a, b) => b.ModTimeNum - a.ModTimeNum);
this.filesList = list;
@@ -203,14 +210,29 @@ export default {
this.curVersion = version;
if (!noFileRefresh) {
this.filePath = [{ label: version, path: '' }];
this.getDirFiles('')
if (this.urlPath) {
const path = this.urlPath.split('/');
const pathList = path.map(item => {
return {
label: item,
path: item,
}
});
const tempFilePath = this.filePath.concat(pathList)
const dir = tempFilePath.map((item) => item.path).join('/');
const usePath = true;
this.getDirFiles(dir, usePath, tempFilePath);
} else {
this.getDirFiles('');
}
}
},
goUploadPage() {
window.location.href = `${this.repoUrl}/modelmanage/model_fileupload_tmpl?type=1&name=${encodeURIComponent(this.state.name)}&id=${this.state.id}`;
const dir = (this.filePath.map((item) => item.path).join('/') + '/').replace(/^\//, '');
window.location.href = `${this.repoUrl}/modelmanage/model_fileupload_tmpl?type=1&name=${encodeURIComponent(this.state.name)}&id=${this.state.id}&dir=${encodeURIComponent(dir)}`;
},
backToModelListPage() {
const list = window.location.href.split('/');
const list = window.location.pathname.split('/');
list.pop();
list.push('show_model');
window.location.href = list.join('/');
@@ -222,11 +244,13 @@ export default {
type: 'warning',
lockScroll: false,
}).then(() => {
let path = this.filePath.map(item => item.path).join('/').replace(/^\//, '');
path = path ? path + '/' : '';
this.loading = true;
deleteModelFile({
repo: this.repoUrl,
id: this.state.id,
fileName: file.FileName,
fileName: path + file.FileName,
}).then(res => {
res = res.data;
if (res.code == '0') {
@@ -271,6 +295,7 @@ export default {
if (urlParams.name) {
this.modelName = urlParams.name;
this.state.name = urlParams.name;
this.urlPath = urlParams.path;
this.loading = true;
getModelInfoByName({
repo: this.repoUrl,


+ 19
- 8
web_src/vuepages/pages/modelmanage/fileupload/index.vue View File

@@ -32,6 +32,13 @@
</el-input>
</div>
</div>
<div class="row">
<div class="r-title"><label class="required">{{ $t('modelManage.uploadPath') }}</label></div>
<div class="r-content">
<el-input size="medium" class="input-disabled" :value="'/' + uploadDir" readonly>
</el-input>
</div>
</div>
<div class="row" style="align-items:flex-start;">
<div class="r-title"><label class="required">{{ $t('modelManage.fileUpload') }}</label></div>
<div class="r-content">
@@ -111,7 +118,7 @@ export default {
repoOwnerName: location.pathname.split('/')[1],
repoName: location.pathname.split('/')[2],
modelData: {},
uploadDir: '',
dropzoneHandler: null,
type: '1', // 1-修改,其它-新增
state: {
@@ -141,6 +148,9 @@ export default {
initModelData() {
const urlParams = getUrlSearchParams();
if (urlParams.name && urlParams.id) {
if (urlParams.dir) {
this.uploadDir = urlParams.dir;
}
this.modelName = urlParams.name;
this.type = urlParams.type || '1';
this.state.name = urlParams.name;
@@ -298,7 +308,7 @@ export default {
md5: file.uniqueIdentifier,
type: this.state.type,
modeluuid: this.state.id,
file_name: file.name,
file_name: this.uploadDir + file.name,
scene: 'model',
}).then(res => {
const data = res.data;
@@ -323,7 +333,7 @@ export default {
size: file.size,
fileType: file.type,
type: this.state.type,
file_name: file.name,
file_name: this.uploadDir + file.name,
scene: 'model',
modeluuid: file.modelUuid,
}).then(res => {
@@ -375,7 +385,7 @@ export default {
size: partSize,
chunkNumber: currentChunk + 1,
type: _this.state.type,
file_name: file.name,
file_name: _this.uploadDir + file.name,
scene: 'model',
});
urls[currentChunk] = res.data.url;
@@ -423,7 +433,7 @@ export default {
return await setCompleteMultipart({
uuid: file.uuid,
uploadID: file.uploadID,
file_name: file.name,
file_name: _this.uploadDir + file.name,
size: file.size,
type: _this.state.type,
scene: 'model',
@@ -546,7 +556,7 @@ export default {
this.uploadLength++;
// 同一模型上传同一个文件
if (file._modelUuid) {
const info = `${this.$t('modelManage.fileHasAlreadyInTheModel')} ${file._modelName}`;
const info = `${this.$t('modelManage.fileExistInTheModel')} ${file.realName}`;
this.uploadError(file, info);
this.updateFileStatus(file, this.$t('modelManage.uploadFailed'), 0, 1, info);
} else { // 秒传
@@ -577,10 +587,11 @@ export default {
this.goFileListPage();
},
goFileListPage() {
const list = window.location.href.split('/');
const list = window.location.pathname.split('/');
list.pop();
list.push('model_filelist_tmpl');
window.location.href = list.join('/') + '?name=' + encodeURIComponent(this.state.name);
const path = this.uploadDir ? this.uploadDir.replace(/\/$/, '') : '';
window.location.href = list.join('/') + '?name=' + encodeURIComponent(this.state.name) + (path ? `&path=${path}` : '');
}
},
beforeMount() { },


+ 14
- 3
web_src/vuepages/pages/modelmanage/graph/index.vue View File

@@ -4,7 +4,8 @@
<NotFound></NotFound>
</div>
<div v-else>
<ModelHeader :tab="'graph'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName" :model="modelData" v-if="JSON.stringify(modelData)!=='{}'">
<ModelHeader :tab="'graph'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName"
:model="modelData" v-if="JSON.stringify(modelData) !== '{}'">
</ModelHeader>
<div class="ui container">
<div ref="graphContainerRef" class="graph-container"></div>
@@ -61,8 +62,18 @@ export default {
node.isParent = node.IsParent;
node.isDerive = !node.IsCurrent && !node.IsParent;
node.creator = model.userName;
if (!node.isCurrent) {
node.link = `/${model.repoOwnerName}/${model.repoName}/modelmanage/model_readme_tmpl?name=${model.name}`;
const link = `/${model.repoOwnerName}/${model.repoName}/modelmanage/model_readme_tmpl?name=${model.name}`;
model.link = link;
if (!node.isCurrent && !node.isParent) {
node.link = link;
}
if (node.isParent) {
const models = node.Models4Parent || [];
models.map(item => {
item.link = `/${item.repoOwnerName}/${item.repoName}/modelmanage/model_readme_tmpl?name=${item.name}`;
})
node.name = models.map(itm => itm.name).join(',');
node.link = '';
}
}
if (node.Type == 0) {


+ 11
- 2
web_src/vuepages/pages/modelmanage/graph/model-graph.css View File

@@ -121,6 +121,7 @@
}

._tree-layout ._tree-node.model-parent .name {
cursor: default;
max-height: 100%;
overflow: hidden;
text-overflow: ellipsis;
@@ -156,6 +157,7 @@
-webkit-line-clamp: 5;
}

._tree-layout ._tree-node.model-parent .name:hover,
._tree-layout ._tree-node.model-current .name:hover {
text-decoration: none;
}
@@ -289,6 +291,13 @@
background: rgba(255, 255, 255, 0.96);
padding: 10px;
z-index: 10001;
max-height: 400px;
overflow-y: auto;
}

._model-info .line {
border-bottom: 1px solid #e1e3e6;
margin: 7px 0;
}

._model-info .tit {
@@ -311,7 +320,7 @@
._model-info .row>div:nth-child(2) {
flex: 1;
color: rgb(16, 16, 16);
width: 0;
width: 0;
}

._model-info .row .dataset-c {
@@ -323,4 +332,4 @@

:lang(en-US) ._model-info .row>div:nth-child(1) {
width: 120px;
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save