|
- package repo
-
- import (
- "bufio"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "os"
- "path"
- "strings"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/cloudbrain"
- "code.gitea.io/gitea/modules/context"
- "code.gitea.io/gitea/modules/git"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/modelarts"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
- "code.gitea.io/gitea/modules/timeutil"
- uuid "github.com/satori/go.uuid"
- )
-
- const (
- tplModelManageConvertIndex = "repo/modelmanage/convertIndex"
- tplModelConvertInfo = "repo/modelmanage/convertshowinfo"
- PYTORCH_ENGINE = 0
- TENSORFLOW_ENGINE = 1
- MINDSPORE_ENGINE = 2
- PADDLE_ENGINE = 4
- MXNET_ENGINE = 6
- ModelMountPath = "/model"
- CodeMountPath = "/code"
- DataSetMountPath = "/dataset"
- LogFile = "log.txt"
- DefaultBranchName = "master"
- SubTaskName = "task1"
- //GpuQueue = "openidgx"
- Success = "S000"
- //GPU_PYTORCH_IMAGE = "dockerhub.pcl.ac.cn:5000/user-images/openi:tensorRT_7_zouap"
- //GPU_TENSORFLOW_IMAGE = "dockerhub.pcl.ac.cn:5000/user-images/openi:tf2onnx"
- //NPU_MINDSPORE_16_IMAGE = "swr.cn-south-222.ai.pcl.cn/openi/mindspore1.6.1_train_v1_openi:v3_ascend"
- //PytorchOnnxBootFile = "convert_pytorch.py"
- //PytorchTrTBootFile = "convert_pytorch_tensorrt.py"
- //MindsporeBootFile = "convert_mindspore.py"
- //TensorFlowNpuBootFile = "convert_tensorflow.py"
- //TensorFlowGpuBootFile = "convert_tensorflow_gpu.py"
-
- //ConvertRepoPath = "https://openi.pcl.ac.cn/zouap/npu_test"
-
- CONVERT_FORMAT_ONNX = 0
- CONVERT_FORMAT_TRT = 1
-
- NetOutputFormat_FP32 = 0
- NetOutputFormat_FP16 = 1
-
- NPU_MINDSPORE_IMAGE_ID = 35
- NPU_TENSORFLOW_IMAGE_ID = 121
-
- //GPU_Resource_Specs_ID = 1 //cpu 1, gpu 1
-
- //NPU_FlavorCode = "modelarts.bm.910.arm.public.1"
- //NPU_PoolID = "pool7908321a"
- )
-
- var (
- TrainResourceSpecs *models.ResourceSpecs
- )
-
- func SaveModelConvert(ctx *context.Context) {
- log.Info("save model convert start.")
- if !ctx.Repo.CanWrite(models.UnitTypeModelManage) {
- ctx.JSON(200, map[string]string{
- "code": "1",
- "msg": ctx.Tr("repo.modelconvert.manage.no_operate_right"),
- })
- return
- }
- name := ctx.Query("name")
- desc := ctx.Query("desc")
- modelId := ctx.Query("modelId")
- modelPath := ctx.Query("modelFile")
- SrcEngine := ctx.QueryInt("srcEngine")
- InputShape := ctx.Query("inputshape")
- InputDataFormat := ctx.Query("inputdataformat")
- DestFormat := ctx.QueryInt("destFormat")
- NetOutputFormat := ctx.QueryInt("netOutputFormat")
-
- task, err := models.QueryModelById(modelId)
- if err != nil {
- log.Error("no such model!", err.Error())
- ctx.JSON(200, map[string]string{
- "code": "1",
- "msg": ctx.Tr("repo.modelconvert.manage.model_not_exist"),
- })
- return
- }
-
- convertList, err := models.QueryModelConvertByRepoID(ctx.Repo.Repository.ID)
- if err == nil {
- for _, convert := range convertList {
- if convert.Name == name {
- log.Info("convert.Name=" + name + " convert.id=" + convert.ID)
- ctx.JSON(200, map[string]string{
- "code": "1",
- "msg": ctx.Tr("repo.modelconvert.manage.create_error1"),
- })
- return
- }
- }
- }
-
- convertList, err = models.QueryModelConvertByUserID(ctx.User.ID)
- if err == nil {
- for _, convert := range convertList {
- if isRunningTask(convert.Status) {
- log.Info("convert.Status=" + convert.Status + " convert.id=" + convert.ID)
- ctx.JSON(200, map[string]string{
- "code": "1",
- "msg": ctx.Tr("repo.modelconvert.manage.create_error2"),
- })
- return
- }
- }
- }
-
- uuid := uuid.NewV4()
- id := uuid.String()
- modelConvert := &models.AiModelConvert{
- ID: id,
- Name: name,
- Description: desc,
- Status: string(models.JobWaiting),
- SrcEngine: SrcEngine,
- RepoId: ctx.Repo.Repository.ID,
- ModelName: task.Name,
- ModelVersion: task.Version,
- ModelId: modelId,
- ModelPath: modelPath,
- DestFormat: DestFormat,
- NetOutputFormat: NetOutputFormat,
- InputShape: InputShape,
- InputDataFormat: InputDataFormat,
- UserId: ctx.User.ID,
- }
- models.SaveModelConvert(modelConvert)
- go goCreateTask(modelConvert, ctx, task)
-
- ctx.JSON(200, map[string]string{
- "id": id,
- "code": "0",
- })
- }
-
- func isRunningTask(status string) bool {
- stopStatus := []string{"COMPLETED", "STOPPED", "FAILED", "START_FAILED", "STOPPING", "SUCCEEDED"}
- for _, sta := range stopStatus {
- if sta == status {
- return false
- }
- }
- return true
- }
-
- func goCreateTask(modelConvert *models.AiModelConvert, ctx *context.Context, task *models.AiModelManage) error {
- if modelConvert.IsGpuTrainTask() {
- log.Info("create gpu train job.")
- return createGpuTrainJob(modelConvert, ctx, task)
- } else {
- //create npu job
- log.Info("create npu train job.")
- return createNpuTrainJob(modelConvert, ctx, task.Path)
- }
- }
-
- func createNpuTrainJob(modelConvert *models.AiModelConvert, ctx *context.Context, modelRelativePath string) error {
- VersionOutputPath := "V0001"
- codeLocalPath := setting.JobPath + modelConvert.ID + modelarts.CodePath
- codeObsPath := "/" + setting.Bucket + modelarts.JobPath + modelConvert.ID + modelarts.CodePath
- outputObsPath := "/" + setting.Bucket + modelarts.JobPath + modelConvert.ID + modelarts.OutputPath + VersionOutputPath + "/"
- logObsPath := "/" + setting.Bucket + modelarts.JobPath + modelConvert.ID + modelarts.LogPath + VersionOutputPath + "/"
- dataPath := "/" + modelRelativePath
-
- _, err := ioutil.ReadDir(codeLocalPath)
- if err == nil {
- deleteLocalDir(codeLocalPath)
- }
- if err := downloadConvertCode(setting.ModelConvert.ConvertRepoPath, codeLocalPath, DefaultBranchName); err != nil {
- log.Error("downloadCode failed, server timed out: %s (%v)", setting.ModelConvert.ConvertRepoPath, err)
- return err
- }
- if err := obsMkdir(setting.CodePathPrefix + modelConvert.ID + modelarts.OutputPath + VersionOutputPath + "/"); err != nil {
- log.Error("Failed to obsMkdir_output: %s (%v)", modelConvert.ID+modelarts.OutputPath, err)
- return err
- }
- if err := obsMkdir(setting.CodePathPrefix + modelConvert.ID + modelarts.LogPath + VersionOutputPath + "/"); err != nil {
- log.Error("Failed to obsMkdir_log: %s (%v)", modelConvert.ID+modelarts.LogPath, err)
- return err
- }
- if err := uploadCodeToObs(codeLocalPath, modelConvert.ID, ""); err != nil {
- log.Error("Failed to uploadCodeToObs: %s (%v)", modelConvert.ID, err)
- return err
- }
- deleteLocalDir(codeLocalPath)
-
- intputshape := strings.Split(modelConvert.InputShape, ",")
- n := "256"
- c := "1"
- h := "28"
- w := "28"
- if len(intputshape) == 4 {
- n = intputshape[0]
- c = intputshape[1]
- h = intputshape[2]
- w = intputshape[3]
- }
-
- var engineId int64
- engineId = int64(NPU_MINDSPORE_IMAGE_ID)
- bootfile := setting.ModelConvert.MindsporeBootFile
- if modelConvert.SrcEngine == TENSORFLOW_ENGINE {
- engineId = int64(NPU_TENSORFLOW_IMAGE_ID)
- bootfile = setting.ModelConvert.TensorFlowNpuBootFile
- }
- userCommand := "/bin/bash /home/work/run_train.sh 's3://" + codeObsPath + "' 'code/" + bootfile + "' '/tmp/log/train.log' --'data_url'='s3://" + dataPath + "' --'train_url'='s3://" + outputObsPath + "'"
- userCommand += " --'model'='" + modelConvert.ModelPath + "'"
- userCommand += " --'n'='" + fmt.Sprint(n) + "'"
- userCommand += " --'c'='" + fmt.Sprint(c) + "'"
- userCommand += " --'h'='" + fmt.Sprint(h) + "'"
- userCommand += " --'w'='" + fmt.Sprint(w) + "'"
-
- req := &modelarts.GenerateTrainJobReq{
- JobName: modelConvert.ID,
- DisplayJobName: modelConvert.Name,
- DataUrl: dataPath,
- Description: modelConvert.Description,
- CodeObsPath: codeObsPath,
- BootFileUrl: codeObsPath + bootfile,
- BootFile: bootfile,
- TrainUrl: outputObsPath,
- FlavorCode: setting.ModelConvert.NPU_FlavorCode,
- WorkServerNumber: 1,
- IsLatestVersion: modelarts.IsLatestVersion,
- EngineID: engineId,
- LogUrl: logObsPath,
- PoolID: setting.ModelConvert.NPU_PoolID,
- //Parameters: param,
- BranchName: DefaultBranchName,
- UserImageUrl: setting.ModelConvert.NPU_MINDSPORE_16_IMAGE,
- UserCommand: userCommand,
- }
- result, err := modelarts.GenerateModelConvertTrainJob(req)
- if err == nil {
- log.Info("jobId=" + fmt.Sprint(result.JobID) + " versionid=" + fmt.Sprint(result.VersionID))
- models.UpdateModelConvertModelArts(modelConvert.ID, fmt.Sprint(result.JobID), fmt.Sprint(result.VersionID))
- } else {
- log.Info("create modelarts taks failed.error=" + err.Error())
- models.UpdateModelConvertFailed(modelConvert.ID, "FAILED", err.Error())
- }
- return err
- }
-
- func downloadConvertCode(repopath string, codePath, branchName string) error {
- //add "file:///" prefix to make the depth valid
- if err := git.Clone(repopath, codePath, git.CloneRepoOptions{Branch: branchName, Depth: 1}); err != nil {
- log.Error("Failed to clone repository: %s (%v)", repopath, err)
- return err
- }
- log.Info("srcPath=" + repopath + " codePath=" + codePath)
- configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
- if err != nil {
- log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
- return err
- }
-
- defer configFile.Close()
-
- pos := int64(0)
- reader := bufio.NewReader(configFile)
- for {
- line, err := reader.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- log.Error("not find the remote-url")
- return nil
- } else {
- log.Error("read error: %v", err)
- return err
- }
- }
-
- if strings.Contains(line, "url") && strings.Contains(line, ".git") {
- originUrl := "\turl = " + repopath + "\n"
- if len(line) > len(originUrl) {
- originUrl += strings.Repeat(" ", len(line)-len(originUrl))
- }
- bytes := []byte(originUrl)
- _, err := configFile.WriteAt(bytes, pos)
- if err != nil {
- log.Error("WriteAt failed:%v", err)
- return err
- }
- break
- }
-
- pos += int64(len(line))
- }
-
- return nil
- }
-
- func downloadFromObsToLocal(task *models.AiModelManage, localPath string) error {
- path := Model_prefix + models.AttachmentRelativePath(task.ID) + "/"
- allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
- if err == nil {
- _, errState := os.Stat(localPath)
- if errState != nil {
- if err = os.MkdirAll(localPath, os.ModePerm); err != nil {
- return err
- }
- }
- for _, oneFile := range allFile {
- if oneFile.IsDir {
- log.Info(" dir name:" + oneFile.FileName)
- } else {
- allFileName := localPath + "/" + oneFile.FileName
- index := strings.LastIndex(allFileName, "/")
- if index != -1 {
- parentDir := allFileName[0:index]
- if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
- log.Info("make dir may be error," + err.Error())
- }
- }
- fDest, err := os.Create(allFileName)
- if err != nil {
- log.Info("create file error, download file failed: %s\n", err.Error())
- return err
- }
- body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName)
- if err != nil {
- log.Info("download file failed: %s\n", err.Error())
- return err
- } else {
- defer body.Close()
- p := make([]byte, 1024)
- var readErr error
- var readCount int
- // 读取对象内容
- for {
- readCount, readErr = body.Read(p)
- if readCount > 0 {
- fDest.Write(p[:readCount])
- }
- if readErr != nil {
- break
- }
- }
- }
- }
- }
- } else {
- log.Info("error,msg=" + err.Error())
- return err
- }
- return nil
- }
-
- func createGpuTrainJob(modelConvert *models.AiModelConvert, ctx *context.Context, model *models.AiModelManage) error {
- modelRelativePath := model.Path
- command := ""
- IMAGE_URL := setting.ModelConvert.GPU_PYTORCH_IMAGE
- dataActualPath := setting.Attachment.Minio.RealPath + modelRelativePath
-
- if modelConvert.SrcEngine == PYTORCH_ENGINE {
- if modelConvert.DestFormat == CONVERT_FORMAT_ONNX {
- command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.PytorchOnnxBootFile)
- } else if modelConvert.DestFormat == CONVERT_FORMAT_TRT {
- command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.PytorchTrTBootFile)
- } else {
- return errors.New("Not support the format.")
- }
- } else if modelConvert.SrcEngine == TENSORFLOW_ENGINE {
- IMAGE_URL = setting.ModelConvert.GPU_TENSORFLOW_IMAGE
- if modelConvert.DestFormat == CONVERT_FORMAT_ONNX {
- command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.TensorFlowGpuBootFile)
- } else {
- return errors.New("Not support the format.")
- }
- //如果模型在OBS上,需要下载到本地,并上传到minio中
- if model.Type == models.TypeCloudBrainTwo {
- relatetiveModelPath := setting.JobPath + modelConvert.ID + "/dataset"
- log.Info("local dataset path:" + relatetiveModelPath)
- downloadFromObsToLocal(model, relatetiveModelPath)
- uploadCodeToMinio(relatetiveModelPath+"/", modelConvert.ID, "/dataset/")
- deleteLocalDir(relatetiveModelPath)
- dataActualPath = setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + modelConvert.ID + "/dataset"
- }
- } else if modelConvert.SrcEngine == PADDLE_ENGINE {
- IMAGE_URL = setting.ModelConvert.GPU_PADDLE_IMAGE
- if modelConvert.DestFormat == CONVERT_FORMAT_ONNX {
- command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.PaddleOnnxBootFile)
- } else {
- return errors.New("Not support the format.")
- }
- } else if modelConvert.SrcEngine == MXNET_ENGINE {
- IMAGE_URL = setting.ModelConvert.GPU_MXNET_IMAGE
- if modelConvert.DestFormat == CONVERT_FORMAT_ONNX {
- command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.MXnetOnnxBootFile)
- } else {
- return errors.New("Not support the format.")
- }
- }
- log.Info("dataActualPath=" + dataActualPath)
-
- log.Info("command=" + command)
-
- codePath := setting.JobPath + modelConvert.ID + CodeMountPath
- downloadConvertCode(setting.ModelConvert.ConvertRepoPath, codePath, DefaultBranchName)
-
- uploadCodeToMinio(codePath+"/", modelConvert.ID, CodeMountPath+"/")
- deleteLocalDir(codePath)
-
- minioCodePath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + modelConvert.ID + "/code"
- log.Info("minio codePath=" + minioCodePath)
-
- modelPath := setting.JobPath + modelConvert.ID + ModelMountPath + "/"
- log.Info("local modelPath=" + modelPath)
- mkModelPath(modelPath)
-
- uploadCodeToMinio(modelPath, modelConvert.ID, ModelMountPath+"/")
- deleteLocalDir(modelPath)
-
- minioModelPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + modelConvert.ID + "/model"
- log.Info("minio model path=" + minioModelPath)
-
- if TrainResourceSpecs == nil {
- json.Unmarshal([]byte(setting.TrainResourceSpecs), &TrainResourceSpecs)
- }
- resourceSpec := TrainResourceSpecs.ResourceSpec[setting.ModelConvert.GPU_Resource_Specs_ID]
- jobResult, err := cloudbrain.CreateJob(modelConvert.ID, models.CreateJobParams{
- JobName: modelConvert.ID,
- RetryCount: 1,
- GpuType: setting.ModelConvert.GpuQueue,
- Image: IMAGE_URL,
- TaskRoles: []models.TaskRole{
- {
- Name: SubTaskName,
- TaskNumber: 1,
- MinSucceededTaskCount: 1,
- MinFailedTaskCount: 1,
- CPUNumber: resourceSpec.CpuNum,
- GPUNumber: resourceSpec.GpuNum,
- MemoryMB: resourceSpec.MemMiB,
- ShmMB: resourceSpec.ShareMemMiB,
- Command: command,
- NeedIBDevice: false,
- IsMainRole: false,
- UseNNI: false,
- },
- },
- Volumes: []models.Volume{
- {
- HostPath: models.StHostPath{
- Path: minioCodePath,
- MountPath: CodeMountPath,
- ReadOnly: false,
- },
- },
- {
- HostPath: models.StHostPath{
- Path: dataActualPath,
- MountPath: DataSetMountPath,
- ReadOnly: true,
- },
- },
- {
- HostPath: models.StHostPath{
- Path: minioModelPath,
- MountPath: ModelMountPath,
- ReadOnly: false,
- },
- },
- },
- })
- if err != nil {
- log.Error("CreateJob failed:", err.Error(), ctx.Data["MsgID"])
- models.UpdateModelConvertFailed(modelConvert.ID, "FAILED", err.Error())
- return err
- }
- if jobResult.Code != Success {
- log.Error("CreateJob(%s) failed:%s", modelConvert.ID, jobResult.Msg, ctx.Data["MsgID"])
- models.UpdateModelConvertFailed(modelConvert.ID, "FAILED", err.Error())
- return errors.New(jobResult.Msg)
- }
-
- var jobID = jobResult.Payload["jobId"].(string)
- log.Info("jobId=" + jobID)
- models.UpdateModelConvertCBTI(modelConvert.ID, jobID)
-
- return nil
- }
-
- func deleteLocalDir(dirpath string) {
- //TODO delete
- _err := os.RemoveAll(dirpath)
- if _err == nil {
- log.Info("Delete local file:" + dirpath)
- } else {
- log.Info("Delete local file error: path=" + dirpath)
- }
- }
-
- func getGpuModelConvertCommand(name string, modelFile string, modelConvert *models.AiModelConvert, bootfile string) string {
- var command string
-
- inputshape := strings.Split(modelConvert.InputShape, ",")
- n := "256"
- c := "1"
- h := "28"
- w := "28"
- if len(inputshape) == 4 {
- n = inputshape[0]
- c = inputshape[1]
- h = inputshape[2]
- w = inputshape[3]
- }
- command += "python3 /code/" + bootfile + " --model " + modelFile + " --n " + n + " --c " + c + " --h " + h + " --w " + w
- if modelConvert.DestFormat == CONVERT_FORMAT_TRT {
- if modelConvert.NetOutputFormat == NetOutputFormat_FP16 {
- command += " --fp16 True"
- } else {
- command += " --fp16 False"
- }
- }
- command += " > " + ModelMountPath + "/" + name + "-" + LogFile
- return command
- }
-
- func DeleteModelConvert(ctx *context.Context) {
- log.Info("delete model convert start.")
- id := ctx.Params(":id")
- task, err := models.QueryModelConvertById(id)
- if err == nil {
- go deleteCloudBrainTask(task)
- }
- err = models.DeleteModelConvertById(id)
- //TODO delete OBS文件及云脑任务
- if err != nil {
- ctx.JSON(500, err.Error())
- } else {
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelmanage/convert_model")
- }
- }
-
- func deleteCloudBrainTask(task *models.AiModelConvert) {
- if task.IsGpuTrainTask() {
- log.Info("delete cloudbrain one resource.")
- dirPath := setting.CBCodePathPrefix + task.ID + "/"
- err := storage.Attachments.DeleteDir(dirPath)
- if err != nil {
- log.Error("DeleteDir(%s) failed:%v", dirPath, err)
- }
- } else {
- log.Info("delete cloudbrain two resource.")
- _, err := modelarts.DelTrainJob(task.CloudBrainTaskId)
- if err != nil {
- log.Error("DelTrainJob(%s) failed:%v", task.CloudBrainTaskId, err.Error())
- }
- DeleteJobStorage(task.ID)
- }
- }
-
- func stopModelConvert(id string) error {
- job, err := models.QueryModelConvertById(id)
- if err != nil {
- return err
- }
- if job.IsGpuTrainTask() {
- err = cloudbrain.StopJob(job.CloudBrainTaskId)
- if err != nil {
- log.Error("Stop cloudbrain Job(%s) failed:%v", job.CloudBrainTaskId, err)
- }
- } else {
- _, err = modelarts.StopTrainJob(job.CloudBrainTaskId, job.ModelArtsVersionId)
- if err != nil {
- log.Error("Stop modelarts Job(%s) failed:%v", job.CloudBrainTaskId, err)
- }
- }
- job.Status = string(models.JobStopped)
- if job.EndTime == 0 {
- job.EndTime = timeutil.TimeStampNow()
- }
- models.ModelConvertSetDuration(job)
- err = models.UpdateModelConvert(job)
- if err != nil {
- log.Error("UpdateModelConvert failed:", err)
- return err
- }
- return nil
- }
-
- func StopModelConvertApi(ctx *context.Context) {
- id := ctx.Query("id")
- log.Info("stop model convert start.id=" + id)
- err := stopModelConvert(id)
- if err == nil {
- ctx.JSON(200, map[string]string{
- "code": "0",
- "msg": "succeed",
- })
- } else {
- ctx.JSON(200, map[string]string{
- "code": "1",
- "msg": err.Error(),
- })
- }
- }
-
- func StopModelConvert(ctx *context.Context) {
- id := ctx.Params(":id")
- log.Info("stop model convert start.id=" + id)
- err := stopModelConvert(id)
- if err != nil {
- ctx.ServerError("Not found task.", err)
- return
- }
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelmanage/convert_model")
- }
-
- func ShowModelConvertInfo(ctx *context.Context) {
- ctx.Data["ID"] = ctx.Query("id")
- ctx.Data["isModelManage"] = true
- ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage)
-
- job, err := models.QueryModelConvertById(ctx.Query("id"))
- if err == nil {
- if job.TrainJobDuration == "" {
- job.TrainJobDuration = "00:00:00"
- }
- ctx.Data["task"] = job
- } else {
- ctx.ServerError("Not found task.", err)
- return
- }
- ctx.Data["Name"] = job.Name
- ctx.Data["canDownload"] = isOperModifyOrDelete(ctx, job.UserId)
- user, err := models.GetUserByID(job.UserId)
- if err == nil {
- job.UserName = user.Name
- job.UserRelAvatarLink = user.RelAvatarLink()
- }
-
- if job.IsGpuTrainTask() {
- ctx.Data["npu_display"] = "none"
- ctx.Data["gpu_display"] = "block"
- if job.CloudBrainTaskId == "" {
- ctx.Data["ExitDiagnostics"] = ""
- ctx.Data["AppExitDiagnostics"] = ""
- ctx.HTML(200, tplModelConvertInfo)
- return
- }
- result, err := cloudbrain.GetJob(job.CloudBrainTaskId)
- if err != nil {
- log.Info("error:" + err.Error())
- ctx.Data["error"] = err.Error()
- ctx.HTML(200, tplModelConvertInfo)
- return
- }
- if result != nil {
- jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
- ctx.Data["result"] = jobRes
- taskRoles := jobRes.TaskRoles
- taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
- ctx.Data["taskRes"] = taskRes
- ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics
- ctx.Data["AppExitDiagnostics"] = jobRes.JobStatus.AppExitDiagnostics
-
- job.Status = jobRes.JobStatus.State
-
- if jobRes.JobStatus.State != string(models.JobWaiting) && jobRes.JobStatus.State != string(models.JobFailed) {
- job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
- job.ContainerID = taskRes.TaskStatuses[0].ContainerID
- job.Status = taskRes.TaskStatuses[0].State
- }
- if jobRes.JobStatus.State != string(models.JobWaiting) {
- models.ModelComputeAndSetDuration(job, jobRes)
- err = models.UpdateModelConvert(job)
- if err != nil {
- log.Error("UpdateModelConvert failed:", err)
- }
- }
- }
- } else {
- if job.CloudBrainTaskId != "" {
- result, err := modelarts.GetTrainJob(job.CloudBrainTaskId, job.ModelArtsVersionId)
- if err != nil {
- log.Info("error:" + err.Error())
- ctx.Data["error"] = err.Error()
- return
- }
- job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
- job.RunTime = result.Duration / 1000
- job.TrainJobDuration = models.ConvertDurationToStr(job.RunTime)
- err = models.UpdateModelConvert(job)
- if err != nil {
- log.Error("UpdateJob failed:", err)
- }
- }
- ctx.Data["npu_display"] = "block"
- ctx.Data["gpu_display"] = "none"
- ctx.Data["ExitDiagnostics"] = ""
- ctx.Data["AppExitDiagnostics"] = ""
- }
-
- ctx.HTML(200, tplModelConvertInfo)
- }
-
- func ConvertModelTemplate(ctx *context.Context) {
- ctx.Data["isModelManage"] = true
- ctx.Data["TRAIN_COUNT"] = 0
- SetModelCount(ctx)
- ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage)
- ShowModelConvertPageInfo(ctx)
- ctx.HTML(200, tplModelManageConvertIndex)
- }
-
- func ShowModelConvertPageInfo(ctx *context.Context) {
- log.Info("ShowModelConvertInfo start.")
- if !isQueryRight(ctx) {
- log.Info("no right.")
- ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
- return
- }
- page := ctx.QueryInt("page")
- if page <= 0 {
- page = 1
- }
- pageSize := ctx.QueryInt("pageSize")
- if pageSize <= 0 {
- pageSize = setting.UI.IssuePagingNum
- }
- modelResult, count, err := GetModelConvertPageData(ctx)
- if err == nil {
- pager := context.NewPagination(int(count), page, pageSize, 5)
- ctx.Data["Page"] = pager
- ctx.Data["Tasks"] = modelResult
- ctx.Data["MODEL_CONVERT_COUNT"] = count
- } else {
- ctx.ServerError("Query data error.", err)
- }
- }
-
- func GetModelConvertById(ctx *context.Context) (*models.AiModelConvert, error) {
- id := ctx.Query("id")
- return models.QueryModelConvertById(id)
- }
-
- func GetModelConvertByName(ctx *context.Context) ([]*models.AiModelConvert, error) {
- name := ctx.Query("name")
- return models.QueryModelConvertByName(name, ctx.Repo.Repository.ID)
- }
-
- func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, int64, error) {
- page := ctx.QueryInt("page")
- if page <= 0 {
- page = 1
- }
- pageSize := ctx.QueryInt("pageSize")
- if pageSize <= 0 {
- pageSize = setting.UI.IssuePagingNum
- }
- repoId := ctx.Repo.Repository.ID
- modelResult, count, err := models.QueryModelConvert(&models.AiModelQueryOptions{
- ListOptions: models.ListOptions{
- Page: page,
- PageSize: pageSize,
- },
- RepoID: repoId,
- })
- if err != nil {
- log.Info("query db error." + err.Error())
- return nil, 0, err
- }
- userIds := make([]int64, len(modelResult))
- for i, model := range modelResult {
- model.IsCanOper = isOperModifyOrDelete(ctx, model.UserId)
- model.IsCanDelete = isCanDelete(ctx, model.UserId)
- userIds[i] = model.UserId
- }
- userNameMap := queryUserName(userIds)
- for _, model := range modelResult {
- value := userNameMap[model.UserId]
- if value != nil {
- model.UserName = value.Name
- model.UserRelAvatarLink = value.RelAvatarLink()
- }
- }
- return modelResult, count, nil
- }
-
- func ModelConvertDownloadModel(ctx *context.Context) {
- log.Info("enter here......")
- id := ctx.Params(":id")
- job, err := models.QueryModelConvertById(id)
- if err != nil {
- ctx.ServerError("Not found task.", err)
- return
- }
- AllDownload := ctx.QueryBool("allDownload")
- if AllDownload {
- if job.IsGpuTrainTask() {
- path := setting.CBCodePathPrefix + job.ID + "/model/"
- allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, path)
- if err == nil {
- returnFileName := job.Name + ".zip"
- MinioDownloadManyFile(path, ctx, returnFileName, allFile)
- } else {
- log.Info("error,msg=" + err.Error())
- ctx.ServerError("no file to download.", err)
- }
- } else {
- Prefix := path.Join(setting.TrainJobModelPath, job.ID, "output/", "V0001", "") + "/"
- log.Info("bucket=" + setting.Bucket + "prefix=" + Prefix)
- allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, Prefix)
- if err == nil {
- returnFileName := job.Name + ".zip"
- ObsDownloadManyFile(Prefix, ctx, returnFileName, allFile)
- } else {
- log.Info("error,msg=" + err.Error())
- ctx.ServerError("no file to download.", err)
- }
- }
- } else {
- parentDir := ctx.Query("parentDir")
- fileName := ctx.Query("fileName")
- jobName := ctx.Query("jobName")
- if job.IsGpuTrainTask() {
- filePath := "jobs/" + jobName + "/model/" + parentDir
- url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
- if err != nil {
- log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
- ctx.ServerError("PresignedGetURL", err)
- return
- }
- //ctx.JSON(200, url)
- http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
- } else {
- ObjectKey := path.Join(setting.TrainJobModelPath, job.ID, "output/", "V0001", parentDir, fileName)
- log.Info("ObjectKey=" + ObjectKey)
- url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, ObjectKey)
- if err != nil {
- log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
- ctx.ServerError("GetObsCreateSignedUrl", err)
- return
- }
- http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
- }
- }
- }
|