@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strings"
@@ -27,19 +28,22 @@ const (
MODEL_NOT_LATEST = 0
)
func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, ctx *context.Context) error {
func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) error {
aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName)
if err != nil {
log.Info("query task error." + err.Error())
return err
aiTask, err = models.GetRepoCloudBrainByJobID(ctx.Repo.Repository.ID, jobId)
if err != nil {
log.Info("query task error." + err.Error())
return err
} else {
log.Info("query gpu train task.")
}
}
uuid := uuid.NewV4()
id := uuid.String()
modelPath := id
var lastNewModelId string
var modelSize int64
cloudType := models.TypeCloudBrainTwo
log.Info("find task name:" + aiTask.JobName)
aimodels := models.QueryModelByName(name, aiTask.RepoID)
@@ -53,7 +57,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio
}
}
}
cloudType = aiTask.Type
cloudType : = aiTask.Type
//download model zip //train type
if cloudType == models.TypeCloudBrainTwo {
modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "", aiTask.TrainUrl)
@@ -61,6 +65,12 @@ func saveModelByParameters(jobId string, versionName string, name string, versio
log.Info("download model from CloudBrainTwo faild." + err.Error())
return err
}
} else if cloudType == models.TypeCloudBrainOne {
modelPath, modelSize, err = downloadModelFromCloudBrainOne(id, aiTask.JobName, "", aiTask.TrainUrl)
if err != nil {
log.Info("download model from CloudBrainOne faild." + err.Error())
return err
}
}
accuracy := make(map[string]string)
accuracy["F1"] = ""
@@ -131,7 +141,7 @@ func SaveNewNameModel(ctx *context.Context) {
return
}
SaveModel(ctx)
ctx.Status(200)
log.Info("save model end.")
}
@@ -143,8 +153,9 @@ func SaveModel(ctx *context.Context) {
version := ctx.Query("Version")
label := ctx.Query("Label")
description := ctx.Query("Description")
engine := ctx.QueryInt("Engine")
trainTaskCreate := ctx.QueryBool("trainTaskCreate")
log.Info("engine=" + fmt.Sprint(engine))
if !trainTaskCreate {
if !ctx.Repo.CanWrite(models.UnitTypeModelManage) {
//ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
@@ -163,14 +174,14 @@ func SaveModel(ctx *context.Context) {
return
}
err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx)
err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx)
if err != nil {
log.Info("save model error." + err.Error())
ctx.Error(500, fmt.Sprintf("save model error. %v", err))
return
}
ctx.Status(200)
log.Info("save model end.")
}
@@ -199,6 +210,22 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir
return dataActualPath, size, nil
}
func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) {
modelActualPath := storage.GetMinioPath(jobName, "/model/")
log.Info("modelActualPath=" + modelActualPath)
modelSrcPrefix := setting.CBCodePathPrefix + jobName + "/model/"
destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/"
bucketName := setting.Attachment.Minio.Bucket
log.Info("destKeyNamePrefix=" + destKeyNamePrefix + " modelSrcPrefix=" + modelSrcPrefix + " bucket=" + bucketName)
size, err := storage.MinioPathCopy(bucketName, modelSrcPrefix, destKeyNamePrefix)
if err == nil {
dataActualPath := bucketName + "/" + destKeyNamePrefix
return dataActualPath, size, nil
} else {
return "", 0, nil
}
}
func DeleteModel(ctx *context.Context) {
log.Info("delete model start.")
id := ctx.Query("ID")
@@ -277,51 +304,117 @@ func DownloadMultiModelFile(ctx *context.Context) {
}
path := Model_prefix + models.AttachmentRelativePath(id) + "/"
if task.Type == models.TypeCloudBrainTwo {
downloadFromCloudBrainTwo(path, task, ctx, id)
} else if task.Type == models.TypeCloudBrainOne {
downloadFromCloudBrainOne(path, task, ctx, id)
}
}
allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
func MinioDownloadManyFile(path string, ctx *context.Context, returnFileName string, allFile []storage.FileInfo) {
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(returnFileName))
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
w := zip.NewWriter(ctx.Resp)
defer w.Close()
for _, oneFile := range allFile {
if oneFile.IsDir {
log.Info("zip dir name:" + oneFile.FileName)
} else {
log.Info("zip file name:" + oneFile.FileName)
fDest, err := w.Create(oneFile.FileName)
if err != nil {
log.Info("create zip entry error, download file failed: %s\n", err.Error())
ctx.ServerError("download file failed:", err)
return
}
log.Info("minio file path=" + (path + oneFile.FileName))
body, err := storage.Attachments.DownloadAFile(setting.Attachment.Minio.Bucket, path+oneFile.FileName)
if err != nil {
log.Info("download file failed: %s\n", err.Error())
ctx.ServerError("download file failed:", err)
return
} else {
defer body.Close()
p := make([]byte, 1024)
var readErr error
var readCount int
// 读取对象内容
for {
readCount, readErr = body.Read(p)
if readCount > 0 {
fDest.Write(p[:readCount])
}
if readErr != nil {
break
}
}
}
}
}
}
func downloadFromCloudBrainOne(path string, task *models.AiModelManage, ctx *context.Context, id string) {
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, path)
if err == nil {
//count++
models.ModifyModelDownloadCount(id)
returnFileName := task.Name + "_" + task.Version + ".zip"
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName)
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
w := zip.NewWriter(ctx.Resp)
defer w.Close()
for _, oneFile := range allFile {
if oneFile.IsDir {
log.Info("zip dir name:" + oneFile.FileName)
MinioDownloadManyFile(path, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
}
func ObsDownloadManyFile(path string, ctx *context.Context, returnFileName string, allFile []storage.FileInfo) {
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(returnFileName))
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
w := zip.NewWriter(ctx.Resp)
defer w.Close()
for _, oneFile := range allFile {
if oneFile.IsDir {
log.Info("zip dir name:" + oneFile.FileName)
} else {
log.Info("zip file name:" + oneFile.FileName)
fDest, err := w.Create(oneFile.FileName)
if err != nil {
log.Info("create zip entry error, download file failed: %s\n", err.Error())
ctx.ServerError("download file failed:", err)
return
}
body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName)
if err != nil {
log.Info("download file failed: %s\n", err.Error())
ctx.ServerError("download file failed:", err)
return
} else {
log.Info("zip file name:" + oneFile.FileName)
fDest, err := w.Create(oneFile.FileName)
if err != nil {
log.Info("create zip entry error, download file failed: %s\n", err.Error())
ctx.ServerError("download file failed:", err)
return
}
body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName)
if err != nil {
log.Info("download file failed: %s\n", err.Error())
ctx.ServerError("download file failed:", err)
return
} else {
defer body.Close()
p := make([]byte, 1024)
var readErr error
var readCount int
// 读取对象内容
for {
readCount, readErr = body.Read(p)
if readCount > 0 {
fDest.Write(p[:readCount])
}
if readErr != nil {
break
}
defer body.Close()
p := make([]byte, 1024)
var readErr error
var readCount int
// 读取对象内容
for {
readCount, readErr = body.Read(p)
if readCount > 0 {
fDest.Write(p[:readCount])
}
if readErr != nil {
break
}
}
}
}
}
}
func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *context.Context, id string) {
allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
if err == nil {
//count++
models.ModifyModelDownloadCount(id)
returnFileName := task.Name + "_" + task.Version + ".zip"
ObsDownloadManyFile(path, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
@@ -374,42 +467,55 @@ func DownloadSingleModelFile(ctx *context.Context) {
ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
return
}
if setting.PROXYURL != "" {
body, err := storage.ObsDownloadAFile(setting.Bucket, path)
if err != nil {
log.Info("download error.")
if task.Type == models.TypeCloudBrainTwo {
if setting.PROXYURL != "" {
body, err := storage.ObsDownloadAFile(setting.Bucket, path)
if err != nil {
log.Info("download error.")
} else {
//count++
models.ModifyModelDownloadCount(id)
defer body.Close()
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
p := make([]byte, 1024)
var readErr error
var readCount int
// 读取对象内容
for {
readCount, readErr = body.Read(p)
if readCount > 0 {
ctx.Resp.Write(p[:readCount])
//fmt.Printf("%s", p[:readCount])
}
if readErr != nil {
break
}
}
}
} else {
url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path)
if err != nil {
log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("GetObsCreateSignedUrl", err)
return
}
//count++
models.ModifyModelDownloadCount(id)
defer body.Close()
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
p := make([]byte, 1024)
var readErr error
var readCount int
// 读取对象内容
for {
readCount, readErr = body.Read(p)
if readCount > 0 {
ctx.Resp.Write(p[:readCount])
//fmt.Printf("%s", p[:readCount])
}
if readErr != nil {
break
}
}
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
}
} else {
url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path)
} else if task.Type == models.TypeCloudBrainOne {
log.Info("start to down load minio file.")
url, err := storage.Attachments.PresignedGetURL(path, fileName)
if err != nil {
log.Error("GetObsCreate SignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("GetObsCreateSignedUrl ", err)
log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("Get minio get SignedUrl failed", err)
return
}
//count++
models.ModifyModelDownloadCount(id)
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
}
}
func ShowModelInfo(ctx *context.Context) {
@@ -684,14 +790,22 @@ func QueryModelListForPredict(ctx *context.Context) {
func QueryModelFileForPredict(ctx *context.Context) {
id := ctx.Query("ID")
model, err := models.QueryModelById(id)
if err != nil {
if err == nil {
if model.Type == models.TypeCloudBrainTwo {
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, _ := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix)
ctx.JSON(http.StatusOK, fileinfos)
} else if model.Type == models.TypeCloudBrainOne {
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:]
fileinfos, _ := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, prefix)
ctx.JSON(http.StatusOK, fileinfos)
}
} else {
log.Error("no such model!", err.Error())
ctx.ServerError("no such model:", err)
return
}
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix)
ctx.JSON(http.StatusOK, fileinfos)
}
func QueryOneLevelModelFile(ctx *context.Context) {
@@ -703,7 +817,16 @@ func QueryOneLevelModelFile(ctx *context.Context) {
ctx.ServerError("no such model:", err)
return
}
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir)
ctx.JSON(http.StatusOK, fileinfos)
if model.Type == models.TypeCloudBrainTwo {
log.Info("TypeCloudBrainTwo list model file.")
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, _ := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir)
ctx.JSON(http.StatusOK, fileinfos)
} else if model.Type == models.TypeCloudBrainOne {
log.Info("TypeCloudBrainOne list model file.")
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:]
fileinfos, _ := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, parentDir)
ctx.JSON(http.StatusOK, fileinfos)
}
}