@@ -278,7 +278,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong")))
return
}
if len(getBootFile(option.File, option.OwnerName, option.ProjectName)) > CharacterLength {
if len(getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName )) > CharacterLength {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long")))
return
}
@@ -287,6 +287,16 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
return
}
var imageIdNpu string
var err error
if option.Type == NPUType {
imageIdNpu, err = getNpuImageId(option)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.parameter_is_wrong")))
return
}
}
isNotebookFileExist, _ := isNoteBookFileExist(ctx, option)
if !isNotebookFileExist {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist")))
@@ -335,14 +345,9 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type))
if noteBook != nil {
if isRepoConfilcts(option, noteBook) {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict")))
return
}
if isNotebookSpecMath(option, noteBook) {
if !isRepoMatch(option, noteBook) {
err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName)
if isNotebookSpecMath(option, noteBook) && isImageMatch(option.Image, noteBook) {
if !isRepoBranchMatch(option, noteBook) {
err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo, option.BranchName), option.BranchName)
if err != nil {
log.Error("download code failed", err)
if !strings.Contains(err.Error(), "already exists and is not an empty directory") {
@@ -352,8 +357,8 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
}
}
if !isRepoFileMatch(option, noteBook) {
if len(noteBook.BootFile)+len(getBootFile(option.File, option.OwnerName, option.ProjectName))+1 <= CharacterLength {
noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName)
if len(noteBook.BootFile)+len(getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName ))+1 <= CharacterLength {
noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName )
} else {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long")))
return
@@ -390,9 +395,14 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
if option.Type <= GPUType {
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo)
} else {
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo)
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo, imageIdNpu )
}
}
func isImageMatch(npuImage string, book *models.Cloudbrain) bool {
return (!book.IsNPUTask()) || book.Image == npuImage
}
func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) {
if ctx.Written() {
@@ -431,7 +441,7 @@ func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOp
return
}
if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) {
if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName ), task, token) {
ctx.JSON(http.StatusOK, models.BaseOKMessageApi)
} else {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed."))
@@ -495,30 +505,13 @@ func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Clo
return spec.AccCardsNum > 0
}
func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")
branches := strings.Split(book.BranchName, ";")
for i, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName {
return true
}
}
}
return false
}
func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
func isRepoBranchMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")
for _, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName {
if len(splits) >= 4 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName && splits[2] == option.BranchName {
return true
}
}
@@ -532,7 +525,7 @@ func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudb
branches := strings.Split(book.BranchName, ";")
for i, bootFile := range bootFiles {
if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile {
if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName ) == bootFile {
return true
}
}
@@ -627,7 +620,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
}
}
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo, option.BranchName ), option.BranchName)
if err != nil {
log.Error("download code failed", err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
@@ -656,7 +649,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
JobType: jobType,
Description: getDescription(option),
BranchName: option.BranchName,
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName ),
Params: "{\"parameter\":[]}",
CommitID: "",
BenchmarkTypeID: 0,
@@ -687,19 +680,19 @@ func getCloudbrainType(optionType int) int {
return models.TypeCloudBrainTwo
}
func getCodePath(jobName string, repo *models.Repository) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name
func getCodePath(jobName string, repo *models.Repository, branchName string ) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name + "/" + branchName
}
func getDescription(option api.CreateFileNotebookJobOption) string {
des := option.OwnerName + "/" + option.ProjectName + "/" + option.File
des := option.OwnerName + "/" + option.ProjectName + "/" + option.BranchName + "/" + option. File
if len(des) <= CharacterLength {
return des
}
return ""
}
func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) {
func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository, imageIdInput string ) {
displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name)
jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
@@ -770,7 +763,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
}
}
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo, option.BranchName ), option.BranchName)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return
@@ -787,13 +780,18 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
Description: getDescription(option),
ImageId: setting.FileNoteBook.ImageIdNPU,
Spec: spec,
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName ),
AutoStopDurationMs: modelarts.AutoStopDurationMs / 4,
BranchName: option.BranchName,
}
if setting.ModelartsCD.Enabled {
req.ImageId = setting.FileNoteBook.ImageIdNPUCD
}
if imageIdInput != "" {
req.ImageId = imageIdInput
}
if setting.ModelartsCD.Enabled {
jobId, err = modelarts_cd.GenerateNotebook(ctx, req)
} else {
jobId, err = modelarts.GenerateNotebook2(ctx, req)
@@ -838,8 +836,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO
return true, nil
}
func getBootFile(filePath string, ownerName string, projectName string) string {
return ownerName + "/" + projectName + "/" + filePath
func getBootFile(filePath string, ownerName string, projectName string, branchName string ) string {
return ownerName + "/" + projectName + "/" + branchName + "/" + filePath
}
func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) {
@@ -1141,3 +1139,18 @@ func DeleteGrampusJob(ctx *context.Context) error {
return nil
}
func getNpuImageId(option api.CreateFileNotebookJobOption) (string, error) {
if option.Type != NPUType {
return "", fmt.Errorf("type is not npu.")
}
if option.Image == "" {
return "", nil
}
for _, imageInfo := range setting.StImageInfos.ImageInfo {
if imageInfo.Value == option.Image {
return imageInfo.Id, nil
}
}
return "", fmt.Errorf("invalid image parameter")
}