#3761 fix-3693 3576

Merged
chenyifan01 merged 4 commits from fix-3693 into V20230307 1 year ago
  1. +1
    -0
      modules/structs/cloudbrain.go
  2. +2
    -2
      routers/api/v1/repo/cloudbrain.go
  3. +58
    -45
      services/cloudbrain/cloudbrainTask/notebook.go
  4. +75
    -44
      web_src/vuepages/pages/notebook/debug/index.vue

+ 1
- 0
modules/structs/cloudbrain.go View File

@@ -59,6 +59,7 @@ type CreateNotebookOption struct {

type CreateFileNotebookJobOption struct {
Type int `json:"type"` //0 CPU 1 GPU 2 NPU
Image string `json:"image"`
File string `json:"file" binding:"Required"`
BranchName string `json:"branch_name" binding:"Required"`
OwnerName string `json:"owner_name" binding:"Required"`


+ 2
- 2
routers/api/v1/repo/cloudbrain.go View File

@@ -167,7 +167,7 @@ func GetFileNoteBookInfo(ctx *context.APIContext) {
"waitCountNpu": waitCountNpu,
"imageCpuDescription": setting.FileNoteBook.ImageCPUDescription,
"imageGpuDescription": setting.FileNoteBook.ImageGPUDescription,
"imageNpuDescription": setting.FileNoteBook.ImageNPUDescription,
"imageNpuDescription": setting.StImageInfos.ImageInfo,
"cloudBrainPaySwitch": setting.CloudBrainPaySwitch,
"PointAccount": a,
})
@@ -182,7 +182,7 @@ func GetFileNoteBookInfo(ctx *context.APIContext) {
"waitCountNpu": waitCountNpu,
"imageCpuDescription": setting.FileNoteBook.ImageCPUDescription,
"imageGpuDescription": setting.FileNoteBook.ImageGPUDescription,
"imageNpuDescription": setting.FileNoteBook.ImageNPUCDDescription,
"imageNpuDescription": setting.StImageInfos.ImageInfo,
"cloudBrainPaySwitch": setting.CloudBrainPaySwitch,
"PointAccount": a,
})


+ 58
- 45
services/cloudbrain/cloudbrainTask/notebook.go View File

@@ -278,7 +278,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong")))
return
}
if len(getBootFile(option.File, option.OwnerName, option.ProjectName)) > CharacterLength {
if len(getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName)) > CharacterLength {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long")))
return
}
@@ -287,6 +287,16 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
return
}

var imageIdNpu string
var err error
if option.Type == NPUType {
imageIdNpu, err = getNpuImageId(option)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.parameter_is_wrong")))
return
}
}

isNotebookFileExist, _ := isNoteBookFileExist(ctx, option)
if !isNotebookFileExist {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist")))
@@ -335,14 +345,9 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type))
if noteBook != nil {

if isRepoConfilcts(option, noteBook) {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict")))
return
}

if isNotebookSpecMath(option, noteBook) {
if !isRepoMatch(option, noteBook) {
err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName)
if isNotebookSpecMath(option, noteBook) && isImageMatch(option.Image, noteBook) {
if !isRepoBranchMatch(option, noteBook) {
err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo, option.BranchName), option.BranchName)
if err != nil {
log.Error("download code failed", err)
if !strings.Contains(err.Error(), "already exists and is not an empty directory") {
@@ -352,8 +357,8 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
}
}
if !isRepoFileMatch(option, noteBook) {
if len(noteBook.BootFile)+len(getBootFile(option.File, option.OwnerName, option.ProjectName))+1 <= CharacterLength {
noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName)
if len(noteBook.BootFile)+len(getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName))+1 <= CharacterLength {
noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName)
} else {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long")))
return
@@ -390,9 +395,14 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
if option.Type <= GPUType {
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo)
} else {
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo)
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo, imageIdNpu)
}

}

func isImageMatch(npuImage string, book *models.Cloudbrain) bool {
return (!book.IsNPUTask()) || book.Image == npuImage

}
func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) {
if ctx.Written() {
@@ -431,7 +441,7 @@ func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOp
return
}

if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) {
if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName), task, token) {
ctx.JSON(http.StatusOK, models.BaseOKMessageApi)
} else {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed."))
@@ -495,30 +505,13 @@ func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Clo
return spec.AccCardsNum > 0
}

func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")
branches := strings.Split(book.BranchName, ";")

for i, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName {
return true
}
}
}

return false

}

func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
func isRepoBranchMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")

for _, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName {
if len(splits) >= 4 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName && splits[2] == option.BranchName {
return true
}
}
@@ -532,7 +525,7 @@ func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudb
branches := strings.Split(book.BranchName, ";")

for i, bootFile := range bootFiles {
if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile {
if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName) == bootFile {
return true
}
}
@@ -627,7 +620,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
}
}

err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo, option.BranchName), option.BranchName)
if err != nil {
log.Error("download code failed", err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
@@ -656,7 +649,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
JobType: jobType,
Description: getDescription(option),
BranchName: option.BranchName,
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName),
Params: "{\"parameter\":[]}",
CommitID: "",
BenchmarkTypeID: 0,
@@ -687,19 +680,19 @@ func getCloudbrainType(optionType int) int {
return models.TypeCloudBrainTwo
}

func getCodePath(jobName string, repo *models.Repository) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name
func getCodePath(jobName string, repo *models.Repository, branchName string) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name + "/" + branchName
}

func getDescription(option api.CreateFileNotebookJobOption) string {
des := option.OwnerName + "/" + option.ProjectName + "/" + option.File
des := option.OwnerName + "/" + option.ProjectName + "/" + option.BranchName + "/" + option.File
if len(des) <= CharacterLength {
return des
}
return ""
}

func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) {
func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository, imageIdInput string) {
displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name)
jobName := util.ConvertDisplayJobNameToJobName(displayJobName)

@@ -770,7 +763,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
}
}

err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo, option.BranchName), option.BranchName)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return
@@ -787,13 +780,18 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
Description: getDescription(option),
ImageId: setting.FileNoteBook.ImageIdNPU,
Spec: spec,
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName, option.BranchName),
AutoStopDurationMs: modelarts.AutoStopDurationMs / 4,
BranchName: option.BranchName,
}

if setting.ModelartsCD.Enabled {
req.ImageId = setting.FileNoteBook.ImageIdNPUCD
}
if imageIdInput != "" {
req.ImageId = imageIdInput
}

if setting.ModelartsCD.Enabled {
jobId, err = modelarts_cd.GenerateNotebook(ctx, req)
} else {
jobId, err = modelarts.GenerateNotebook2(ctx, req)
@@ -838,8 +836,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO
return true, nil
}

func getBootFile(filePath string, ownerName string, projectName string) string {
return ownerName + "/" + projectName + "/" + filePath
func getBootFile(filePath string, ownerName string, projectName string, branchName string) string {
return ownerName + "/" + projectName + "/" + branchName + "/" + filePath
}

func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) {
@@ -1141,3 +1139,18 @@ func DeleteGrampusJob(ctx *context.Context) error {

return nil
}

func getNpuImageId(option api.CreateFileNotebookJobOption) (string, error) {
if option.Type != NPUType {
return "", fmt.Errorf("type is not npu.")
}
if option.Image == "" {
return "", nil
}
for _, imageInfo := range setting.StImageInfos.ImageInfo {
if imageInfo.Value == option.Image {
return imageInfo.Id, nil
}
}
return "", fmt.Errorf("invalid image parameter")
}

+ 75
- 44
web_src/vuepages/pages/notebook/debug/index.vue View File

@@ -64,7 +64,13 @@
<span>{{npuSpec}}</span>
</div>
<div class="detail-spec">
<span>{{$t('image')}}:{{notebookInfo.imageNpuDescription}}</span>
<!-- <span>{{$t('image')}}:{{notebookInfo.imageNpuDescription}}</span> -->
<div class="select-image">
<label>{{$t('image')}}:</label>
<select @click.stop="" class="ui dropdown select-image-label" v-model="selecedImage" >
<option v-for="item in npuImageType" :key="item.id" :value="item.value">{{item.value}} </option>
</select>
</div>
</div>
</div>
<div class="resource-select" :style="notebookInfo.cloudBrainPaySwitch ? {marginBottom: '10px'} : ''">
@@ -241,6 +247,7 @@ export default {

useUnitPrice: '',
useTime: '',
selecedImage:''
};
},
methods: {
@@ -286,6 +293,9 @@ export default {
getFileNotebook().then((res)=>{
if(res.data.code==0){
this.notebookInfo = res.data
if (this.activeLoadFirst) {
this.selecedImage = this.notebookInfo.imageNpuDescription[0].value
}
this.getPointInfo()
}else{
Message.error(res.data.message)
@@ -342,15 +352,14 @@ export default {
},
getFileInfoReadyNotebook(data,index){
getFileInfoNotebook(data).then((res)=>{
console.log(res)
if(res.data.code===0){
if(index===2){
this.btnStatus[2]=2
this.deubgUrlNpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/modelarts/notebook/${data.job_id}/debug?file=${this.fileInfo.owner_name}/${this.fileInfo.project_name}/${this.fileInfo.file}`
this.deubgUrlNpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/modelarts/notebook/${data.job_id}/debug?file=${this.fileInfo.owner_name}/${this.fileInfo.project_name}/${this.fileInfo.branch_name}/${this.fileInfo.file}`
this.deubgUrlNpuStop = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/modelarts/notebook/${data.job_id}/stop`
}else{
this.btnStatus[index]=2
this.deubgUrlGpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/cloudbrain/${data.job_id}/debug?file=${this.fileInfo.owner_name}/${this.fileInfo.project_name}/${this.fileInfo.file}`
this.deubgUrlGpu = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/cloudbrain/${data.job_id}/debug?file=${this.fileInfo.owner_name}/${this.fileInfo.project_name}/${this.fileInfo.branch_name}/${this.fileInfo.file}`
this.deubgUrlGpuStop = `${AppSubUrl}/${this.fileInfo.sign_name}/${this.notebookInfo.projectName}/cloudbrain/${data.job_id}/stop`
}
clearInterval(timerCb3)
@@ -378,42 +387,50 @@ export default {
})
},
createTask(index){
this.btnStatus[index]=1
const data = {type:index,...this.fileInfo}
let repoPath = `repos/${this.fileInfo.owner_name}/${this.fileInfo.project_name}`
createNotebook(data).then((res)=>{
if(res.data.code===0 && res.status===200){
if(index===2){
timerCb2 = setInterval(() => {
setTimeout(this.getCb2NotebookInfo(repoPath,res.data.message,data), 0)
}, 10000)
}else{
timerCb1 = setInterval(() => {
setTimeout(this.getCb1NotebookInfo(repoPath,res.data.message,index,data), 0)
}, 10000)
}
this.alertCb = false
}else if(res.data.code==2){
this.btnStatus[index]=0
this.alertCb = true
}else{
this.btnStatus[index]=0
Message.error(res.data.message)
}
}).catch((err)=>{
if(err.response.status===403 && err.response.data.code===1 ){
location.href=`${AppSubUrl}/authentication/wechat/bind`
}
if(err.response.status===401){
location.href=`${AppSubUrl}/user/login?redirect_to=${encodeURIComponent(location.origin+location.pathname + '?type=login&card='+ this.selectIndex)}`
return
this.btnStatus[index] = 1
if (index === 2) {
this.fileInfo.image = this.selecedImage
}
const data = {type:index,...this.fileInfo}
let repoPath = `repos/${this.fileInfo.owner_name}/${this.fileInfo.project_name}`
createNotebook(data).then((res)=>{
if(res.data.code===0 && res.status===200){
if(index===2){
timerCb2 = setInterval(() => {
setTimeout(this.getCb2NotebookInfo(repoPath,res.data.message,data), 0)
}, 10000)
}else{
timerCb1 = setInterval(() => {
setTimeout(this.getCb1NotebookInfo(repoPath,res.data.message,index,data), 0)
}, 10000)
}
this.alertCb = false
}else if(res.data.code==2){
this.btnStatus[index]=0
this.alertCb = true
}else{
this.btnStatus[index]=0
Message.error(res.data.message)
}
this.btnStatus[index]=0
this.alertCb = false
Message.error(err)
})
}).catch((err)=>{
if(err.response.status===403 && err.response.data.code===1 ){
location.href=`${AppSubUrl}/authentication/wechat/bind`
}
if(err.response.status===401){
location.href=`${AppSubUrl}/user/login?redirect_to=${encodeURIComponent(location.origin+location.pathname + '?type=login&card='+ this.selectIndex)}`
return
}
this.btnStatus[index]=0
this.alertCb = false
Message.error(err)
})
},
handler() {
this.getNotebookInfo()
this.dialogVisible = true;
this.getPointInfo()
}
},
computed: {
@@ -447,6 +464,9 @@ export default {
}
return npu_spec
},
npuImageType() {
return this.notebookInfo.imageNpuDescription
},
gpuSpec(){
let gpu_spec = `${this.$t("notebook.specification")}:GPU: ${this.notebookInfo.specGpu.acc_cards_num}*${this.notebookInfo.specGpu.acc_card_type}, CPU: ${this.notebookInfo.specGpu.cpu_cores}`
if(this.notebookInfo.specGpu.gpu_mem_gi_b!==0){
@@ -464,6 +484,9 @@ export default {
beforeDestroy() {
clearInterval(timerCb1)
clearInterval(timerCb2)
document
.querySelector("#notebook-debug")
.removeEventListener("click", this.handler);
},
mounted() {
const selfData = document.querySelector('#__vue-self-data')
@@ -481,11 +504,7 @@ export default {
}
document
.querySelector("#notebook-debug")
.addEventListener("click", function () {
that.getNotebookInfo()
that.dialogVisible = true;
that.getPointInfo()
});
.addEventListener("click", this.handler);
},
};
</script>
@@ -622,6 +641,18 @@ export default {
letter-spacing: 0px;
line-height: 24px;
text-decoration: none;
.select-image{
color: rgb(50, 145, 248);
display: flex;
align-items: center;
.select-image-label{
border: none !important;
background: transparent;
color: rgb(50, 145, 248);
padding-left: 0;
margin-left: -5px;
}
}
}
}
.resource-select {


Loading…
Cancel
Save