diff --git a/admin-portal/src/views/devManager/components/notebook/notebookProfile.vue b/admin-portal/src/views/devManager/components/notebook/notebookProfile.vue index b7a7daba..7123b8cc 100644 --- a/admin-portal/src/views/devManager/components/notebook/notebookProfile.vue +++ b/admin-portal/src/views/devManager/components/notebook/notebookProfile.vue @@ -63,6 +63,12 @@ {{ profileInfo.command }} + +
+ 自动停止: + {{profileInfo.autoStopDuration == -1 ? '任务不会自动停止' : profileInfo.autoStopDuration/3600 + '小时'}} +
+ diff --git a/deploy/upgrade.md b/deploy/upgrade.md index 7a960d5d..78b30499 100644 --- a/deploy/upgrade.md +++ b/deploy/upgrade.md @@ -72,8 +72,16 @@ kubectl delete -f nvidia-device-plugin.yml ## v4.3.2 -升级前 +升级前： 1. 执行 ``` kubectl create ns fluid-system +``` + +## v4.3.4 + +升级后 +1. 执行以下sql语句，其中auto_stop_duration为当前默认的notebook自动停止时间（秒），可替换为当前部署配置文件autoStopIntervalSec配置项的值 +``` +update notebook set auto_stop_duration = 14400 where auto_stop_duration = 0 ``` \ No newline at end of file diff --git a/openai-portal/src/views/modelDev/components/notebook/notebookCreation.vue b/openai-portal/src/views/modelDev/components/notebook/notebookCreation.vue index 929f5025..8d0e0fda 100644 --- a/openai-portal/src/views/modelDev/components/notebook/notebookCreation.vue +++ b/openai-portal/src/views/modelDev/components/notebook/notebookCreation.vue @@ -98,6 +98,17 @@ + + + + + + 小时 + + 高级设置 @@ -109,11 +120,11 @@ - + -
服务端口环境变量为OCTOPUS_NOTEBOOK_PORT，基础URL环境变量为OCTOPUS_NOTEBOOK_BASE_URL -
+
+ 服务端口环境变量为OCTOPUS_NOTEBOOK_PORT，基础URL环境变量为OCTOPUS_NOTEBOOK_BASE_URL +
@@ -155,6 +166,7 @@ return callback(); }; return { + isAutoStop: true, specificationVisible: false, poolList: [], ruleForm: { @@ -171,7 +183,8 @@ taskNumber: 1, resourcePool: "", specification: "", - command: "" + command: "", + autoStopDuration: 4 }, rules: { name: [ @@ -358,8 +371,12 @@ datasetVersion: this.ruleForm.dataSetVersion || "", taskNumber: this.ruleForm.taskNumber, resourcePool: this.ruleForm.resourcePool, - command: this.ruleForm.command + command: this.ruleForm.command, + autoStopDuration: this.ruleForm.autoStopDuration * 3600 }; + if(!this.isAutoStop) { + param.autoStopDuration = -1 + } const confirmInfo = this.$createElement this.$confirm( '温馨提示', { diff --git a/openai-portal/src/views/modelDev/components/notebook/notebookProfile.vue b/openai-portal/src/views/modelDev/components/notebook/notebookProfile.vue index d20c6df1..21912176 100644 --- a/openai-portal/src/views/modelDev/components/notebook/notebookProfile.vue +++ b/openai-portal/src/views/modelDev/components/notebook/notebookProfile.vue @@ -63,6 +63,12 @@ {{ profileInfo.command }}
+ +
+ 自动停止: + {{profileInfo.autoStopDuration == -1 ? '任务不会自动停止' : profileInfo.autoStopDuration/3600 + '小时'}} +
+ diff --git a/server/admin-server/api/v1/develop.proto b/server/admin-server/api/v1/develop.proto index e4e5a1d4..86684dfd 100644 --- a/server/admin-server/api/v1/develop.proto +++ b/server/admin-server/api/v1/develop.proto @@ -93,6 +93,7 @@ message Notebook { string resourcePool=27; string exitMsg = 28; string command = 29; + int64 autoStopDuration = 30; } message ListNotebookReply { diff --git a/server/base-server/api/v1/develop.proto b/server/base-server/api/v1/develop.proto index 79ea1c13..b4fe44be 100644 --- a/server/base-server/api/v1/develop.proto +++ b/server/base-server/api/v1/develop.proto @@ -52,6 +52,7 @@ message CreateNotebookRequest { map envs = 15; string command = 16; bool disableMountUserHome = 17; + int64 autoStopDuration = 18; } message CreateNotebookReply { @@ -127,6 +128,7 @@ message Notebook { string imageUrl = 25; string exitMsg = 26; string command = 27; + int64 autoStopDuration = 28; } message ListNotebookReply { diff --git a/server/base-server/internal/data/dao/model/develop.go b/server/base-server/internal/data/dao/model/develop.go index dbc1966c..506c1fb8 100644 --- a/server/base-server/internal/data/dao/model/develop.go +++ b/server/base-server/internal/data/dao/model/develop.go @@ -38,6 +38,7 @@ type Notebook struct { Envs sql.Map `gorm:"type:json;comment:环境变量"` Command string `gorm:"type:text;comment:启动命令"` DisableMountUserHome bool `gorm:"default:false;comment:是否不挂载userhome目录"` + AutoStopDuration int64 `gorm:"type:int;not null;default:0;comment:自动停止时间（秒）"` DeletedAt soft_delete.DeletedAt `gorm:"uniqueIndex:name_userId_spaceId,priority:4"` } diff --git a/server/base-server/internal/service/develop/develop.go b/server/base-server/internal/service/develop/develop.go index 800f650e..5b2f8927 100644 --- a/server/base-server/internal/service/develop/develop.go +++ b/server/base-server/internal/service/develop/develop.go @@ -417,6 +417,9 @@ func (s *developService) CreateNotebook(ctx context.Context, req *api.CreateNote nb.Status = constant.PREPARING nb.NotebookJobId = jobId nb.TaskNumber = int(req.TaskNumber) + if req.AutoStopDuration == 0 { + nb.AutoStopDuration = s.conf.Service.Develop.AutoStopIntervalSec + } nbJob := &model.NotebookJob{ Id: jobId, diff --git a/server/base-server/internal/service/develop/task.go b/server/base-server/internal/service/develop/task.go index 833a1821..b8f9edd6 100644 --- a/server/base-server/internal/service/develop/task.go +++ b/server/base-server/internal/service/develop/task.go @@ -59,10 +59,10 @@ func (s *developService) startNotebookTask() { utils.HandlePanic(ctx, func(i ...interface{}) { for pageIndex := 1; ; pageIndex++ { notebookJobs, err := s.data.DevelopDao.ListNotebookJob(ctx, &model.NotebookJobQuery{ - PageIndex: pageIndex, - PageSize: taskPageSize, - StartedAtLt: time.Now().Add(-time.Second * time.Duration(s.conf.Service.Develop.AutoStopIntervalSec)).Unix(), - StatusList: utils.NonCompletedStates(), + PageIndex: pageIndex, + PageSize: taskPageSize, + //StartedAtLt: time.Now().Add(-time.Second * time.Duration(s.conf.Service.Develop.AutoStopIntervalSec)).Unix(), + StatusList: utils.NonCompletedStates(), }) if err != nil { @@ -74,12 +74,33 @@ func (s *developService) startNotebookTask() { break } + nbs := make(map[string]*model.Notebook) + nbIds := make([]string, 0) for _, j := range notebookJobs { - _, err := s.StopNotebook(ctx, &api.StopNotebookRequest{Id: j.NotebookId}) - if err != nil { - s.log.Errorf(ctx, "StopNotebook err: %s", err) - continue + nbIds = append(nbIds, j.NotebookId) + } + + notebooks, _, err := s.data.DevelopDao.ListNotebook(ctx, &model.NotebookQuery{Ids: nbIds}) + for _, n := range notebooks { + nbs[n.Id] = n + } + + for _, j := range notebookJobs { + duration := int64(0) + if nbs[j.NotebookId].AutoStopDuration == 0 { + duration = s.conf.Service.Develop.AutoStopIntervalSec + } else { + duration = nbs[j.NotebookId].AutoStopDuration + } + + if duration > 0 && j.StartedAt != nil && time.Now().Sub(*j.StartedAt).Seconds() >= float64(duration) { + _, err := s.StopNotebook(ctx, &api.StopNotebookRequest{Id: j.NotebookId}) + if err != nil { + s.log.Errorf(ctx, "StopNotebook err: %s", err) + continue + } } + } } })() diff --git a/server/openai-server/api/v1/develop.proto b/server/openai-server/api/v1/develop.proto index 5765ecbd..3676c5dc 100644 --- a/server/openai-server/api/v1/develop.proto +++ b/server/openai-server/api/v1/develop.proto @@ -92,6 +92,7 @@ message CreateNotebookRequest { map envs = 13; string command = 14; bool disableMountUserHome = 15; + int64 autoStopDuration = 16; } message CreateNotebookReply { @@ -171,6 +172,7 @@ message Notebook { string imageUrl = 24; string exitMsg = 25; string command = 26; + int64 autoStopDuration = 27; } message ListNotebookReply {