@@ -295,15 +295,20 @@ sendjob: | |||||
return &result, nil | return &result, nil | ||||
} | } | ||||
func GetTrainJobLog(jobID string) (string, error) { | |||||
func GetTrainJobLog(jobID string, nodeId ...int) (string, error) { | |||||
checkSetting() | checkSetting() | ||||
client := getRestyClient() | client := getRestyClient() | ||||
var logContent string | var logContent string | ||||
url := HOST + urlTrainJob + "/" + jobID + "/task/0/replica/0/log" | |||||
if len(nodeId) > 0 { | |||||
url = HOST + urlTrainJob + "/" + jobID + "/task/0/replica/0/log/node/" + strconv.Itoa(nodeId[0]) | |||||
} | |||||
res, err := client.R(). | res, err := client.R(). | ||||
SetAuthToken(TOKEN). | SetAuthToken(TOKEN). | ||||
SetResult(&logContent). | SetResult(&logContent). | ||||
Get(HOST + urlTrainJob + "/" + jobID + "/task/0/replica/0/log") | |||||
Get(url) | |||||
if err != nil { | if err != nil { | ||||
return logContent, fmt.Errorf("resty GetTrainJobLog: %v", err) | return logContent, fmt.Errorf("resty GetTrainJobLog: %v", err) | ||||
@@ -324,11 +329,14 @@ func GetTrainJobLog(jobID string) (string, error) { | |||||
return logContent, nil | return logContent, nil | ||||
} | } | ||||
func GetGrampusMetrics(jobID string, startTime int64, endTime int64) (models.NewModelArtsMetricStatisticResult, error) { | |||||
func GetGrampusMetrics(jobID string, startTime int64, endTime int64, nodeId ...int) (models.NewModelArtsMetricStatisticResult, error) { | |||||
checkSetting() | checkSetting() | ||||
client := getRestyClient() | client := getRestyClient() | ||||
var result models.NewModelArtsMetricStatisticResult | var result models.NewModelArtsMetricStatisticResult | ||||
url := HOST + urlTrainJob + "/" + jobID + "/task/0/replica/0/metrics" | url := HOST + urlTrainJob + "/" + jobID + "/task/0/replica/0/metrics" | ||||
if len(nodeId) > 0 { | |||||
url = HOST + urlTrainJob + "/" + jobID + "/task/0/replica/0/metrics/node/" + strconv.Itoa(nodeId[0]) | |||||
} | |||||
if startTime > 0 { | if startTime > 0 { | ||||
var step int64 = 60 | var step int64 = 60 | ||||
@@ -1244,8 +1244,11 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.GrampusStopJob) | m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.GrampusStopJob) | ||||
m.Get("/log", repo_ext.GrampusGetLog) | m.Get("/log", repo_ext.GrampusGetLog) | ||||
m.Get("/metrics", repo_ext.GrampusMetrics) | m.Get("/metrics", repo_ext.GrampusMetrics) | ||||
m.Get("/metrics/:nodeId", repo_ext.GrampusMetrics) | |||||
m.Get("/log/:nodeId", repo_ext.GrampusGetLog) | |||||
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload) | m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload) | ||||
m.Get("/download_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo_ext.GrampusDownloadLog) | m.Get("/download_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo_ext.GrampusDownloadLog) | ||||
m.Get("/download_log/:nodeId", cloudbrain.AdminOrJobCreaterRightForTrain, repo_ext.GrampusDownloadLog) | |||||
m.Get("/job_event", repo_ext.GrampusTrainJobEvents) | m.Get("/job_event", repo_ext.GrampusTrainJobEvents) | ||||
}) | }) | ||||
}) | }) | ||||
@@ -8,6 +8,7 @@ import ( | |||||
"net/http" | "net/http" | ||||
"os" | "os" | ||||
"path" | "path" | ||||
"strconv" | |||||
"strings" | "strings" | ||||
"time" | "time" | ||||
"unicode/utf8" | "unicode/utf8" | ||||
@@ -1660,7 +1661,18 @@ func GrampusDownloadLog(ctx *context.Context) { | |||||
return | return | ||||
} | } | ||||
content, err := grampus.GetTrainJobLog(job.JobID) | |||||
nodeIdStr := ctx.Params(":nodeId") | |||||
var content string | |||||
if nodeIdStr != "" { | |||||
nodeId, _ := strconv.Atoi(nodeIdStr) | |||||
if job.WorkServerNumber < 2 || nodeId > job.WorkServerNumber-1 { | |||||
ctx.NotFound("query parameter is wrong", nil) | |||||
return | |||||
} | |||||
content, err = grampus.GetTrainJobLog(job.JobID, nodeId) | |||||
} else { | |||||
content, err = grampus.GetTrainJobLog(job.JobID) | |||||
} | |||||
if err != nil { | if err != nil { | ||||
log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"]) | log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"]) | ||||
content = "" | content = "" | ||||
@@ -1696,7 +1708,19 @@ func GrampusGetLog(ctx *context.Context) { | |||||
exitDiagnostics = result.ExitDiagnostics | exitDiagnostics = result.ExitDiagnostics | ||||
} | } | ||||
content, err := grampus.GetTrainJobLog(job.JobID) | |||||
nodeIdStr := ctx.Params(":nodeId") | |||||
var content string | |||||
if nodeIdStr != "" { | |||||
nodeId, _ := strconv.Atoi(nodeIdStr) | |||||
if job.WorkServerNumber < 2 || nodeId > job.WorkServerNumber-1 { | |||||
ctx.NotFound("query parameter is wrong", nil) | |||||
return | |||||
} | |||||
content, err = grampus.GetTrainJobLog(job.JobID, nodeId) | |||||
} else { | |||||
content, err = grampus.GetTrainJobLog(job.JobID) | |||||
} | |||||
if err != nil { | if err != nil { | ||||
log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"]) | log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"]) | ||||
ctx.JSON(http.StatusOK, map[string]interface{}{ | ctx.JSON(http.StatusOK, map[string]interface{}{ | ||||
@@ -1734,7 +1758,17 @@ func GrampusMetrics(ctx *context.Context) { | |||||
} | } | ||||
var result models.NewModelArtsMetricStatisticResult | var result models.NewModelArtsMetricStatisticResult | ||||
if job.IsNPUTask() { | if job.IsNPUTask() { | ||||
result, err = grampus.GetGrampusMetrics(job.JobID, 0, 0) | |||||
nodeIdStr := ctx.Params(":nodeId") | |||||
if nodeIdStr != "" { | |||||
nodeId, _ := strconv.Atoi(nodeIdStr) | |||||
if job.WorkServerNumber < 2 || nodeId > job.WorkServerNumber-1 { | |||||
ctx.NotFound("query parameter is wrong", nil) | |||||
return | |||||
} | |||||
result, err = grampus.GetGrampusMetrics(job.JobID, 0, 0, nodeId) | |||||
} else { | |||||
result, err = grampus.GetGrampusMetrics(job.JobID, 0, 0) | |||||
} | |||||
} else if job.IsGPUTask() { | } else if job.IsGPUTask() { | ||||
startTime := int64(job.StartTime) | startTime := int64(job.StartTime) | ||||
if startTime == 0 { | if startTime == 0 { | ||||
Dear OpenI User
Thank you for your continuous support to the Openl Qizhi Community AI Collaboration Platform. In order to protect your usage rights and ensure network security, we updated the Openl Qizhi Community AI Collaboration Platform Usage Agreement in January 2024. The updated agreement specifies that users are prohibited from using intranet penetration tools. After you click "Agree and continue", you can continue to use our services. Thank you for your cooperation and understanding.
For more agreement content, please refer to the《Openl Qizhi Community AI Collaboration Platform Usage Agreement》