|
|
@@ -27,6 +27,8 @@ const ( |
|
|
|
urlTrainJob = "/training-jobs" |
|
|
|
urlResourceSpecs = "/job/resource-specs" |
|
|
|
urlTrainJobConfig = "/training-job-configs" |
|
|
|
urlDeployModel = "/models" |
|
|
|
urlDeployService = "/services" |
|
|
|
errorCodeExceedLimit = "ModelArts.0118" |
|
|
|
|
|
|
|
//notebook 2.0 |
|
|
@@ -1033,6 +1035,17 @@ func DelTrainJob(jobID string) (*models.TrainJobResult, error) { |
|
|
|
client := getRestyClient() |
|
|
|
var result models.TrainJobResult |
|
|
|
|
|
|
|
//get cloudbrain job by jobid |
|
|
|
finetuneJob, _ := models.GetCloudbrainByJobID(jobID) |
|
|
|
log.Info("调试:%s", finetuneJob.FineTune) |
|
|
|
if finetuneJob.FineTune { |
|
|
|
err := ServiceDelete(jobID) |
|
|
|
if err != nil { |
|
|
|
log.Error("盘古微调部署: Delete Deploy failed:%s %v", jobID, err.Error()) |
|
|
|
return &result, fmt.Errorf("delete train job [%s] failed [%v]", jobID, err.Error()) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
|
|
|
|
sendjob: |
|
|
@@ -1524,3 +1537,337 @@ sendjob: |
|
|
|
|
|
|
|
return &result, nil |
|
|
|
} |
|
|
|
|
|
|
|
func createDeployModel(createDeployModelParams models.CreateDeployModelParams) (*models.CreateDeployModelResult, error) { |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
var result models.CreateDeployModelResult |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
req, _ := json.Marshal(createDeployModelParams) |
|
|
|
log.Info("postapi json: %s", req) |
|
|
|
|
|
|
|
sendjob: |
|
|
|
res, err := client.R(). |
|
|
|
SetHeader("Content-Type", "application/json"). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
SetBody(createDeployModelParams). |
|
|
|
SetResult(&result). |
|
|
|
Post(HOST + "/v1/" + setting.ProjectID + urlDeployModel) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
return nil, fmt.Errorf("resty create deploy model: %s", err) |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var temp models.ErrorResult |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
log.Error("createDeployModel failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
if res.StatusCode() == http.StatusBadGateway { |
|
|
|
return &result, fmt.Errorf(UnknownErrorPrefix+"createDeployModel failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} else { |
|
|
|
return &result, fmt.Errorf("createDeployModel failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
return &result, nil |
|
|
|
} |
|
|
|
|
|
|
|
func GetDeployModel(modelID string) (*models.GetDeployModelResult, error) { |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
var result models.GetDeployModelResult |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
|
|
|
|
sendjob: |
|
|
|
//https://modelarts.cn-east-3.myhuaweicloud.com/v1/{project_id}/models/{model_id} |
|
|
|
res, err := client.R(). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
SetResult(&result). |
|
|
|
Get(HOST + "/v1/" + setting.ProjectID + urlDeployModel + "/" + modelID) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
return nil, fmt.Errorf("resty GetDeployModel: %v", err) |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var temp models.ErrorResult |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
log.Error("GetTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
return &result, fmt.Errorf("获取部署模型失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} |
|
|
|
|
|
|
|
return &result, nil |
|
|
|
} |
|
|
|
|
|
|
|
func createDeployService(CreateDeployServiceParams models.CreateDeployServiceParams) (*models.CreateDeployServiceResult, error) { |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
var result models.CreateDeployServiceResult |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
req, _ := json.Marshal(CreateDeployServiceParams) |
|
|
|
log.Info("postapi json: %s", req) |
|
|
|
|
|
|
|
sendjob: |
|
|
|
res, err := client.R(). |
|
|
|
SetHeader("Content-Type", "application/json"). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
SetBody(CreateDeployServiceParams). |
|
|
|
SetResult(&result). |
|
|
|
Post(HOST + "/v1/" + setting.ProjectID + urlDeployService) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
return nil, fmt.Errorf("resty create deploy model: %s", err) |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var temp models.ErrorResult |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
log.Error("createDeployModel failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
if res.StatusCode() == http.StatusBadGateway { |
|
|
|
return &result, fmt.Errorf(UnknownErrorPrefix+"createDeployModel failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} else { |
|
|
|
return &result, fmt.Errorf("createDeployModel failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
return &result, nil |
|
|
|
} |
|
|
|
|
|
|
|
func GetDeployService(serviceID string) (*models.GetDeployServiceResult, error) { |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
var result models.GetDeployServiceResult |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
|
|
|
|
sendjob: |
|
|
|
//https://modelarts.cn-east-3.myhuaweicloud.com/v1/{project_id}/models/{model_id} |
|
|
|
res, err := client.R(). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
SetResult(&result). |
|
|
|
Get(HOST + "/v1/" + setting.ProjectID + urlDeployService + "/" + serviceID) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
return nil, fmt.Errorf("resty GetDeployService: %v", err) |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var temp models.ErrorResult |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
log.Error("GetTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
return &result, fmt.Errorf("获取部署模型失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} |
|
|
|
|
|
|
|
return &result, nil |
|
|
|
} |
|
|
|
|
|
|
|
func DeleteDeployModel(modelID string) error { |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
retry := 0 |
|
|
|
|
|
|
|
sendjob: |
|
|
|
res, err := client.R(). |
|
|
|
SetHeader("Content-Type", "application/json"). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
Delete(HOST + "/v1/" + setting.ProjectID + urlDeployModel + "/" + modelID) |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var temp models.ErrorResult |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
return fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
log.Error("DelTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
return fmt.Errorf("删除部署模型失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} |
|
|
|
|
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func DeleteDeployService(serviceID string) error { |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
|
|
|
|
sendjob: |
|
|
|
res, err := client.R(). |
|
|
|
SetHeader("Content-Type", "application/json"). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
Delete(HOST + "/v1/" + setting.ProjectID + urlDeployService + "/" + serviceID) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
return fmt.Errorf("resty DelDeployService: %v", err) |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var temp models.ErrorResult |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
return fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
log.Error("DelTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
return fmt.Errorf("删除部署任务失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) |
|
|
|
} |
|
|
|
|
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func UpdateDeployService(serviceID string, UpdateDeployServiceParams models.UpdateDeployServiceParams) error { |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
|
|
|
|
sendjob: |
|
|
|
res, err := client.R(). |
|
|
|
SetHeader("Content-Type", "application/json"). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
SetBody(UpdateDeployServiceParams). |
|
|
|
Put(HOST + "/v1/" + setting.ProjectID + urlDeployService + "/" + serviceID) |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var temp models.ErrorResult |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
//return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
// https://modelarts-inference.cloudbrain2.pcl.ac.cn/v1/infers/cc1e11c5-16a0-4202-a031-819a8e178a0d |
|
|
|
func SendInferenceDeploy(inferAddr string, text string) (*models.PanguInferResult, error) { |
|
|
|
panguInferParams := &models.PanguInferParams{ |
|
|
|
Text: text, |
|
|
|
} |
|
|
|
checkSetting() |
|
|
|
client := getRestyClient() |
|
|
|
var result models.PanguInferResult |
|
|
|
|
|
|
|
retry := 0 |
|
|
|
req, _ := json.Marshal(panguInferParams) |
|
|
|
log.Info("盘古微调部署:postapi json: ", req) |
|
|
|
log.Info("盘古微调部署 推理地址: ", inferAddr) |
|
|
|
log.Info("盘古微调部署 推理token: ", TOKEN) |
|
|
|
|
|
|
|
sendjob: |
|
|
|
res, err := client.R(). |
|
|
|
SetHeader("Content-Type", "application/json"). |
|
|
|
SetHeader("X-Auth-Token", TOKEN). |
|
|
|
SetAuthToken(TOKEN). |
|
|
|
SetBody(panguInferParams). |
|
|
|
SetResult(&result). |
|
|
|
Post(inferAddr) |
|
|
|
//Post(HOST + "/v1/" + urlDeployInfer + serviceID) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
return nil, fmt.Errorf("resty post text inference failed: %s", err) |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() == http.StatusUnauthorized && retry < 1 { |
|
|
|
retry++ |
|
|
|
_ = getToken() |
|
|
|
goto sendjob |
|
|
|
} |
|
|
|
|
|
|
|
if res.StatusCode() != http.StatusOK { |
|
|
|
var erroResult models.PanguInferError |
|
|
|
if err = json.Unmarshal([]byte(res.String()), &erroResult); err != nil { |
|
|
|
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) |
|
|
|
} |
|
|
|
log.Error("盘古微调部署 inference failed(%d):%s", res.StatusCode(), res.String()) |
|
|
|
log.Error("盘古微调部署 %s, %s", erroResult.ErrorCode, erroResult.ErrorMsg) |
|
|
|
return &result, fmt.Errorf("盘古微调部署 部署任务推理失败(%d): %s", res.StatusCode(), erroResult.ErrorMsg) |
|
|
|
} |
|
|
|
log.Info("盘古微调部署 推理返回状态码: %s", res.StatusCode()) |
|
|
|
log.Info("盘古微调部署 推理结果: %s", result) |
|
|
|
|
|
|
|
return &result, nil |
|
|
|
} |
|
|
|
|
|
|
|
func ServiceDelete(jobID string) error { |
|
|
|
if deploy, _ := models.GetModelartsDeployByJobID(jobID); deploy != nil { |
|
|
|
if deploy.Status == "STOP" || deploy.Status == "FAILED" { |
|
|
|
err := DeleteDeployService(deploy.ServiceID) |
|
|
|
if err != nil { |
|
|
|
log.Error("盘古微调部署: Delete DeployService API failed:%s %v", jobID, err.Error()) |
|
|
|
return err |
|
|
|
} |
|
|
|
err = DeleteDeployModel(deploy.ModelID) |
|
|
|
if err != nil { |
|
|
|
log.Error("盘古微调部署: Delete DeployModel API failed:%s %v", jobID, err.Error()) |
|
|
|
return err |
|
|
|
} |
|
|
|
|
|
|
|
err = models.DeleteModelartsDeploy(jobID) |
|
|
|
if err != nil { |
|
|
|
log.Error("盘古微调部署: Delete ModelartsDeploy from DB failed:%s %v", jobID, err.Error()) |
|
|
|
return err |
|
|
|
} |
|
|
|
} else { |
|
|
|
log.Error("the job(%s) is a deploying finetune job, can be not deleted", jobID) |
|
|
|
return fmt.Errorf("the job(%s) is a deploying finetune job, can be not deleted", jobID) |
|
|
|
} |
|
|
|
} |
|
|
|
return nil |
|
|
|
} |