@@ -47,12 +47,24 @@ type AiModelManage struct { | |||
IsCanDownload bool `xorm:"-" json:"isCanDownload"` | |||
IsCollected bool `xorm:"-" json:"isCollected"` | |||
RepoName string `xorm:"-" json:"repoName"` | |||
RepoDisplayName string `xorm:"-" json:"repoDisplayName"` | |||
RepoOwnerName string `xorm:"-" json:"repoOwnerName"` | |||
ReferenceCount int `xorm:"NOT NULL DEFAULT 0" json:"referenceCount"` | |||
CollectedCount int `xorm:"NOT NULL DEFAULT 0" json:"collectedCount"` | |||
ModelFileList []storage.FileInfo `xorm:"-" json:"modelFileList"` | |||
} | |||
type AiModelFile struct { | |||
ID int64 `xorm:"pk autoincr"` | |||
ModelID string `xorm:"UNIQUE(s)"` | |||
Name string `xorm:"varchar(400) UNIQUE(s)"` | |||
Path string `xorm:"varchar(400) NULL"` | |||
Description string `xorm:"varchar(400) NULL"` | |||
DownloadCount int64 `xorm:"DEFAULT 0"` | |||
Size int64 `xorm:"DEFAULT 0"` | |||
CreatedUnix timeutil.TimeStamp `xorm:"created"` | |||
} | |||
type AiModelCollect struct { | |||
ID int64 `xorm:"pk autoincr"` | |||
ModelID string `xorm:"UNIQUE(s)"` | |||
@@ -113,6 +125,7 @@ type AiModelQueryOptions struct { | |||
LabelFilter string | |||
FrameFilter int | |||
ComputeResourceFilter string | |||
NotNeedEmpty bool | |||
} | |||
func (a *AiModelConvert) IsGpuTrainTask() bool { | |||
@@ -393,15 +406,6 @@ func ModifyModelSize(id string, size int64) error { | |||
return nil | |||
} | |||
func AddModelInferenceCount(id string) error { | |||
sess := x.NewSession() | |||
defer sess.Close() | |||
if _, err := sess.Exec("UPDATE `ai_model_manage` SET reference_count = reference_count + 1 WHERE id = ?", id); err != nil { | |||
return err | |||
} | |||
return nil | |||
} | |||
func ModifyModelStatus(id string, modelSize int64, status int, modelPath string, statusDesc string) error { | |||
var sess *xorm.Session | |||
sess = x.ID(id) | |||
@@ -464,6 +468,19 @@ func QueryModelByRepoId(repoId int64) []*AiModelManage { | |||
return aiModelManageList | |||
} | |||
func DeleteModelByRepoId(repoId int64) error { | |||
sess := x.NewSession() | |||
defer sess.Close() | |||
re, err := sess.Delete(&AiModelManage{ | |||
RepoId: repoId, | |||
}) | |||
if err != nil { | |||
return err | |||
} | |||
log.Info("success to delete DeleteModelByRepoId from db.re=" + fmt.Sprint((re))) | |||
return nil | |||
} | |||
func QueryModelByPath(path string) (*AiModelManage, error) { | |||
modelManage := new(AiModelManage) | |||
has, err := x.Where("path=?", path).Get(modelManage) | |||
@@ -479,93 +496,72 @@ func QueryModelByPath(path string) (*AiModelManage, error) { | |||
func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { | |||
sess := x.NewSession() | |||
defer sess.Close() | |||
var cond = builder.NewCond() | |||
var where string | |||
where += " ai_model_manage.user_id > 0 " | |||
if opts.RepoID > 0 { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.repo_id": opts.RepoID}, | |||
) | |||
where += " and ai_model_manage.repo_id= " + fmt.Sprint(opts.RepoID) | |||
} | |||
if opts.UserID > 0 { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.user_id": opts.UserID}, | |||
) | |||
where += " and ai_model_manage.user_id=" + fmt.Sprint(opts.UserID) | |||
} | |||
if opts.New >= 0 { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.new": opts.New}, | |||
) | |||
where += " and ai_model_manage.new=" + fmt.Sprint(opts.New) | |||
} | |||
if len(opts.ModelID) > 0 { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.id": opts.ModelID}, | |||
) | |||
where += " and ai_model_manage.id='" + fmt.Sprint(opts.ModelID) + "'" | |||
} | |||
if (opts.Type) >= 0 { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.type": opts.Type}, | |||
) | |||
where += " and ai_model_manage.type=" + fmt.Sprint(opts.Type) | |||
} | |||
if (opts.Status) >= 0 { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.status": opts.Status}, | |||
) | |||
where += " and ai_model_manage.status=" + fmt.Sprint(opts.Status) | |||
} | |||
if !opts.IsQueryPrivate { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.is_private": false}, | |||
) | |||
where += " and ai_model_manage.is_private=false" | |||
} | |||
if opts.IsRecommend { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.recommend": 1}, | |||
) | |||
where += " and ai_model_manage.recommend=1" | |||
} | |||
if opts.FrameFilter >= 0 { | |||
if opts.FrameFilter == 2 { | |||
cond = cond.And( | |||
builder.In("ai_model_manage.engine", []int{2, 121, 122}), | |||
) | |||
where += " and ai_model_manage.engine in (2,121,122)" | |||
} else { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.engine": opts.FrameFilter}, | |||
) | |||
where += " and ai_model_manage.engine=" + fmt.Sprint(opts.FrameFilter) | |||
} | |||
} | |||
if opts.LabelFilter != "" { | |||
cond = cond.And( | |||
builder.Like{"ai_model_manage.label", opts.LabelFilter}, | |||
) | |||
where += " and ai_model_manage.label ILIKE '%" + opts.LabelFilter + "%'" | |||
} | |||
if opts.ComputeResourceFilter != "" { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_manage.compute_resource": opts.ComputeResourceFilter}, | |||
) | |||
where += " and ai_model_manage.compute_resource ILIKE '%" + opts.ComputeResourceFilter + "%'" | |||
} | |||
if opts.Namelike != "" { | |||
cond = cond.And( | |||
builder.Like{"ai_model_manage.name", opts.Namelike}, | |||
) | |||
where += " and ( ai_model_manage.name ILIKE '%" + opts.Namelike + "%'" | |||
where += " or ai_model_manage.description ILIKE '%" + opts.Namelike + "%'" | |||
where += " or ai_model_manage.label ILIKE '%" + opts.Namelike + "%')" | |||
} | |||
if opts.NotNeedEmpty { | |||
where += " and ai_model_manage.size > 0 " | |||
} | |||
var count int64 | |||
var err error | |||
if opts.IsCollected { | |||
cond = cond.And( | |||
builder.Eq{"ai_model_collect.user_id": opts.CollectedUserId}, | |||
) | |||
count, err = sess.Join("INNER", "ai_model_collect", "ai_model_manage.id = ai_model_collect.model_id").Where(cond).Count(new(AiModelManage)) | |||
where += " and ai_model_collect.user_id=" + fmt.Sprint(opts.CollectedUserId) | |||
count, err = sess.Join("INNER", "ai_model_collect", "ai_model_manage.id = ai_model_collect.model_id").Where(where).Count(new(AiModelManage)) | |||
if err != nil { | |||
log.Info("error=" + err.Error()) | |||
return nil, 0, fmt.Errorf("Count: %v", err) | |||
} | |||
} else { | |||
count, err = sess.Where(cond).Count(new(AiModelManage)) | |||
count, err = sess.Where(where).Count(new(AiModelManage)) | |||
if err != nil { | |||
log.Info("error=" + err.Error()) | |||
return nil, 0, fmt.Errorf("Count: %v", err) | |||
} | |||
} | |||
@@ -588,8 +584,9 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { | |||
} | |||
sess.OrderBy(orderby) | |||
aiModelManages := make([]*AiModelManage, 0, setting.UI.IssuePagingNum) | |||
if err := sess.Table("ai_model_manage").Where(cond). | |||
if err := sess.Table("ai_model_manage").Where(where). | |||
Find(&aiModelManages); err != nil { | |||
log.Info("error=" + err.Error()) | |||
return nil, 0, fmt.Errorf("Find: %v", err) | |||
} | |||
@@ -737,3 +734,43 @@ func QueryModelCollectedStatus(modelIds []string, userId int64) map[string]*AiMo | |||
} | |||
return result | |||
} | |||
func SaveModelFile(modelFile *AiModelFile) error { | |||
sess := x.NewSession() | |||
defer sess.Close() | |||
re, err := sess.Insert(modelFile) | |||
if err != nil { | |||
log.Info("insert modelFile error." + err.Error()) | |||
return err | |||
} | |||
log.Info("success to save modelFile db.re=" + fmt.Sprint((re))) | |||
return nil | |||
} | |||
func DeleteModelFile(modelFile *AiModelFile) error { | |||
sess := x.NewSession() | |||
defer sess.Close() | |||
re, err := sess.Delete(modelFile) | |||
if err != nil { | |||
log.Info("delete modelFile error." + err.Error()) | |||
return err | |||
} | |||
log.Info("success to delete modelFile db.re=" + fmt.Sprint((re))) | |||
return nil | |||
} | |||
func QueryModelFileByModelId(modelId string) []*AiModelFile { | |||
sess := x.NewSession() | |||
defer sess.Close() | |||
modelFileList := make([]*AiModelFile, 0) | |||
var cond = builder.NewCond() | |||
cond = cond.And( | |||
builder.Eq{"model_id": modelId}, | |||
) | |||
result := make([]*AiModelFile, 0) | |||
err := sess.Table(new(AiModelFile)).Where(cond).Find(&modelFileList) | |||
if err != nil { | |||
log.Info("query AiModelFile failed, err=" + err.Error()) | |||
} | |||
return result | |||
} |
@@ -206,6 +206,8 @@ type Cloudbrain struct { | |||
ModelVersion string //模型版本 | |||
CkptName string //权重文件名称 | |||
ModelId string //模型ID | |||
ModelRepoName string `xorm:"-"` | |||
ModelRepoOwnerName string `xorm:"-"` | |||
PreTrainModelUrl string //预训练模型地址 | |||
ResultUrl string //推理结果的obs路径 | |||
ResultJson string `xorm:"varchar(4000)"` | |||
@@ -1673,6 +1675,19 @@ type GrampusStopJobResponse struct { | |||
Status string `json:"status"` | |||
} | |||
type GrampusModelMigrateInfoResponse struct { | |||
GrampusResult | |||
DestBucket string `json:"destBucket"` | |||
DestEndpoint string `json:"destEndpoint"` | |||
DestObjectKey string `json:"destObjectKey"` | |||
DestProxy string `json:"destProxy"` | |||
FailedReason string `json:"failedReason"` | |||
SrcBucket string `json:"srcBucket"` | |||
SrcEndpoint string `json:"srcEndpoint"` | |||
SrcObjectKey string `json:"srcObjectKey"` | |||
Status int `json:"status"` //0:初始化 1:成功 2:失败 3:调度中 | |||
} | |||
type GetGrampusJobEventsResponse struct { | |||
GrampusResult | |||
JobEvents []GrampusJobEvents `json:"jobEvents"` | |||
@@ -1725,6 +1740,7 @@ type GrampusDataset struct { | |||
ObjectKey string `json:"objectKey"` | |||
ContainerPath string `json:"containerPath"` | |||
ReadOnly bool `json:"readOnly"` | |||
GetBackEndpoint string `json:"getBackEndpoint"` | |||
} | |||
type CreateGrampusJobRequest struct { | |||
@@ -2058,12 +2074,28 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { | |||
} | |||
} | |||
session.Commit() | |||
increaseModelReference(session, cloudbrain.ModelId) | |||
go IncreaseDatasetUseCount(cloudbrain.Uuid) | |||
go OperateRepoAITaskNum(cloudbrain.RepoID, 1) | |||
//go IncreaseModelRefernceCount(cloudbrain) | |||
return nil | |||
} | |||
func increaseModelReference(session *xorm.Session, modelId string) { | |||
if modelId != "" { | |||
log.Info("increase model count.") | |||
if _, err := session.Exec("UPDATE `ai_model_manage` SET reference_count = reference_count + 1 WHERE id = ?", modelId); err != nil { | |||
log.Info("err=" + err.Error()) | |||
} | |||
} | |||
} | |||
// func IncreaseModelRefernceCount(cloudbrain *Cloudbrain) { | |||
// if cloudbrain.ModelId != "" { | |||
// AddModelInferenceCount(cloudbrain.ModelId) | |||
// } | |||
// } | |||
func getRepoCloudBrain(cb *Cloudbrain) (*Cloudbrain, error) { | |||
has, err := x.Get(cb) | |||
if err != nil { | |||
@@ -2477,7 +2509,7 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { | |||
if err = sess.Commit(); err != nil { | |||
return err | |||
} | |||
increaseModelReference(sess, new.ModelId) | |||
go IncreaseDatasetUseCount(new.Uuid) | |||
return nil | |||
} | |||
@@ -2615,7 +2647,6 @@ func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { | |||
Join("left", "`user`", condition). | |||
Join("left", "cloudbrain_spec", "cloudbrain.id = cloudbrain_spec.cloudbrain_id"). | |||
Count(new(CloudbrainInfo)) | |||
} | |||
if err != nil { | |||
@@ -2972,12 +3003,12 @@ func LoadSpecs4CloudbrainInfo(tasks []*CloudbrainInfo) error { | |||
func GetCloudBrainByModelId(modelId string) ([]*Cloudbrain, error) { | |||
cloudBrains := make([]*Cloudbrain, 0) | |||
err := x.AllCols().Where("model_id=?", modelId).Find(&cloudBrains) | |||
err := x.AllCols().Where("model_id=?", modelId).OrderBy("created_unix asc").Find(&cloudBrains) | |||
return cloudBrains, err | |||
} | |||
func GetCloudBrainByRepoIdAndModelName(repoId int64, modelName string) ([]*Cloudbrain, error) { | |||
cloudBrains := make([]*Cloudbrain, 0) | |||
err := x.AllCols().Where("model_name=? and repo_id=?", modelName, repoId).Find(&cloudBrains) | |||
err := x.AllCols().Where("model_name=? and repo_id=?", modelName, repoId).OrderBy("created_unix asc").Find(&cloudBrains) | |||
return cloudBrains, err | |||
} |
@@ -0,0 +1,215 @@ | |||
package models | |||
import ( | |||
"code.gitea.io/gitea/modules/log" | |||
"errors" | |||
"time" | |||
"xorm.io/builder" | |||
"code.gitea.io/gitea/modules/timeutil" | |||
) | |||
type GrampusMigrateResponse int | |||
const ( | |||
GrampusMigrateResponseMigrateInit GrampusMigrateResponse = 0 | |||
GrampusMigrateResponseSuccess GrampusMigrateResponse = 1 | |||
GrampusMigrateResponseFailed GrampusMigrateResponse = 2 | |||
GrampusMigrateResponseMigrating GrampusMigrateResponse = 3 | |||
GrampusMigrateResponseNoNeedMigrate GrampusMigrateResponse = 4 | |||
) | |||
func (r GrampusMigrateResponse) ConvertToModelMigrateStep() ModelMigrateStep { | |||
switch r { | |||
case GrampusMigrateResponseMigrateInit: | |||
return GrampusMigrateInit | |||
case GrampusMigrateResponseSuccess: | |||
return GrampusMigrateSuccess | |||
case GrampusMigrateResponseFailed: | |||
return GrampusMigrateFailed | |||
case GrampusMigrateResponseMigrating: | |||
return GrampusMigrating | |||
case GrampusMigrateResponseNoNeedMigrate: | |||
return GrampusMigrateNoNeed | |||
} | |||
return -1 | |||
} | |||
type ModelMigrateStep int | |||
const ( | |||
GrampusMigrateInit ModelMigrateStep = 0 | |||
GrampusMigrating ModelMigrateStep = 1 | |||
GrampusMigrateSuccess ModelMigrateStep = 2 | |||
GrampusMigrateFailed ModelMigrateStep = 3 | |||
GrampusMigrateNoNeed ModelMigrateStep = 4 | |||
BucketMoving ModelMigrateStep = 10 | |||
BucketMoveSuccess ModelMigrateStep = 11 | |||
BucketMoveFailed ModelMigrateStep = 12 | |||
) | |||
func (m ModelMigrateStep) GetStatus() ModelMigrateStatus { | |||
switch m { | |||
case BucketMoveSuccess, GrampusMigrateNoNeed: | |||
return ModelMigrateSuccess | |||
case GrampusMigrateFailed, BucketMoveFailed: | |||
return ModelMigrateFailed | |||
case GrampusMigrateInit: | |||
return ModelMigrateWaiting | |||
case GrampusMigrateSuccess, GrampusMigrating, BucketMoving: | |||
return ModelMigrating | |||
} | |||
return -1 | |||
} | |||
type ModelMigrateStatus int | |||
const ( | |||
ModelMigrateSuccess ModelMigrateStatus = 0 | |||
ModelMigrating ModelMigrateStatus = 1 | |||
ModelMigrateFailed ModelMigrateStatus = 2 | |||
ModelMigrateWaiting ModelMigrateStatus = 3 | |||
) | |||
var UnFinishedMigrateSteps = []ModelMigrateStep{GrampusMigrateInit, GrampusMigrating, GrampusMigrateSuccess, BucketMoving} | |||
type ModelMigrateRecord struct { | |||
ID int64 `xorm:"pk autoincr"` | |||
CloudbrainID int64 `xorm:"INDEX NOT NULL unique"` | |||
DestBucket string | |||
DestEndpoint string | |||
DestObjectKey string | |||
DestProxy string | |||
SrcBucket string | |||
SrcEndpoint string | |||
SrcObjectKey string | |||
Status ModelMigrateStatus `xorm:"NOT NULL DEFAULT 3"` | |||
CurrentStep ModelMigrateStep `xorm:"NOT NULL DEFAULT 0"` | |||
RetryCount int | |||
CreatedUnix timeutil.TimeStamp `xorm:"created"` | |||
UpdatedUnix timeutil.TimeStamp `xorm:"updated"` | |||
DeletedAt time.Time `xorm:"deleted"` | |||
Remark string | |||
} | |||
func (r *ModelMigrateRecord) IsFinished() bool { | |||
for _, s := range UnFinishedMigrateSteps { | |||
if s == r.CurrentStep { | |||
return false | |||
} | |||
} | |||
return true | |||
} | |||
func updateModelMigrateRecordCols(e Engine, record *ModelMigrateRecord, cols ...string) error { | |||
_, err := e.ID(record.ID).Cols(cols...).Update(record) | |||
return err | |||
} | |||
func UpdateModelMigrateRecordCols(record *ModelMigrateRecord, cols ...string) error { | |||
return updateModelMigrateRecordCols(x, record, cols...) | |||
} | |||
func IncreaseModelMigrateRetryCount(recordId int64) error { | |||
_, err := x.ID(recordId).Incr("retry_count", 1).Update(&ModelMigrateRecord{}) | |||
return err | |||
} | |||
func UpdateModelMigrateStatusByStep(record *ModelMigrateRecord, newStep ModelMigrateStep) error { | |||
status := newStep.GetStatus() | |||
if status < 0 { | |||
log.Error("Step format error.id = %d,newStep = %d", record.ID, newStep) | |||
return errors.New("Step format error") | |||
} | |||
record.Status = status | |||
record.CurrentStep = newStep | |||
//正常情况下状态只能向更大的状态更新 | |||
n, err := x.Where(builder.NewCond().And(builder.Eq{"id": record.ID}). | |||
And(builder.Lt{"current_step": newStep})). | |||
Cols("status", "current_step"). | |||
Update(record) | |||
if err != nil { | |||
log.Error("UpdateModelMigrateStatusByStep err.%v", err) | |||
return err | |||
} | |||
if n == 0 { | |||
log.Error("UpdateModelMigrateStatusByStep total num is 0.r.ID=%d", record.ID) | |||
return errors.New("current_step not valid") | |||
} | |||
return nil | |||
} | |||
func RollBackMigrateStatus(record *ModelMigrateRecord, newStep ModelMigrateStep) error { | |||
status := newStep.GetStatus() | |||
if status < 0 { | |||
log.Error("Step format error.id = %d,newStep = %d", record.ID, newStep) | |||
return errors.New("Step format error") | |||
} | |||
record.Status = status | |||
record.CurrentStep = newStep | |||
_, err := x.ID(record.ID). | |||
Cols("status", "current_step"). | |||
Update(record) | |||
if err != nil { | |||
log.Error("RollBackMigrateStatus err.%v", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func UpdateModelMigrateRecordByStep(record *ModelMigrateRecord) error { | |||
n, err := x. | |||
Where(builder.NewCond().And(builder.Eq{"id": record.ID})). | |||
Update(record) | |||
if err != nil { | |||
log.Error("UpdateModelMigrateRecordByStep err. ID=%d err=%v", record.ID, err) | |||
return err | |||
} | |||
if n == 0 { | |||
log.Error("UpdateModelMigrateRecordByStep total num is 0.r.ID=%d", record.ID) | |||
return errors.New("current_step not valid") | |||
} | |||
return nil | |||
} | |||
func GetUnfinishedModelMigrateRecords() ([]*ModelMigrateRecord, error) { | |||
records := make([]*ModelMigrateRecord, 0, 10) | |||
return records, x. | |||
Where(builder.NewCond().And(builder.In("current_step", UnFinishedMigrateSteps))). | |||
Limit(100). | |||
Find(&records) | |||
} | |||
func InsertModelMigrateRecord(record *ModelMigrateRecord) (_ *ModelMigrateRecord, err error) { | |||
if _, err := x.Insert(record); err != nil { | |||
return nil, err | |||
} | |||
return record, nil | |||
} | |||
func GetModelMigrateRecordByCloudbrainId(cloudbrainId int64) (*ModelMigrateRecord, error) { | |||
r := &ModelMigrateRecord{} | |||
if has, err := x.Where("cloudbrain_id = ?", cloudbrainId).Get(r); err != nil { | |||
log.Error("GetModelMigrateRecordByCloudbrainId err. %v", err) | |||
return nil, err | |||
} else if !has { | |||
return nil, ErrRecordNotExist{} | |||
} | |||
return r, nil | |||
} | |||
func GetModelMigrateRecordById(id int64) (*ModelMigrateRecord, error) { | |||
r := &ModelMigrateRecord{} | |||
if has, err := x.ID(id).Get(r); err != nil { | |||
log.Error("GetModelMigrateRecordByCloudbrainId err. %v", err) | |||
return nil, err | |||
} else if !has { | |||
return nil, ErrRecordNotExist{} | |||
} | |||
return r, nil | |||
} |
@@ -171,6 +171,8 @@ func init() { | |||
new(RepoConvergeInfo), | |||
new(UserRole), | |||
new(AiModelCollect), | |||
new(AiModelFile), | |||
new(ModelMigrateRecord), | |||
) | |||
tablesStatistic = append(tablesStatistic, | |||
@@ -1706,7 +1706,9 @@ func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err e | |||
if err != nil { | |||
return err | |||
} | |||
_, err = e.Where("repo_id = ?", repo.ID).Cols("is_private").Update(&AiModelManage{ | |||
IsPrivate: true, | |||
}) | |||
} else { | |||
//If repo has become public, we need set dataset to public | |||
_, err = e.Where("repo_id = ? and status <> 2", repo.ID).Cols("status").Update(&Dataset{ | |||
@@ -1870,6 +1872,9 @@ func DeleteRepository(doer *User, uid, repoID int64) error { | |||
// Delete dataset attachment record and remove related files | |||
deleteDatasetAttachmentByRepoId(sess, repoID) | |||
deleteModelByRepoId(repoID) | |||
if err = deleteBeans(sess, | |||
&Access{RepoID: repo.ID}, | |||
&Action{RepoID: repo.ID}, | |||
@@ -2055,6 +2060,22 @@ func DeleteRepository(doer *User, uid, repoID int64) error { | |||
return nil | |||
} | |||
func deleteModelByRepoId(repoId int64) { | |||
models := QueryModelByRepoId(repoId) | |||
if models != nil { | |||
for _, model := range models { | |||
log.Info("bucket=" + setting.Bucket + " path=" + model.Path) | |||
if len(model.Path) > (len(setting.Bucket) + 1) { | |||
err := storage.ObsRemoveObject(setting.Bucket, model.Path[len(setting.Bucket)+1:]) | |||
if err != nil { | |||
log.Info("Failed to delete model. id=" + model.ID) | |||
} | |||
} | |||
} | |||
} | |||
DeleteModelByRepoId(repoId) | |||
} | |||
func deleteDatasetAttachmentByRepoId(sess *xorm.Session, repoId int64) error { | |||
attachments := make([]*Attachment, 0) | |||
if err := sess.Join("INNER", "dataset", "dataset.id = attachment.dataset_id"). | |||
@@ -69,7 +69,7 @@ type CreateCloudBrainInferencForm struct { | |||
JobType string `form:"job_type" binding:"Required"` | |||
BenchmarkCategory string `form:"get_benchmark_category"` | |||
GpuType string `form:"gpu_type"` | |||
TrainUrl string `form:"train_url"` | |||
PreTrainModelUrl string `form:"pre_train_model_url"` | |||
TestUrl string `form:"test_url"` | |||
Description string `form:"description"` | |||
ResourceSpecId int `form:"resource_spec_id" binding:"Required"` | |||
@@ -81,7 +81,7 @@ type CreateModelArtsInferenceJobForm struct { | |||
FlavorName string `form:"flaver_names" binding:"Required"` | |||
EngineName string `form:"engine_names" binding:"Required"` | |||
LabelName string `form:"label_names" binding:"Required"` | |||
TrainUrl string `form:"train_url" binding:"Required"` | |||
PreTrainModelUrl string `form:"pre_train_model_url" binding:"Required"` | |||
ModelName string `form:"model_name" binding:"Required"` | |||
ModelVersion string `form:"model_version" binding:"Required"` | |||
CkptName string `form:"ckpt_name" binding:"Required"` | |||
@@ -372,9 +372,6 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { | |||
if err != nil { | |||
return "", err | |||
} | |||
if req.ModelId != "" { | |||
models.AddModelInferenceCount(req.ModelId) | |||
} | |||
task, err := models.GetCloudbrainByJobID(jobID) | |||
if err != nil { | |||
@@ -5,6 +5,7 @@ | |||
package cron | |||
import ( | |||
"code.gitea.io/gitea/services/ai_task_service/schedule" | |||
"context" | |||
"time" | |||
@@ -249,6 +250,17 @@ func registerHandleScheduleRecord() { | |||
}) | |||
} | |||
func registerHandleModelMigrateRecord() { | |||
RegisterTaskFatal("handle_model_migrate_record", &BaseConfig{ | |||
Enabled: true, | |||
RunAtStart: false, | |||
Schedule: "@every 1m", | |||
}, func(ctx context.Context, _ *models.User, _ Config) error { | |||
schedule.HandleUnfinishedMigrateRecords() | |||
return nil | |||
}) | |||
} | |||
func registerRewardPeriodTask() { | |||
RegisterTaskFatal("reward_period_task", &BaseConfig{ | |||
Enabled: true, | |||
@@ -335,4 +347,6 @@ func initBasicTasks() { | |||
registerHandleScheduleRecord() | |||
registerHandleCloudbrainDurationStatistic() | |||
registerHandleModelMigrateRecord() | |||
} |
@@ -354,6 +354,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str | |||
EndPoint: getEndPoint(), | |||
ObjectKey: req.CodeObsPath + cloudbrain.DefaultBranchName + ".zip", | |||
} | |||
outputGrampus = models.GrampusDataset{ | |||
GetBackEndpoint: getEndPoint(), | |||
} | |||
} else if ProcessorTypeGPU == req.ProcessType { | |||
datasetGrampus = getDatasetGPUGrampus(req.DatasetInfos, "/tmp/dataset") | |||
if len(req.ModelName) != 0 { | |||
@@ -378,6 +381,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str | |||
} | |||
outputGrampus = models.GrampusDataset{ | |||
ContainerPath: "/tmp/output", | |||
GetBackEndpoint: setting.Attachment.Minio.Endpoint, | |||
} | |||
} else if ProcessorTypeGCU == req.ProcessType { | |||
@@ -404,8 +408,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str | |||
} | |||
outputGrampus = models.GrampusDataset{ | |||
ContainerPath: "/tmp/output", | |||
GetBackEndpoint: setting.Attachment.Minio.Endpoint, | |||
} | |||
} | |||
modelGrampusJson, _ := json.Marshal(modelGrampus) | |||
@@ -393,6 +393,72 @@ sendjob: | |||
return &result, nil | |||
} | |||
func PostModelMigrate(jobID string) (*models.GrampusModelMigrateInfoResponse, error) { | |||
checkSetting() | |||
client := getRestyClient() | |||
var result models.GrampusModelMigrateInfoResponse | |||
retry := 0 | |||
sendjob: | |||
res, err := client.R(). | |||
//SetHeader("Content-Type", "application/json"). | |||
SetAuthToken(TOKEN). | |||
SetResult(&result). | |||
Post(HOST + urlTrainJob + "/" + jobID + "/modelMigrate") | |||
if err != nil { | |||
return &result, fmt.Errorf("resty ModelMigrate: %v", err) | |||
} | |||
log.Info("call modelMigrate res=%+v", res) | |||
if result.ErrorCode == errorIllegalToken && retry < 1 { | |||
retry++ | |||
log.Info("retry get token") | |||
_ = getToken() | |||
goto sendjob | |||
} | |||
if result.ErrorCode != 0 { | |||
log.Error("ModelMigrate failed(%d): %s", result.ErrorCode, result.ErrorMsg) | |||
return &result, fmt.Errorf("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) | |||
} | |||
return &result, nil | |||
} | |||
func ModelMigrateInfo(jobID string) (*models.GrampusModelMigrateInfoResponse, error) { | |||
checkSetting() | |||
client := getRestyClient() | |||
var result models.GrampusModelMigrateInfoResponse | |||
retry := 0 | |||
sendjob: | |||
res, err := client.R(). | |||
//SetHeader("Content-Type", "application/json"). | |||
SetAuthToken(TOKEN). | |||
SetResult(&result). | |||
Get(HOST + urlTrainJob + "/" + jobID + "/modelMigrateInfo") | |||
if err != nil { | |||
return &result, fmt.Errorf("resty ModelMigrateInfo: %v", err) | |||
} | |||
log.Info("call modelMigrateInfo res=%+v", res) | |||
if result.ErrorCode == errorIllegalToken && retry < 1 { | |||
retry++ | |||
log.Info("retry get token") | |||
_ = getToken() | |||
goto sendjob | |||
} | |||
if result.ErrorCode != 0 { | |||
log.Error("ModelMigrateInfo failed(%d): %s", result.ErrorCode, result.ErrorMsg) | |||
return &result, fmt.Errorf("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) | |||
} | |||
return &result, nil | |||
} | |||
func GetAiCenters(pageIndex, pageSize int) (*models.GetGrampusAiCentersResult, error) { | |||
checkSetting() | |||
client := getRestyClient() | |||
@@ -41,7 +41,8 @@ type ASTTransformer struct{} | |||
func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { | |||
metaData := meta.GetItems(pc) | |||
firstChild := node.FirstChild() | |||
createTOC := false | |||
createTOC := setting.Markdown.EnableToc | |||
var toc = []Header{} | |||
rc := &RenderConfig{ | |||
Meta: "", | |||
@@ -154,6 +155,25 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa | |||
} | |||
return ast.WalkContinue, nil | |||
}) | |||
if node.HasChildren() { | |||
children := make([]ast.Node, 0, node.ChildCount()) | |||
child := node.FirstChild() | |||
for child != nil { | |||
children = append(children, child) | |||
child = child.NextSibling() | |||
} | |||
node.RemoveChildren(node) | |||
contentNode := ast.NewDocument() | |||
contentNode.SetAttributeString("class", []byte("markdown-content")) | |||
for _, child := range children { | |||
contentNode.AppendChild(contentNode, child) | |||
} | |||
node.AppendChild(node, contentNode) | |||
} | |||
if createTOC && len(toc) > 0 { | |||
lang := rc.Lang | |||
@@ -161,6 +181,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa | |||
lang = setting.Langs[0] | |||
} | |||
tocNode := createTOCNode(toc, lang) | |||
firstChild = node.FirstChild() | |||
if tocNode != nil { | |||
node.InsertBefore(node, firstChild, tocNode) | |||
} | |||
@@ -248,12 +269,14 @@ func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { | |||
func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { | |||
n := node.(*ast.Document) | |||
if val, has := n.AttributeString("lang"); has { | |||
var err error | |||
if entering { | |||
_, err = w.WriteString("<div") | |||
if err == nil { | |||
_, err = w.WriteString(fmt.Sprintf(` lang=%q`, val)) | |||
if n.Attributes() != nil { | |||
html.RenderAttributes(w, n, html.GlobalAttributeFilter) | |||
} | |||
} | |||
if err == nil { | |||
_, err = w.WriteRune('>') | |||
@@ -265,7 +288,25 @@ func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast. | |||
if err != nil { | |||
return ast.WalkStop, err | |||
} | |||
/**if val, has := n.AttributeString("lang"); has { | |||
var err error | |||
if entering { | |||
_, err = w.WriteString("<div") | |||
if err == nil { | |||
_, err = w.WriteString(fmt.Sprintf(` lang=%q`, val)) | |||
} | |||
if err == nil { | |||
_, err = w.WriteRune('>') | |||
} | |||
} else { | |||
_, err = w.WriteString("</div>") | |||
} | |||
if err != nil { | |||
return ast.WalkStop, err | |||
} | |||
}*/ | |||
return ast.WalkContinue, nil | |||
} | |||
@@ -8,18 +8,24 @@ import ( | |||
"fmt" | |||
"net/url" | |||
"github.com/unknwon/i18n" | |||
"github.com/yuin/goldmark/ast" | |||
) | |||
func createTOCNode(toc []Header, lang string) ast.Node { | |||
details := NewDetails() | |||
summary := NewSummary() | |||
summary.AppendChild(summary, ast.NewString([]byte(i18n.Tr(lang, "toc")))) | |||
details.AppendChild(details, summary) | |||
sidebar := ast.NewDocument() | |||
sidebar.SetAttributeString("class", []byte("markdown_catalog")) | |||
scrollContainer := ast.NewDocument() | |||
scrollContainer.SetAttributeString("class", []byte("scroll-container")) | |||
toggleContainer := ast.NewDocument() | |||
toggleContainer.SetAttributeString("class", []byte("toggle-container")) | |||
toggleIcon := ast.NewDocument() | |||
toggleIcon.SetAttributeString("class", []byte("icon ri-arrow-drop-left-line")) | |||
container := ast.NewDocument() | |||
container.SetAttributeString("class", []byte("container")) | |||
ul := ast.NewList('-') | |||
details.AppendChild(details, ul) | |||
ul.SetAttributeString("class", []byte("markdown_toc")) | |||
topul:=ul | |||
currentLevel := 6 | |||
for _, header := range toc { | |||
if header.Level < currentLevel { | |||
@@ -28,22 +34,34 @@ func createTOCNode(toc []Header, lang string) ast.Node { | |||
} | |||
for _, header := range toc { | |||
for currentLevel > header.Level { | |||
ul = ul.Parent().(*ast.List) | |||
ul = ul.Parent().Parent().(*ast.List) | |||
currentLevel-- | |||
} | |||
for currentLevel < header.Level { | |||
newLi := ast.NewListItem(currentLevel * 2) | |||
newLi.SetAttributeString("class", []byte("no-catalog-li")) | |||
newL := ast.NewList('-') | |||
ul.AppendChild(ul, newL) | |||
newL.SetAttributeString("class", []byte("markdown_toc")) | |||
newLi.AppendChild(newLi, newL) | |||
ul.AppendChild(ul, newLi) | |||
currentLevel++ | |||
ul = newL | |||
} | |||
li := ast.NewListItem(currentLevel * 2) | |||
li.SetAttributeString("class", []byte("catalog-li")) | |||
a := ast.NewLink() | |||
a.Destination = []byte(fmt.Sprintf("#%s", url.PathEscape(header.ID))) | |||
a.AppendChild(a, ast.NewString([]byte(header.Text))) | |||
a.SetAttributeString("title", []byte(header.Text)) | |||
li.AppendChild(li, a) | |||
ul.AppendChild(ul, li) | |||
} | |||
container.AppendChild(container,topul) | |||
scrollContainer.AppendChild(scrollContainer,container) | |||
toggleContainer.AppendChild(toggleContainer,toggleIcon) | |||
sidebar.AppendChild(sidebar, toggleContainer) | |||
sidebar.AppendChild(sidebar, scrollContainer) | |||
return details | |||
//return details | |||
return sidebar | |||
} |
@@ -69,7 +69,7 @@ func ReplaceSanitizer() { | |||
sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^((icon(\s+[\p{L}\p{N}_-]+)+)|(ui checkbox)|(ui checked checkbox)|(emoji))$`)).OnElements("span") | |||
// Allow generally safe attributes | |||
generalSafeAttrs := []string{"abbr", "accept", "accept-charset", | |||
generalSafeAttrs := []string{"abbr", "accept", "accept-charset", "class", | |||
"accesskey", "action", "align", "alt", | |||
"aria-describedby", "aria-hidden", "aria-label", "aria-labelledby", | |||
"axis", "border", "cellpadding", "cellspacing", "char", | |||
@@ -0,0 +1,41 @@ | |||
package model_schedule | |||
import ( | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/notification/base" | |||
) | |||
type scheduleNotifier struct { | |||
base.NullNotifier | |||
} | |||
var ( | |||
_ base.Notifier = &scheduleNotifier{} | |||
) | |||
// NewNotifier create a new wechatNotifier notifier | |||
func NewNotifier() base.Notifier { | |||
return &scheduleNotifier{} | |||
} | |||
func (*scheduleNotifier) NotifyChangeCloudbrainStatus(cloudbrain *models.Cloudbrain, oldStatus string) { | |||
if !cloudbrain.IsTerminal() { | |||
return | |||
} | |||
log.Info("try to InsertModelMigrateRecord.cloudbrainId=%d oldStatus=%s newStatus=%d", cloudbrain.ID, oldStatus, cloudbrain.Status) | |||
switch cloudbrain.Type { | |||
case models.TypeC2Net: | |||
if cloudbrain.JobType == string(models.JobTypeDebug) { | |||
return | |||
} | |||
_, err := models.InsertModelMigrateRecord(&models.ModelMigrateRecord{ | |||
CloudbrainID: cloudbrain.ID, | |||
Status: models.ModelMigrating, | |||
CurrentStep: models.GrampusMigrating, | |||
}) | |||
if err != nil { | |||
log.Error("InsertModelMigrateRecord err.cloudbrain.id=%d err=%v", cloudbrain.ID, err) | |||
} | |||
} | |||
} |
@@ -11,6 +11,7 @@ import ( | |||
"code.gitea.io/gitea/modules/notification/base" | |||
"code.gitea.io/gitea/modules/notification/indexer" | |||
"code.gitea.io/gitea/modules/notification/mail" | |||
"code.gitea.io/gitea/modules/notification/model_schedule" | |||
"code.gitea.io/gitea/modules/notification/reward" | |||
"code.gitea.io/gitea/modules/notification/ui" | |||
"code.gitea.io/gitea/modules/notification/webhook" | |||
@@ -41,6 +42,7 @@ func NewContext() { | |||
RegisterNotifier(action.NewNotifier()) | |||
RegisterNotifier(wechatNotifier.NewNotifier()) | |||
RegisterNotifier(reward.NewNotifier()) | |||
RegisterNotifier(model_schedule.NewNotifier()) | |||
} | |||
// NotifyUploadAttachment notifies attachment upload message to notifiers | |||
@@ -0,0 +1,9 @@ | |||
package redis_key | |||
import "fmt" | |||
const MODEL_SCHEDULE_PREFIX = "model_schedule" | |||
func RecordHandleLock(jobId string) string { | |||
return KeyJoin(MODEL_SCHEDULE_PREFIX, fmt.Sprint(jobId), "handle") | |||
} |
@@ -312,9 +312,11 @@ var ( | |||
EnableHardLineBreak bool | |||
CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` | |||
FileExtensions []string | |||
EnableToc bool | |||
}{ | |||
EnableHardLineBreak: true, | |||
FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd", ","), | |||
EnableToc: true, | |||
} | |||
// Admin settings | |||
@@ -677,6 +679,9 @@ var ( | |||
DeductTaskRangeForFirst time.Duration | |||
DeductTaskMinTimestamp int64 | |||
//model-migrate config | |||
UseLocalMinioMigrate bool | |||
//badge config | |||
BadgeIconMaxFileSize int64 | |||
BadgeIconMaxWidth int | |||
@@ -1679,6 +1684,9 @@ func NewContext() { | |||
DeductTaskRangeForFirst = sec.Key("DEDUCT_TASK_RANGE_FOR_FIRST").MustDuration(3 * time.Hour) | |||
DeductTaskMinTimestamp = sec.Key("DEDUCT_TASK_MIN_TIMESTAMP").MustInt64(0) | |||
sec = Cfg.Section("model-migrate") | |||
UseLocalMinioMigrate = sec.Key("USE_LOCAL_MINIO_MIGRATE").MustBool(false) | |||
sec = Cfg.Section("icons") | |||
BadgeIconMaxFileSize = sec.Key("BADGE_ICON_MAX_FILE_SIZE").MustInt64(1048576) | |||
BadgeIconMaxWidth = sec.Key("BADGE_ICON_MAX_WIDTH").MustInt(4096) | |||
@@ -371,11 +371,13 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative | |||
for { | |||
output, err := ObsCli.ListObjects(input) | |||
if err == nil { | |||
log.Info("Page:%d\n", index) | |||
log.Info("Page:%d\n input.Prefix=v%", index, input.Prefix) | |||
log.Info("input.Prefix=" + input.Prefix) | |||
index++ | |||
for _, val := range output.Contents { | |||
var isDir bool | |||
var fileName string | |||
log.Info("val.key=" + val.Key) | |||
if val.Key == input.Prefix { | |||
continue | |||
} | |||
@@ -418,6 +420,53 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative | |||
return fileInfos, nil | |||
} | |||
func GetOneLevelObjectsUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { | |||
input := &obs.ListObjectsInput{} | |||
input.Bucket = bucket | |||
input.Prefix = prefixRootPath + relativePath | |||
input.Delimiter = "/" | |||
if !strings.HasSuffix(input.Prefix, "/") { | |||
input.Prefix += "/" | |||
} | |||
fileInfos := make([]FileInfo, 0) | |||
prefixLen := len(input.Prefix) | |||
index := 1 | |||
output, err := ObsCli.ListObjects(input) | |||
if err != nil { | |||
if obsError, ok := err.(obs.ObsError); ok { | |||
log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) | |||
} | |||
return nil, err | |||
} | |||
log.Info("Page:%d\n", index) | |||
index++ | |||
for _, val := range output.Contents { | |||
var fileName string | |||
if val.Key == input.Prefix { | |||
continue | |||
} | |||
fileName = val.Key[prefixLen:] | |||
fileInfo := FileInfo{ | |||
ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), | |||
FileName: fileName, | |||
Size: val.Size, | |||
IsDir: false, | |||
ParenDir: relativePath, | |||
} | |||
fileInfos = append(fileInfos, fileInfo) | |||
} | |||
for _, val := range output.CommonPrefixes { | |||
fileName := strings.TrimSuffix(strings.TrimPrefix(val, input.Prefix), "/") | |||
fileInfo := FileInfo{ | |||
FileName: fileName, | |||
IsDir: true, | |||
ParenDir: strings.TrimPrefix(val, prefixRootPath), | |||
} | |||
fileInfos = append(fileInfos, fileInfo) | |||
} | |||
return fileInfos, nil | |||
} | |||
func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { | |||
input := &obs.ListObjectsInput{} | |||
input.Bucket = bucket | |||
@@ -1,18 +1,15 @@ | |||
package urchin | |||
import ( | |||
"encoding/json" | |||
"fmt" | |||
"strings" | |||
"time" | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/grampus" | |||
"code.gitea.io/gitea/modules/labelmsg" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/storage" | |||
"encoding/json" | |||
"fmt" | |||
"github.com/minio/minio-go" | |||
"strings" | |||
) | |||
type DecompressReq struct { | |||
@@ -30,253 +27,6 @@ func getUrfsClient() { | |||
urfsClient = New() | |||
} | |||
func GetAITaskOutPutBack(cloudbrainID int64, jobName, centerId, computerResource string) error { | |||
switch computerResource { | |||
case models.NPUResource: | |||
return GetNPUDataBack(cloudbrainID, jobName, centerId) | |||
case models.GPUResource: | |||
return GetGPUDataBack(cloudbrainID, jobName, centerId) | |||
case models.GCUResource: | |||
return GetGCUDataBack(cloudbrainID, jobName, centerId) | |||
} | |||
return nil | |||
} | |||
func GetGPUDataBack(cloudbrainID int64, jobName, centerId string) error { | |||
endpoint := grampus.GetRemoteEndPoint(centerId) | |||
bucket := grampus.BucketRemote | |||
objectKey := grampus.GetGPUModelObjectKey4Grampus(jobName) | |||
destPeerHost := grampus.GetCenterProxy(setting.Grampus.GPULocalCenterID) | |||
getUrfsClient() | |||
var res *PeerResult | |||
var err error | |||
var retryIntervalList = []time.Duration{1 * time.Minute, 1 * time.Minute, 3 * time.Minute} | |||
for i, retryInterval := range retryIntervalList { | |||
res, err = urfsClient.ScheduleDirToPeerByKey(endpoint, bucket, objectKey, destPeerHost) | |||
if err == nil { | |||
log.Info("ScheduleDirToPeerByKey res=%v", res) | |||
break | |||
} | |||
log.Error("ScheduleDataToPeerByKey failed:%v, ObjectKey is:%s,retry in %v", err, objectKey, retryInterval) | |||
time.Sleep(retryInterval) | |||
// If it's the last retry, break | |||
if i == len(retryIntervalList)-1 { | |||
res, err = urfsClient.ScheduleDirToPeerByKey(endpoint, bucket, objectKey, destPeerHost) | |||
} | |||
} | |||
// If err is still not nil after retrying, insert a default value | |||
if err != nil { | |||
log.Error("ScheduleDataToPeerByKey failed info is EndPoint:%s,Bucket:%s,ObjectKey:%s,ProxyServer:%s,TargetObjectKey:%s,error:%v", | |||
endpoint, bucket, objectKey, destPeerHost, grampus.GetGPUModelObjectKey(jobName), err) | |||
_, err = models.InsertScheduleRecord(&models.ScheduleRecord{ | |||
CloudbrainID: cloudbrainID, | |||
EndPoint: endpoint, | |||
Bucket: bucket, | |||
ObjectKey: objectKey, | |||
ProxyServer: destPeerHost, | |||
Status: models.StorageUrchinScheduleFailed, | |||
IsDir: true, | |||
ComputeSource: models.GPUResource, | |||
TargetObjectKey: grampus.GetGPUModelObjectKey(jobName), | |||
Remark: interceptErrorMessages(err), | |||
LocalOperateStatus: models.MoveBucketWaiting, | |||
}) | |||
if err != nil { | |||
log.Error("InsertScheduleRecord failed:%v", err) | |||
return err | |||
} | |||
return fmt.Errorf("GetBackModel failed after retrying:%v", err) | |||
} | |||
_, err = models.InsertScheduleRecord(&models.ScheduleRecord{ | |||
CloudbrainID: cloudbrainID, | |||
EndPoint: endpoint, | |||
Bucket: bucket, | |||
ObjectKey: objectKey, | |||
ProxyServer: destPeerHost, | |||
Status: res.StatusCode, | |||
IsDir: true, | |||
ComputeSource: models.GPUResource, | |||
TargetObjectKey: grampus.GetGPUModelObjectKey(jobName), | |||
LocalOperateStatus: models.MoveBucketWaiting, | |||
}) | |||
if err != nil { | |||
log.Error("InsertScheduleRecord failed:%v", err) | |||
return err | |||
} | |||
r, err := models.GetScheduleRecordByCloudbrainID(cloudbrainID) | |||
if err != nil { | |||
log.Error("GetScheduleRecordByCloudbrainID err.cloudbrainID=%d err=%v", cloudbrainID, err) | |||
return err | |||
} | |||
err = handleScheduleResult(r, res) | |||
if err != nil { | |||
log.Error("GetGPUDataBack handleScheduleResult err.%v", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func GetGCUDataBack(cloudbrainID int64, jobName, centerId string) error { | |||
endpoint := grampus.GetRemoteEndPoint(centerId) | |||
bucket := grampus.BucketRemote | |||
objectKey := grampus.GetGPUModelObjectKey4Grampus(jobName) | |||
destPeerHost := grampus.GetCenterProxy(setting.Grampus.GPULocalCenterID) | |||
getUrfsClient() | |||
var res *PeerResult | |||
var err error | |||
var retryIntervalList = []time.Duration{1 * time.Minute, 1 * time.Minute, 3 * time.Minute} | |||
for i, retryInterval := range retryIntervalList { | |||
res, err = urfsClient.ScheduleDirToPeerByKey(endpoint, bucket, objectKey, destPeerHost) | |||
if err == nil { | |||
break | |||
} | |||
log.Error("ScheduleDataToPeerByKey failed:%v, retry in %v", err, retryInterval) | |||
time.Sleep(retryInterval) | |||
// If it's the last retry, break | |||
if i == len(retryIntervalList)-1 { | |||
res, err = urfsClient.ScheduleDirToPeerByKey(endpoint, bucket, objectKey, destPeerHost) | |||
} | |||
} | |||
// If err is still not nil after retrying, insert a default value | |||
if err != nil { | |||
log.Error("ScheduleDataToPeerByKey failed info is EndPoint:%s,Bucket:%s,ObjectKey:%s,ProxyServer:%s,TargetObjectKey:%s,error:%v", | |||
endpoint, bucket, objectKey, destPeerHost, grampus.GetGPUModelObjectKey(jobName), err) | |||
_, err = models.InsertScheduleRecord(&models.ScheduleRecord{ | |||
CloudbrainID: cloudbrainID, | |||
EndPoint: endpoint, | |||
Bucket: bucket, | |||
ObjectKey: objectKey, | |||
ProxyServer: destPeerHost, | |||
Status: models.StorageUrchinScheduleFailed, | |||
IsDir: true, | |||
ComputeSource: models.GCUResource, | |||
TargetObjectKey: grampus.GetGPUModelObjectKey(jobName), | |||
LocalOperateStatus: models.MoveBucketWaiting, | |||
}) | |||
if err != nil { | |||
log.Error("InsertScheduleRecord failed:%v", err) | |||
return err | |||
} | |||
return fmt.Errorf("GetBackModel failed after retrying:%v", err) | |||
} | |||
_, err = models.InsertScheduleRecord(&models.ScheduleRecord{ | |||
CloudbrainID: cloudbrainID, | |||
EndPoint: endpoint, | |||
Bucket: bucket, | |||
ObjectKey: objectKey, | |||
ProxyServer: destPeerHost, | |||
Status: res.StatusCode, | |||
IsDir: true, | |||
ComputeSource: models.GCUResource, | |||
TargetObjectKey: grampus.GetGPUModelObjectKey(jobName), | |||
LocalOperateStatus: models.MoveBucketWaiting, | |||
}) | |||
if err != nil { | |||
log.Error("InsertScheduleRecord failed:%v", err) | |||
return err | |||
} | |||
r, err := models.GetScheduleRecordByCloudbrainID(cloudbrainID) | |||
if err != nil { | |||
log.Error("GetScheduleRecordByCloudbrainID err.cloudbrainID=%d err=%v", cloudbrainID, err) | |||
return err | |||
} | |||
err = handleScheduleResult(r, res) | |||
if err != nil { | |||
log.Error("GetGCUDataBack handleScheduleResult err.%v", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func GetNPUDataBack(cloudbrainID int64, jobName, centerId string) error { | |||
endpoint := grampus.GetRemoteEndPoint(centerId) | |||
bucket := grampus.BucketRemote | |||
objectKey := grampus.GetNpuModelObjectKey(jobName) | |||
destPeerHost := grampus.GetCenterProxy(setting.Grampus.LocalCenterID) | |||
getUrfsClient() | |||
var res *PeerResult | |||
var err error | |||
var retryIntervalList = []time.Duration{1 * time.Minute, 1 * time.Minute, 3 * time.Minute} | |||
for i, retryInterval := range retryIntervalList { | |||
res, err = urfsClient.ScheduleDataToPeerByKey(endpoint, bucket, objectKey, destPeerHost) | |||
if err == nil { | |||
break | |||
} | |||
log.Error("ScheduleDataToPeerByKey failed:%v, ObjectKey is:%s,retry in %v", err, objectKey, retryInterval) | |||
time.Sleep(retryInterval) | |||
// If it's the last retry, break | |||
if i == len(retryIntervalList)-1 { | |||
res, err = urfsClient.ScheduleDataToPeerByKey(endpoint, bucket, objectKey, destPeerHost) | |||
} | |||
} | |||
// If err is still not nil after retrying, insert a default value | |||
if err != nil { | |||
log.Error("ScheduleDataToPeerByKey failed after retrying, errorInfo is EndPoint:%s,Bucket:%s,ObjectKey:%s,ProxyServer:%s,error:%v", | |||
endpoint, bucket, objectKey, destPeerHost, err) | |||
_, err = models.InsertScheduleRecord(&models.ScheduleRecord{ | |||
CloudbrainID: cloudbrainID, | |||
EndPoint: endpoint, | |||
Bucket: bucket, | |||
ObjectKey: objectKey, | |||
ProxyServer: destPeerHost, | |||
Status: models.StorageUrchinScheduleFailed, | |||
IsDir: false, | |||
ComputeSource: models.NPUResource, | |||
Remark: interceptErrorMessages(err), | |||
LocalOperateStatus: models.MoveBucketWaiting, | |||
}) | |||
if err != nil { | |||
log.Error("InsertScheduleRecord failed:%v", err) | |||
return err | |||
} | |||
return fmt.Errorf("GetBackModel failed after retrying:%v", err) | |||
} | |||
_, err = models.InsertScheduleRecord(&models.ScheduleRecord{ | |||
CloudbrainID: cloudbrainID, | |||
EndPoint: endpoint, | |||
Bucket: bucket, | |||
ObjectKey: objectKey, | |||
ProxyServer: destPeerHost, | |||
Status: res.StatusCode, | |||
IsDir: false, | |||
ComputeSource: models.NPUResource, | |||
LocalOperateStatus: models.MoveBucketWaiting, | |||
}) | |||
if err != nil { | |||
log.Error("InsertScheduleRecord failed:%v", err) | |||
return err | |||
} | |||
r, err := models.GetScheduleRecordByCloudbrainID(cloudbrainID) | |||
if err != nil { | |||
log.Error("GetScheduleRecordByCloudbrainID err.cloudbrainID=%d err=%v", cloudbrainID, err) | |||
return err | |||
} | |||
err = handleScheduleResult(r, res) | |||
if err != nil { | |||
log.Error("GetNPUDataBack handleScheduleResult err.%v", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func tryScheduleDir(endpoint, bucket, objectKey, dstPeer string) { | |||
println("new request dstPeer: ", dstPeer) | |||
urfs := New() | |||
@@ -0,0 +1,11 @@ | |||
package util | |||
func TruncateString(msg string, maxLength int) string { | |||
if msg == "" { | |||
return "" | |||
} | |||
if len(msg) < maxLength { | |||
maxLength = len(msg) | |||
} | |||
return msg[0:maxLength] | |||
} |
@@ -1264,6 +1264,7 @@ modelarts.infer_job.boot_file_helper=The startup file is the entry file for your | |||
modelarts.infer_job.continue_helper=Check Reuse to copy the output result file of the last training task | |||
modelarts.train_job.resource_helper=The "resource specification" is the hardware you use to run the task. In order for more people to use the resources of this platform, please select according to your actual needs | |||
modelarts.infer_job.tooltip = The model has been deleted and cannot be viewed. | |||
modelarts.infer_job.model_cant_see = You are currently unable to view the model, possibly due to permission restrictions or the model has been deleted. | |||
modelarts.download_log=Download log file | |||
modelarts.log_file = Log file | |||
modelarts.fullscreen_log_file = View in full screen | |||
@@ -1326,7 +1327,7 @@ model.manage.engine=Model engine | |||
model.manage.select.engine=Select model engine | |||
model.manage.modelfile=Model file | |||
model.manage.modellabel=Model label | |||
model.manage.modeldesc=Model description | |||
model.manage.modeldesc=Model brief introduction | |||
model.manage.modelaccess=Model Access | |||
model.manage.modelaccess.public=Public | |||
model.manage.modelaccess.private=Private | |||
@@ -2140,7 +2141,7 @@ settings.wiki_deletion_success = The repository wiki data has been deleted. | |||
settings.delete = Delete This Repository | |||
settings.delete_desc = Deleting a repository is permanent and cannot be undone. | |||
settings.delete_notices_1 = - This operation <strong>CANNOT</strong> be undone. | |||
settings.delete_notices_2 = - This operation will permanently delete the <strong>%s</strong> repository including code, issues, comments, wiki data and collaborator settings. | |||
settings.delete_notices_2 = - This operation will permanently delete the <strong>%s</strong> repository including the code, dataset, model, cloudbrain tasks, tasks, merge requests, and other contents. | |||
settings.delete_notices_fork_1 = - Forks of this repository will become independent after deletion. | |||
settings.deletion_success = The repository has been deleted. | |||
settings.update_settings_success = The repository settings have been updated. | |||
@@ -1276,6 +1276,7 @@ modelarts.infer_job.boot_file_helper=启动文件是您程序执行的入口文 | |||
modelarts.infer_job.continue_helper=勾选复用将拷贝上次训练任务输出结果文件 | |||
modelarts.train_job.resource_helper=「资源规格」是您运行该任务使用的硬件,为了更多人能够使用本平台的资源,请按照您的实际需求进行选择。 | |||
modelarts.infer_job.tooltip = 该模型已删除,无法查看。 | |||
modelarts.infer_job.model_cant_see = 您暂时无法查看该模型,可能因为权限限制或模型已被删除。 | |||
modelarts.download_log=下载日志文件 | |||
modelarts.log_file=日志文件 | |||
modelarts.fullscreen_log_file=全屏查看 | |||
@@ -1339,7 +1340,7 @@ model.manage.engine=模型框架 | |||
model.manage.select.engine=选择模型框架 | |||
model.manage.modelfile=模型文件 | |||
model.manage.modellabel=模型标签 | |||
model.manage.modeldesc=模型描述 | |||
model.manage.modeldesc=模型简介 | |||
model.manage.modelaccess=模型权限 | |||
model.manage.modelaccess.public=公开 | |||
model.manage.modelaccess.private=私有 | |||
@@ -2156,7 +2157,7 @@ settings.wiki_deletion_success=项目百科数据删除成功! | |||
settings.delete=删除本项目 | |||
settings.delete_desc=删除项目是永久性的, 无法撤消。 | |||
settings.delete_notices_1=- 此操作 <strong>不可以</strong> 被回滚。 | |||
settings.delete_notices_2=- 此操作将永久删除项目 <strong>%s</strong>,包括 Git 数据、 任务、评论、百科和协作者的操作权限。 | |||
settings.delete_notices_2=- 此操作将永久删除项目 <strong>%s</strong>,包括该项目中的代码、数据集、模型、云脑任务、任务、合并请求等内容。 | |||
settings.delete_notices_fork_1=- 在此项目删除后,它的派生项目将变成独立项目。 | |||
settings.deletion_success=项目已被删除。 | |||
settings.deletion_notice_cloudbrain=请先停止项目内正在运行的云脑任务,然后再删除项目。 | |||
@@ -0,0 +1 @@ | |||
<svg height="56" viewBox="0 0 12 56" width="12" xmlns="http://www.w3.org/2000/svg"><path d="m3.86950483 4.06524758 8.13049517-4.06524758v56l-8.13049517-4.0652476c-2.3714882-1.1857441-3.86950483-3.6095859-3.86950483-6.2609903v-35.3475242c0-2.65140439 1.49801663-5.07524622 3.86950483-6.26099032z" fill="#e3e9ed" fill-rule="evenodd" transform="matrix(-1 0 0 1 12 0)"/></svg> |
@@ -109,6 +109,7 @@ func AdminModelManage(ctx *context.Context) { | |||
if repo != nil { | |||
model.RepoName = repo.Name | |||
model.RepoOwnerName = repo.OwnerName | |||
model.RepoDisplayName = repo.DisplayName() | |||
} | |||
} | |||
} | |||
@@ -1053,6 +1053,7 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
m.Put("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GeneralCloudBrainJobStop) | |||
m.Group("/model", func() { | |||
m.Get("/schedule_status", repo.GetModelScheduleStatus) | |||
m.Post("/reschedule", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.RetryModelSchedule) | |||
}) | |||
}) | |||
}) | |||
@@ -6,6 +6,7 @@ | |||
package repo | |||
import ( | |||
"code.gitea.io/gitea/services/ai_task_service/schedule" | |||
"encoding/json" | |||
"net/http" | |||
"path" | |||
@@ -20,8 +21,6 @@ import ( | |||
"code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" | |||
"code.gitea.io/gitea/modules/urfs_client/urchin" | |||
"code.gitea.io/gitea/modules/notification" | |||
"code.gitea.io/gitea/modules/grampus" | |||
@@ -169,9 +168,6 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { | |||
} | |||
if oldStatus != job.Status { | |||
notification.NotifyChangeCloudbrainStatus(job, oldStatus) | |||
if models.IsTrainJobTerminal(job.Status) && len(result.JobInfo.Tasks[0].CenterID) == 1 { | |||
go urchin.GetAITaskOutPutBack(job.ID, job.JobName, result.JobInfo.Tasks[0].CenterID[0], job.ComputeResource) | |||
} | |||
} | |||
err = models.UpdateTrainJobVersion(job) | |||
if err != nil { | |||
@@ -191,7 +187,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { | |||
func GetModelScheduleStatus(ctx *context.APIContext) { | |||
jobID := ctx.Params(":jobid") | |||
status, err := cloudbrainTask.GetModelScheduleStatus(jobID) | |||
status, err := schedule.GetModelScheduleStatus(jobID) | |||
if err != nil { | |||
ctx.JSON(http.StatusOK, response.OuterResponseError(err)) | |||
return | |||
@@ -200,6 +196,16 @@ func GetModelScheduleStatus(ctx *context.APIContext) { | |||
ctx.JSON(http.StatusOK, response.OuterSuccessWithData(m)) | |||
} | |||
func RetryModelSchedule(ctx *context.APIContext) { | |||
jobID := ctx.Params(":jobid") | |||
err := schedule.RetryModelMigrate(jobID) | |||
if err != nil { | |||
ctx.JSON(http.StatusOK, response.OuterResponseError(err)) | |||
return | |||
} | |||
ctx.JSON(http.StatusOK, response.OuterSuccess()) | |||
} | |||
func TrainJobForModelConvertGetLog(ctx *context.APIContext) { | |||
var ( | |||
err error | |||
@@ -361,9 +367,7 @@ func DelTrainJobVersion(ctx *context.APIContext) { | |||
return | |||
} | |||
if task.Status != string(models.ModelArtsTrainJobImageFailed) && task.Status != string(models.ModelArtsTrainJobSubmitFailed) && task.Status != string(models.ModelArtsTrainJobDeleteFailed) && | |||
task.Status != string(models.ModelArtsTrainJobCompleted) && task.Status != string(models.ModelArtsTrainJobFailed) && | |||
task.Status != string(models.ModelArtsTrainJobKilled) && task.Status != string(models.ModelArtsTrainJobCanceled) && task.Status != string(models.ModelArtsTrainJobLost) { | |||
if !task.IsTerminal() { | |||
log.Error("the job(%s) version has not been stopped", task.JobName) | |||
ctx.NotFound(err) | |||
return | |||
@@ -466,14 +470,41 @@ func ModelList(ctx *context.APIContext) { | |||
return | |||
} | |||
status := models.ModelScheduleSucceed | |||
status := models.ModelMigrateSuccess | |||
if task.Type == models.TypeC2Net { | |||
if !task.IsTerminal() { | |||
log.Info("GetModelScheduleStatus job is not terminal.jobId=%s", jobID) | |||
status = models.JobNoTeminal | |||
} else { | |||
status, err = schedule.GetModelScheduleStatus(task.JobID) | |||
if err != nil { | |||
log.Error("GetModelScheduleStatus(%s) failed:%v", task.JobName, err.Error()) | |||
return | |||
} | |||
} | |||
} | |||
if status != models.ModelMigrateSuccess { | |||
ctx.JSON(http.StatusOK, map[string]interface{}{ | |||
"JobID": jobID, | |||
"VersionName": versionName, | |||
"StatusOK": status, | |||
"Path": dirArray, | |||
"Dirs": []storage.FileInfo{}, | |||
"task": task, | |||
"PageIsCloudBrain": true, | |||
}) | |||
return | |||
} | |||
var fileInfos []storage.FileInfo | |||
if task.ComputeResource == models.NPUResource { | |||
prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName), "/") | |||
if !strings.HasSuffix(prefix, "/") { | |||
prefix += "/" | |||
} | |||
fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) | |||
fileInfos, err = storage.GetOneLevelObjectsUnderDir(setting.Bucket, prefix, parentDir) | |||
if err != nil { | |||
log.Info("get TrainJobListModel failed:", err) | |||
ctx.ServerError("GetObsListObject:", err) | |||
@@ -504,19 +535,6 @@ func ModelList(ctx *context.APIContext) { | |||
}) | |||
} | |||
if task.Type == models.TypeC2Net { | |||
if !task.IsTerminal() { | |||
log.Info("GetModelScheduleStatus job is not terminal.jobId=%s", jobID) | |||
status = models.JobNoTeminal | |||
} else { | |||
status, err = cloudbrainTask.GetModelScheduleStatus(task.JobID) | |||
if err != nil { | |||
log.Error("GetModelScheduleStatus(%s) failed:%v", task.JobName, err.Error()) | |||
return | |||
} | |||
} | |||
} | |||
ctx.JSON(http.StatusOK, map[string]interface{}{ | |||
"JobID": jobID, | |||
"VersionName": versionName, | |||
@@ -144,7 +144,9 @@ func saveModelByParameters(jobId string, versionName string, name string, versio | |||
go asyncToCopyModel(aiTask, id, modelSelectedFile) | |||
log.Info("save model end.") | |||
if !model.IsPrivate { | |||
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, id, name, models.ActionCreateNewModelTask) | |||
} | |||
return id, nil | |||
} | |||
@@ -156,6 +158,7 @@ func asyncToCopyModel(aiTask *models.Cloudbrain, id string, modelSelectedFile st | |||
log.Info("download model from CloudBrainTwo faild." + err.Error()) | |||
} else { | |||
updateStatus(id, modelSize, STATUS_FINISHED, modelPath, "") | |||
insertModelFile(id) | |||
} | |||
} else if aiTask.ComputeResource == models.GPUResource { | |||
@@ -165,7 +168,24 @@ func asyncToCopyModel(aiTask *models.Cloudbrain, id string, modelSelectedFile st | |||
log.Info("download model from CloudBrainOne faild." + err.Error()) | |||
} else { | |||
updateStatus(id, modelSize, STATUS_FINISHED, modelPath, "") | |||
insertModelFile(id) | |||
} | |||
} | |||
} | |||
func insertModelFile(id string) { | |||
model, _ := models.QueryModelById(id) | |||
files, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, model.Path[len(setting.Bucket)+1:]) | |||
if err != nil { | |||
log.Info("Failed to query model size from obs. id=" + id) | |||
} | |||
for _, file := range files { | |||
modelFile := &models.AiModelFile{ | |||
ModelID: id, | |||
Name: file.FileName, | |||
Size: file.Size, | |||
} | |||
models.SaveModelFile(modelFile) | |||
} | |||
} | |||
@@ -228,6 +248,13 @@ func SaveLocalModel(ctx *context.Context) { | |||
engine := ctx.QueryInt("engine") | |||
taskType := ctx.QueryInt("type") | |||
isPrivate := ctx.QueryBool("isPrivate") | |||
if ctx.Repo.Repository.IsPrivate { | |||
if !isPrivate { | |||
re["msg"] = "Private repo cannot create public model." | |||
ctx.JSON(200, re) | |||
return | |||
} | |||
} | |||
modelActualPath := "" | |||
computeResource := "" | |||
if taskType == models.TypeCloudBrainOne { | |||
@@ -305,7 +332,9 @@ func SaveLocalModel(ctx *context.Context) { | |||
models.UpdateRepositoryUnits(ctx.Repo.Repository, units, deleteUnitTypes) | |||
log.Info("save model end.") | |||
if !model.IsPrivate { | |||
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, id, name, models.ActionCreateNewModelTask) | |||
} | |||
re["code"] = "0" | |||
re["id"] = id | |||
ctx.JSON(200, re) | |||
@@ -319,11 +348,10 @@ func getSize(files []storage.FileInfo) int64 { | |||
return size | |||
} | |||
func UpdateModelSize(modeluuid string) { | |||
func UpdateModelSize(modeluuid string, objectName string) { | |||
model, err := models.QueryModelById(modeluuid) | |||
if err == nil { | |||
var size int64 | |||
if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { | |||
files, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, model.Path[len(setting.Bucket)+1:]) | |||
if err != nil { | |||
@@ -331,8 +359,24 @@ func UpdateModelSize(modeluuid string) { | |||
} | |||
size = getSize(files) | |||
models.ModifyModelSize(modeluuid, size) | |||
modelFileName := objectName | |||
index := strings.LastIndex(objectName, "/") | |||
if index > 0 { | |||
modelFileName = objectName[index+1:] | |||
} | |||
log.Info("modelFileName=" + modelFileName) | |||
for _, file := range files { | |||
log.Info("fileName=" + file.FileName) | |||
if file.FileName == modelFileName { | |||
modelFile := &models.AiModelFile{ | |||
ModelID: modeluuid, | |||
Name: file.FileName, | |||
Size: file.Size, | |||
} | |||
models.SaveModelFile(modelFile) | |||
} | |||
} | |||
} | |||
if model.Size == 0 && size > 0 { | |||
go repository.ResetRepoModelNum(model.RepoId) | |||
} | |||
@@ -359,6 +403,14 @@ func SaveModel(ctx *context.Context) { | |||
re := map[string]string{ | |||
"code": "-1", | |||
} | |||
isPrivate := ctx.QueryBool("isPrivate") | |||
if ctx.Repo.Repository.IsPrivate { | |||
if !isPrivate { | |||
re["msg"] = "Private repo cannot create public model." | |||
ctx.JSON(200, re) | |||
return | |||
} | |||
} | |||
if JobId == "" || VersionName == "" { | |||
re["msg"] = "JobId or VersionName is null." | |||
ctx.JSON(200, re) | |||
@@ -482,6 +534,11 @@ func DeleteModelFile(ctx *context.Context) { | |||
} else { | |||
log.Info("delete obs file size is:" + fmt.Sprint(totalSize)) | |||
models.ModifyModelSize(id, model.Size-totalSize) | |||
modelFile := &models.AiModelFile{ | |||
Name: fileName, | |||
ModelID: id, | |||
} | |||
models.DeleteModelFile(modelFile) | |||
} | |||
} | |||
@@ -882,15 +939,20 @@ func QueryModelById(ctx *context.Context) { | |||
func ShowSingleModel(ctx *context.Context) { | |||
name := ctx.Query("name") | |||
log.Info("Show single ModelInfo start.name=" + name) | |||
modelArrays := models.QueryModelByName(name, ctx.Repo.Repository.ID) | |||
modelResult := make([]*models.AiModelManage, 0) | |||
isCanReadPrivateModel := isQueryPrivateModel(ctx) | |||
userIds := make([]int64, len(modelArrays)) | |||
for i, model := range modelArrays { | |||
model.IsCanOper = isOperModifyOrDelete(ctx, model.UserId) | |||
model.IsCanDownload = isCanDownload(ctx, model) | |||
model.IsCanDelete = isCanDelete(ctx, model.UserId) | |||
model.RepoName = ctx.Repo.Repository.Name | |||
model.RepoOwnerName = ctx.Repo.Repository.OwnerName | |||
model.RepoDisplayName = ctx.Repo.Repository.DisplayName() | |||
userIds[i] = model.UserId | |||
if ctx.User != nil { | |||
re := models.QueryModelCollectByUserId(model.ID, ctx.User.ID) | |||
@@ -898,10 +960,16 @@ func ShowSingleModel(ctx *context.Context) { | |||
model.IsCollected = true | |||
} | |||
} | |||
if model.IsPrivate { | |||
if !isCanReadPrivateModel { | |||
continue | |||
} | |||
} | |||
modelResult = append(modelResult, model) | |||
} | |||
userNameMap := queryUserName(userIds) | |||
for _, model := range modelArrays { | |||
for _, model := range modelResult { | |||
removeIpInfo(model) | |||
value := userNameMap[model.UserId] | |||
if value != nil { | |||
@@ -909,7 +977,7 @@ func ShowSingleModel(ctx *context.Context) { | |||
model.UserRelAvatarLink = value.RelAvatarLink() | |||
} | |||
} | |||
ctx.JSON(http.StatusOK, modelArrays) | |||
ctx.JSON(http.StatusOK, modelResult) | |||
} | |||
func removeIpInfo(model *models.AiModelManage) { | |||
@@ -1332,11 +1400,11 @@ func queryOneLevelModelFile(model *models.AiModelManage, parentDir string) []sto | |||
fileinfos := make([]storage.FileInfo, 0) | |||
if model.Type == models.TypeCloudBrainTwo { | |||
log.Info("TypeCloudBrainTwo list model file.") | |||
prefix := model.Path[len(setting.Bucket)+1:] | |||
prefix := model.Path[len(setting.Bucket)+1:] + parentDir | |||
fileinfos, _ = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, "") | |||
} else if model.Type == models.TypeCloudBrainOne { | |||
log.Info("TypeCloudBrainOne list model file.") | |||
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:] | |||
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:] + parentDir | |||
fileinfos, _ = storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "") | |||
} | |||
if fileinfos == nil { | |||
@@ -24,7 +24,7 @@ const ( | |||
tplModelFileList = "repo/modelmanage/filelist" | |||
tplModelSetting = "repo/modelmanage/setting" | |||
tplModelEvolutionMap = "repo/modelmanage/evolution_map" | |||
README_FILE_NAME = "README.MD" | |||
README_FILE_NAME = "README.md" | |||
) | |||
type ModelMap struct { | |||
@@ -65,6 +65,7 @@ func ModelSquareData(ctx *context.Context) { | |||
if frameFilterStr != "" { | |||
frameFilterInt, _ = strconv.Atoi(frameFilterStr) | |||
} | |||
notNeedEmpty := ctx.QueryBool("notNeedEmpty") | |||
computeResourceFilter := ctx.Query("compute_resource") | |||
var IsQueryPrivate bool | |||
var user_id int64 | |||
@@ -142,6 +143,7 @@ func ModelSquareData(ctx *context.Context) { | |||
Namelike: Namelike, | |||
SortType: SortType, | |||
RepoID: repo_id, | |||
NotNeedEmpty: notNeedEmpty, | |||
}) | |||
if err != nil { | |||
ctx.ServerError("Cloudbrain", err) | |||
@@ -175,6 +177,7 @@ func ModelSquareData(ctx *context.Context) { | |||
if repo != nil { | |||
model.RepoName = repo.Name | |||
model.RepoOwnerName = repo.OwnerName | |||
model.RepoDisplayName = repo.DisplayName() | |||
} | |||
} | |||
if ctx.User != nil && modelCollect != nil { | |||
@@ -185,7 +188,7 @@ func ModelSquareData(ctx *context.Context) { | |||
} else { | |||
model.IsCollected = false | |||
} | |||
if needModelFile { | |||
if needModelFile && len(model.Path) > 0 { | |||
//查询模型文件列表 | |||
model.ModelFileList = modelmanage.QueryModelFileByModel(model) | |||
} | |||
@@ -238,6 +241,7 @@ func setModelRepo(model *models.AiModelManage) { | |||
if err == nil { | |||
model.RepoName = repo.Name | |||
model.RepoOwnerName = repo.OwnerName | |||
model.RepoDisplayName = repo.DisplayName() | |||
} | |||
} | |||
@@ -252,6 +256,7 @@ func ModelEvolutionMapData(ctx *context.Context) { | |||
repo, err := models.GetRepositoryByID(model.RepoId) | |||
model.RepoName = repo.Name | |||
model.RepoOwnerName = repo.OwnerName | |||
model.RepoDisplayName = repo.DisplayName() | |||
if err == nil { | |||
setModelUser(model) | |||
currentNode := &ModelMap{ | |||
@@ -346,6 +351,7 @@ func findChild(currentNode *ModelMap) { | |||
if childModel.TrainTaskInfo != "" { | |||
childModel.RepoName = node.RepoName | |||
childModel.RepoOwnerName = node.RepoOwnerName | |||
childModel.RepoDisplayName = node.RepoDisplayName | |||
log.Info("childModel.RepoName=" + childModel.RepoName) | |||
log.Info("childModel.RepoOwnerName=" + childModel.RepoOwnerName) | |||
var task models.Cloudbrain | |||
@@ -669,7 +669,7 @@ func createForNPU(ctx *context.Context, jobName string) error { | |||
log.Info("engine_id=" + fmt.Sprint(engineID)) | |||
poolID := ctx.Query("pool_id") | |||
repo := ctx.Repo.Repository | |||
modelId := ctx.Query("model_id") | |||
trainUrl := ctx.Query("pre_train_model_url") | |||
modelName := ctx.Query("model_name") | |||
modelVersion := ctx.Query("model_version") | |||
@@ -814,6 +814,7 @@ func createForNPU(ctx *context.Context, jobName string) error { | |||
ModelName: modelName, | |||
ModelVersion: modelVersion, | |||
CkptName: ckptName, | |||
ModelId: modelId, | |||
ResultUrl: resultObsPath, | |||
Spec: spec, | |||
DatasetName: datasetNames, | |||
@@ -314,7 +314,7 @@ func CompleteModelMultipart(ctx *context.Context) { | |||
return | |||
} | |||
//更新模型大小信息 | |||
UpdateModelSize(modeluuid) | |||
UpdateModelSize(modeluuid, fileChunk.ObjectName) | |||
ctx.JSON(200, map[string]string{ | |||
"result_code": "0", | |||
@@ -19,8 +19,6 @@ import ( | |||
cloudbrainService "code.gitea.io/gitea/services/cloudbrain" | |||
"code.gitea.io/gitea/modules/urfs_client/urchin" | |||
"code.gitea.io/gitea/modules/dataset" | |||
"code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" | |||
@@ -680,7 +678,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra | |||
ModelVersion: form.ModelVersion, | |||
CkptName: form.CkptName, | |||
ModelId: form.ModelId, | |||
TrainUrl: form.TrainUrl, | |||
TrainUrl: form.PreTrainModelUrl, | |||
LabelName: labelName, | |||
Spec: spec, | |||
} | |||
@@ -1979,9 +1977,11 @@ func SyncCloudbrainStatus() { | |||
} | |||
if result != nil { | |||
if len(result.JobInfo.Tasks) > 0 { | |||
if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { | |||
task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] | |||
} | |||
} | |||
oldStatus := task.Status | |||
task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) | |||
task.Duration = result.JobInfo.RunSec | |||
@@ -2000,9 +2000,6 @@ func SyncCloudbrainStatus() { | |||
task.CorrectCreateUnix() | |||
if oldStatus != task.Status { | |||
notification.NotifyChangeCloudbrainStatus(task, oldStatus) | |||
if models.IsTrainJobTerminal(task.Status) && len(result.JobInfo.Tasks[0].CenterID) == 1 { | |||
go urchin.GetAITaskOutPutBack(task.ID, task.JobName, result.JobInfo.Tasks[0].CenterID[0], task.ComputeResource) | |||
} | |||
} | |||
err = models.UpdateJob(task) | |||
if err != nil { | |||
@@ -16,7 +16,6 @@ import ( | |||
"code.gitea.io/gitea/services/lock" | |||
"code.gitea.io/gitea/modules/urfs_client/urchin" | |||
"code.gitea.io/gitea/routers/response" | |||
"code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" | |||
@@ -1383,11 +1382,6 @@ func GrampusNotebookShow(ctx *context.Context) { | |||
task.CorrectCreateUnix() | |||
if oldStatus != task.Status { | |||
notification.NotifyChangeCloudbrainStatus(task, oldStatus) | |||
if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource { | |||
if len(result.JobInfo.Tasks[0].CenterID) == 1 { | |||
urchin.GetNPUDataBack(task.ID, task.JobName, result.JobInfo.Tasks[0].CenterID[0]) | |||
} | |||
} | |||
} | |||
} | |||
err = models.UpdateJob(task) | |||
@@ -1539,9 +1533,6 @@ func GrampusTrainJobShow(ctx *context.Context) { | |||
task.CorrectCreateUnix() | |||
if oldStatus != task.Status { | |||
notification.NotifyChangeCloudbrainStatus(task, oldStatus) | |||
if models.IsTrainJobTerminal(task.Status) && len(result.JobInfo.Tasks[0].CenterID) == 1 { | |||
go urchin.GetAITaskOutPutBack(task.ID, task.JobName, result.JobInfo.Tasks[0].CenterID[0], task.ComputeResource) | |||
} | |||
} | |||
} | |||
err = models.UpdateJob(task) | |||
@@ -1580,6 +1571,7 @@ func GrampusTrainJobShow(ctx *context.Context) { | |||
ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false) | |||
ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) | |||
ctx.Data["displayJobName"] = task.DisplayJobName | |||
ctx.Data["canReschedule"] = cloudbrain.CanDeleteJob(ctx, task) | |||
ctx.Data["ai_center"] = cloudbrainService.GetAiCenterShow(task.AiCenter, ctx) | |||
@@ -2017,9 +2017,7 @@ func TrainJobDel(ctx *context.Context) { | |||
} | |||
for _, task := range VersionListTasks { | |||
if task.Status != string(models.ModelArtsTrainJobImageFailed) && task.Status != string(models.ModelArtsTrainJobSubmitFailed) && task.Status != string(models.ModelArtsTrainJobDeleteFailed) && | |||
task.Status != string(models.ModelArtsTrainJobCompleted) && task.Status != string(models.ModelArtsTrainJobFailed) && | |||
task.Status != string(models.ModelArtsTrainJobKilled) && task.Status != string(models.ModelArtsTrainJobCanceled) && task.Status != string(models.ModelArtsTrainJobLost) { | |||
if !task.IsTerminal() { | |||
log.Error("the job(%s) version has not been stopped", task.JobName) | |||
ctx.RenderWithErr("the job version has not been stopped", tplModelArtsTrainJobIndex, nil) | |||
return | |||
@@ -2164,11 +2162,11 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference | |||
LabelName := form.LabelName | |||
isLatestVersion := modelarts.IsLatestVersion | |||
VersionCount := modelarts.VersionCountOne | |||
trainUrl := form.TrainUrl | |||
trainUrl := form.PreTrainModelUrl | |||
modelName := form.ModelName | |||
modelVersion := form.ModelVersion | |||
ckptName := form.CkptName | |||
ckptUrl := "/" + form.TrainUrl + form.CkptName | |||
ckptUrl := "/" + form.PreTrainModelUrl + form.CkptName | |||
log.Info("ckpt url:" + ckptUrl) | |||
errStr := checkInferenceJobMultiNode(ctx.User.ID, form.WorkServerNumber) | |||
@@ -2501,6 +2499,18 @@ func InferenceJobIndex(ctx *context.Context) { | |||
if tasks[i].ComputeResource == "" { | |||
tasks[i].ComputeResource = models.NPUResource | |||
} | |||
if tasks[i].ModelId != "" { | |||
model, err := models.QueryModelById(tasks[i].ModelId) | |||
if err == nil && model != nil { | |||
if model.RepoId != tasks[i].RepoID { | |||
repo, err := models.GetRepositoryByID(model.RepoId) | |||
if err == nil && repo != nil { | |||
tasks[i].ModelRepoName = repo.Name | |||
tasks[i].ModelRepoOwnerName = repo.OwnerName | |||
} | |||
} | |||
} | |||
} | |||
} | |||
isQueryPrivate := isQueryPrivateModel(ctx) | |||
repoId := ctx.Repo.Repository.ID | |||
@@ -2680,7 +2690,7 @@ func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModel | |||
ctx.Data["model_version"] = form.ModelVersion | |||
ctx.Data["ckpt_name"] = form.CkptName | |||
ctx.Data["model_id"] = form.ModelId | |||
ctx.Data["train_url"] = form.TrainUrl | |||
ctx.Data["pre_train_model_url"] = form.PreTrainModelUrl | |||
ctx.Data["datasetType"] = models.TypeCloudBrainTwo | |||
waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") | |||
ctx.Data["WaitCount"] = waitCount | |||
@@ -721,10 +721,10 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { | |||
log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05")) | |||
warnEmailMessage := "用户统计信息入库失败,请尽快定位。" | |||
startYear := time.Date(USER_YEAR, 1, 1, 0, 0, 0, 1, t.Location()) | |||
endYear := startYear.AddDate(1, 0, 0) | |||
//startYear := time.Date(USER_YEAR, 1, 1, 0, 0, 0, 1, t.Location()) | |||
//endYear := startYear.AddDate(1, 0, 0) | |||
models.RefreshUserYearTable(startYear, endYear) | |||
//models.RefreshUserYearTable(startYear, endYear) | |||
//query wiki data | |||
log.Info("start to time count data") | |||
@@ -0,0 +1,374 @@ | |||
package schedule | |||
import ( | |||
"bytes" | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/grampus" | |||
"code.gitea.io/gitea/modules/labelmsg" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/redis/redis_key" | |||
"code.gitea.io/gitea/modules/redis/redis_lock" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/storage" | |||
"code.gitea.io/gitea/modules/util" | |||
"encoding/json" | |||
"errors" | |||
"fmt" | |||
"github.com/minio/minio-go" | |||
"os/exec" | |||
"path" | |||
"strings" | |||
"time" | |||
) | |||
const NPUModelDefaultName = "models.zip" | |||
func GetModelScheduleStatus(jobId string) (models.ModelMigrateStatus, error) { | |||
job, err := models.GetCloudbrainByJobID(jobId) | |||
if err != nil { | |||
log.Error("GetModelScheduleStatus GetCloudbrainByJobID err.jobId=%s err=%v", jobId, err) | |||
return 0, errors.New("jobId not correct") | |||
} | |||
if !job.IsTerminal() { | |||
log.Info("GetModelScheduleStatus job is not terminal.jobId=%s", jobId) | |||
return models.ModelMigrateWaiting, nil | |||
} | |||
record, err := models.GetModelMigrateRecordByCloudbrainId(job.ID) | |||
if err != nil { | |||
log.Error("GetModelScheduleStatus GetModelMigrateRecordByCloudbrainId err.jobId=%s err=%v", jobId, err) | |||
if models.IsErrRecordNotExist(err) { | |||
return models.ModelMigrateSuccess, nil | |||
} | |||
return models.ModelMigrateFailed, err | |||
} | |||
if !record.IsFinished() { | |||
go HandleUnfinishedMigrateRecord(record) | |||
} | |||
return record.Status, nil | |||
} | |||
func RetryModelMigrate(jobId string) error { | |||
job, err := models.GetCloudbrainByJobID(jobId) | |||
if err != nil { | |||
log.Error("RetryModelMigrate GetCloudbrainByJobID err.jobId=%s err=%v", jobId, err) | |||
return errors.New("jobId not correct") | |||
} | |||
if !job.IsTerminal() { | |||
log.Info("RetryModelMigrate job is not terminal.jobId=%s", jobId) | |||
return errors.New("task is not terminal") | |||
} | |||
//避免并发问题,先尝试获取锁,获取锁以后再查最新的记录 | |||
lock := redis_lock.NewDistributeLock(redis_key.RecordHandleLock(jobId)) | |||
success, err := lock.LockWithWait(10*time.Second, 10*time.Second) | |||
if err != nil { | |||
log.Error("HandleUnfinishedMigrateRecord lock err.jobId=%d %v", jobId, err) | |||
return err | |||
} | |||
if !success { | |||
log.Error("HandleUnfinishedMigrateRecord lock failed.ID=%d ", jobId) | |||
return nil | |||
} | |||
defer lock.UnLock() | |||
record, err := models.GetModelMigrateRecordByCloudbrainId(job.ID) | |||
if err != nil { | |||
log.Error("RetryModelMigrate GetModelMigrateRecordByCloudbrainId err.jobId=%s err=%v", jobId, err) | |||
if models.IsErrRecordNotExist(err) { | |||
return nil | |||
} | |||
return err | |||
} | |||
//只有两种情况可以再次调度,一是虎鲸调度失败 二是本地移桶失败 | |||
if record.CurrentStep == models.GrampusMigrateFailed { | |||
log.Info("retry PostModelMigrate. record.id = %d", record.ID) | |||
_, err := grampus.PostModelMigrate(jobId) | |||
if err != nil { | |||
log.Error("PostModelMigrate err.%v", err) | |||
return err | |||
} | |||
models.IncreaseModelMigrateRetryCount(record.ID) | |||
if err := models.RollBackMigrateStatus(record, models.GrampusMigrating); err != nil { | |||
log.Error("UpdateModelMigrateStatusByStep err.%v", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
if record.CurrentStep == models.BucketMoveFailed { | |||
log.Info("retry BucketMove. record.id = %d", record.ID) | |||
if err := models.RollBackMigrateStatus(record, models.GrampusMigrateSuccess); err != nil { | |||
log.Error("UpdateModelMigrateStatusByStep err.%v", err) | |||
return err | |||
} | |||
models.IncreaseModelMigrateRetryCount(record.ID) | |||
return nil | |||
} | |||
return errors.New("No need to retry,the model migration has been successful or is in the process.") | |||
} | |||
func HandleUnfinishedMigrateRecords() { | |||
list, err := models.GetUnfinishedModelMigrateRecords() | |||
if err != nil { | |||
log.Error("GetUnfinishedModelMigrateRecords err=%v", err) | |||
return | |||
} | |||
for _, r := range list { | |||
HandleUnfinishedMigrateRecord(r) | |||
} | |||
} | |||
func HandleUnfinishedMigrateRecord(r *models.ModelMigrateRecord) error { | |||
cloudbrain, err := models.GetCloudbrainByID(fmt.Sprint(r.CloudbrainID)) | |||
if err != nil { | |||
log.Error("GetCloudbrainByID err. %v", err) | |||
return err | |||
} | |||
lock := redis_lock.NewDistributeLock(redis_key.RecordHandleLock(cloudbrain.JobID)) | |||
success, err := lock.LockWithWait(10*time.Second, 10*time.Second) | |||
if err != nil { | |||
log.Error("HandleUnfinishedMigrateRecord lock err.ID=%d %v", r.ID, err) | |||
return err | |||
} | |||
if !success { | |||
log.Error("HandleUnfinishedMigrateRecord lock failed.ID=%d ", r.ID) | |||
return nil | |||
} | |||
defer lock.UnLock() | |||
//拿到锁以后重新查询一次 | |||
r, err = models.GetModelMigrateRecordById(r.ID) | |||
if err != nil { | |||
log.Error("RetryModelMigrate GetModelMigrateRecordById err.Id=%s err=%v", r.ID, err) | |||
if models.IsErrRecordNotExist(err) { | |||
return nil | |||
} | |||
return err | |||
} | |||
if r.CurrentStep == models.GrampusMigrateInit || r.CurrentStep == models.GrampusMigrating { | |||
if err := UpdateModelMigrateStatusFromGrampus(r, cloudbrain.JobID); err != nil { | |||
log.Error("UpdateModelMigrateStatusFromGrampus err. %v", err) | |||
return err | |||
} | |||
} | |||
if r.CurrentStep == models.GrampusMigrateSuccess { | |||
if err := LocalMigrateOperate(cloudbrain.JobName, cloudbrain.ComputeResource, r); err != nil { | |||
log.Error("LocalMigrateOperate err. %v", err) | |||
return err | |||
} | |||
} | |||
if r.CurrentStep == models.BucketMoving { | |||
//尝试查询NPU结果目录下是否有文件,有文件则认为已经解压成功 | |||
if cloudbrain.ComputeResource == models.NPUResource && IsNPUModelDirHasFile(cloudbrain.JobName, cloudbrain.VersionName) { | |||
TryToUpdateNPUMoveBucketResult(r, cloudbrain.JobName, cloudbrain.VersionName) | |||
} | |||
} | |||
return nil | |||
} | |||
func UpdateModelMigrateStatusFromGrampus(r *models.ModelMigrateRecord, jobId string) error { | |||
res, err := grampus.ModelMigrateInfo(jobId) | |||
if err != nil { | |||
log.Error("ModelMigrateInfo err. r.ID=%d %v", r.ID, err) | |||
return err | |||
} | |||
log.Info("grampus ModelMigrateInfo r.ID=%d res=%+v", r.ID, res) | |||
newStep := models.GrampusMigrateResponse(res.Status).ConvertToModelMigrateStep() | |||
if newStep == r.CurrentStep { | |||
log.Info("The status has not changed. r.ID=%d status=%d", r.ID, res.Status) | |||
return nil | |||
} | |||
err = updateModelMigrateFromRes(r, res) | |||
if err != nil { | |||
log.Error("updateModelMigrateFromRes err. r.ID=%d %v", r.ID, err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func LocalMigrateOperate(jobName, computeSource string, r *models.ModelMigrateRecord) error { | |||
log.Info("Grampus model migrate succeed,objectKey = %s computeSource= %s", r.DestObjectKey, computeSource) | |||
err := models.UpdateModelMigrateStatusByStep(r, models.BucketMoving) | |||
if err != nil { | |||
log.Error("UpdateModelMigrateStatusByStep err. r.ID=%d step=%d err=%v", r.ID, models.BucketMoveFailed, err) | |||
return err | |||
} | |||
if computeSource == models.NPUResource { | |||
//因为NPU的输出会被压缩,因此需要解压+移桶 | |||
decompress(r.DestBucket+"/"+r.DestObjectKey, setting.Bucket+"/"+strings.TrimSuffix(r.DestObjectKey, models.ModelSuffix)) | |||
} else { | |||
//因为调度无法指定桶,所以调度成功后我们还需要移桶 | |||
if setting.UseLocalMinioMigrate { | |||
if err := MoveBucketJust4LocalMinio(r.DestObjectKey, grampus.GetGPUModelObjectKey(jobName), r.DestBucket, setting.Attachment.Minio.Bucket); err != nil { | |||
log.Error("MoveBucketJust4LocalMinio err.%v", err) | |||
if tmpErr := models.UpdateModelMigrateStatusByStep(r, models.BucketMoveFailed); tmpErr != nil { | |||
log.Error("UpdateModelMigrateStatusByStep error. r.ID=%d step=%d err=%v", r.ID, models.BucketMoveFailed, tmpErr) | |||
} | |||
return err | |||
} | |||
} else { | |||
if err := MoveBucketInOpenIMinio(r.DestObjectKey, grampus.GetGPUModelObjectKey(jobName), r.DestBucket, setting.Attachment.Minio.Bucket); err != nil { | |||
log.Error("MoveBucketInOpenIMinio err.%v", err) | |||
if tmpErr := models.UpdateModelMigrateStatusByStep(r, models.BucketMoveFailed); tmpErr != nil { | |||
log.Error("UpdateModelMigrateStatusByStep error. r.ID=%d step=%d err=%v", r.ID, models.BucketMoveFailed, tmpErr) | |||
} | |||
return err | |||
} | |||
} | |||
if err := models.UpdateModelMigrateStatusByStep(r, models.BucketMoveSuccess); err != nil { | |||
log.Error("UpdateModelMigrateStatusByStep error. r.ID=%d step=%d err=%v", r.ID, models.BucketMoveSuccess, err) | |||
} | |||
} | |||
return nil | |||
} | |||
func TryToUpdateNPUMoveBucketResult(record *models.ModelMigrateRecord, jobName, versionName string) error { | |||
if IsNPUModelDirHasFile(jobName, versionName) { | |||
if err := models.UpdateModelMigrateStatusByStep(record, models.BucketMoveSuccess); err != nil { | |||
log.Error("UpdateModelMigrateStatusByStep error. r.ID=%d step=%d err=%v", record.ID, models.BucketMoveSuccess, err) | |||
return err | |||
} | |||
} | |||
return nil | |||
} | |||
func updateModelMigrateFromRes(r *models.ModelMigrateRecord, res *models.GrampusModelMigrateInfoResponse) error { | |||
step := models.GrampusMigrateResponse(res.Status).ConvertToModelMigrateStep() | |||
err := models.UpdateModelMigrateStatusByStep(r, step) | |||
if err != nil { | |||
log.Error("UpdateModelMigrateStatusByStep err,ID=%d err=%v", r.ID, err) | |||
return err | |||
} | |||
r.DestBucket = res.DestBucket | |||
r.DestEndpoint = res.DestEndpoint | |||
r.DestObjectKey = res.DestObjectKey | |||
r.DestProxy = res.DestProxy | |||
r.Remark = strings.TrimPrefix(r.Remark+";"+util.TruncateString(res.FailedReason, 200), ";") | |||
r.SrcBucket = res.SrcBucket | |||
r.SrcEndpoint = res.SrcEndpoint | |||
r.SrcObjectKey = res.SrcObjectKey | |||
err = models.UpdateModelMigrateRecordByStep(r) | |||
if err != nil { | |||
log.Error("updateModelMigrateFromRes UpdateModelMigrateRecord error.id=%d.err=%v", r.ID, err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func MoveBucketInOpenIMinio(objectKeyPrefix, targetObjectPrefix, oldBucket, newBucket string) error { | |||
var core = storage.ScheduleMinioCore | |||
objectInfo := core.Client.ListObjects(oldBucket, objectKeyPrefix, true, nil) | |||
log.Info("MoveBucketInOpenIMinio start.objectKeyPrefix=%s", objectKeyPrefix) | |||
count := 0 | |||
for object := range objectInfo { | |||
count++ | |||
if object.Err != nil { | |||
log.Error("MoveBucketInOpenIMinio object.Err=%v", object.Err) | |||
return object.Err | |||
} | |||
log.Debug("MoveBucketInOpenIMinio object.Key=%s", object.Key) | |||
newObjectKey := strings.Replace(object.Key, objectKeyPrefix, targetObjectPrefix, 1) | |||
err := MoveMinioFileBucket(core, object.Key, newObjectKey, oldBucket, newBucket) | |||
if err != nil { | |||
log.Error("MoveBucketInOpenIMinio MoveMinioFileBucket object.Key=%s Err=%v", object.Key, err) | |||
continue | |||
} | |||
} | |||
log.Info("MoveBucketInOpenIMinio finished.objectKeyPrefix=%s ,total=%d", objectKeyPrefix, count) | |||
return nil | |||
} | |||
func MoveBucketJust4LocalMinio(objectKeyPrefix, targetObjectPrefix, oldBucket, newBucket string) error { | |||
oldPath := path.Join(setting.Attachment.Minio.RealPath, oldBucket, objectKeyPrefix) | |||
newPath := path.Join(setting.Attachment.Minio.RealPath, newBucket, targetObjectPrefix) | |||
log.Info("MoveBucketJust4LocalMinio start.oldPath=%s newPath=%s", oldPath, newPath) | |||
//重命名原有文件夹,防止已有该文件 | |||
err, errStr := sudoMv(newPath, fmt.Sprintf("%s_%d", newPath, time.Now().Unix())) | |||
if err != nil { | |||
log.Error("MoveBucketJust4LocalMinio sudoMv error.oldPath=%s newPath=%s Err=%v errStr=%s ", oldPath, newPath, err, errStr) | |||
} | |||
//移动(重命名)文件夹 | |||
err, errStr = sudoMv(oldPath, newPath) | |||
if err != nil { | |||
log.Error("MoveBucketJust4LocalMinio sudoMv error.oldPath=%s newPath=%s Err=%v errStr=%s ", oldPath, newPath, err, errStr) | |||
return err | |||
} | |||
log.Info("MoveBucketInOpenIMinio finished.oldPath=%s newPath=%s ", oldPath, newPath) | |||
return nil | |||
} | |||
func sudoMv(oldPath, newPath string) (error, string) { | |||
c := fmt.Sprintf("sudo mv %s %s", oldPath, newPath) | |||
log.Info("start to sudoMv,oldPath=%s newPath=%s", oldPath, newPath) | |||
cmd := exec.Command("/bin/sh", "-c", c) | |||
var stdout, stderr bytes.Buffer | |||
cmd.Stdout = &stdout // 标准输出 | |||
cmd.Stderr = &stderr // 标准错误 | |||
err := cmd.Run() | |||
outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) | |||
log.Debug("out:\n%s\nerr:\n%s\n", outStr, errStr) | |||
if err != nil { | |||
log.Error("cmd.Run() failed,oldPath=%s newPath=%s err=%v\n", oldPath, newPath, err) | |||
return err, errStr | |||
} | |||
return nil, errStr | |||
} | |||
func MoveMinioFileBucket(core *minio.Core, oldObjectKey, newObjectKey, oldBucket, newBucket string) error { | |||
_, err := core.CopyObject(oldBucket, oldObjectKey, newBucket, newObjectKey, map[string]string{}) | |||
if err != nil { | |||
log.Error("MoveBucketInOpenIMinio CopyObject err oldObjectKey=%s .%v", oldObjectKey, err) | |||
return err | |||
} | |||
err = core.RemoveObject(oldBucket, oldObjectKey) | |||
if err != nil { | |||
log.Error("MoveBucketInOpenIMinio RemoveObject err oldObjectKey=%s .%v", oldObjectKey, err) | |||
} | |||
return err | |||
} | |||
type DecompressReq struct { | |||
SourceFile string `json:"source_file"` | |||
DestPath string `json:"dest_path"` | |||
} | |||
func decompress(sourceFile, destPath string) { | |||
req, _ := json.Marshal(DecompressReq{ | |||
SourceFile: sourceFile, | |||
DestPath: destPath, | |||
}) | |||
err := labelmsg.SendDecompressAttachToLabelOBS(string(req)) | |||
if err != nil { | |||
log.Error("SendDecompressTask to labelsystem (%s) failed:%s", sourceFile, err.Error()) | |||
} | |||
} | |||
func IsNPUModelDirHasFile(jobName string, versionName string) bool { | |||
prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, versionName), "/") | |||
if !strings.HasSuffix(prefix, "/") { | |||
prefix += "/" | |||
} | |||
fileInfos, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, "") | |||
if err != nil { | |||
log.Info("IsNPUModelDirHasFile.get TrainJobListModel failed:", err) | |||
return false | |||
} | |||
if len(fileInfos) > 0 { | |||
return true | |||
} | |||
return len(fileInfos) > 0 | |||
} |
@@ -1,93 +0,0 @@ | |||
package cloudbrainTask | |||
import ( | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/storage" | |||
"errors" | |||
"path" | |||
"strings" | |||
) | |||
func GetModelScheduleStatus(jobId string) (models.ModelScheduleStatus, error) { | |||
job, err := models.GetCloudbrainByJobID(jobId) | |||
if err != nil { | |||
log.Error("GetModelScheduleStatus GetCloudbrainByJobID err.jobId=%s err=%v", jobId, err) | |||
return 0, errors.New("jobId not correct") | |||
} | |||
if !job.IsTerminal() { | |||
log.Info("GetModelScheduleStatus job is not terminal.jobId=%s", jobId) | |||
return models.ModelScheduleWaiting, nil | |||
} | |||
record, err := models.GetScheduleRecordByCloudbrainID(job.ID) | |||
if err != nil { | |||
log.Error("GetModelScheduleStatus GetScheduleRecordByCloudbrainID err.jobId=%s err=%v", jobId, err) | |||
if models.IsErrRecordNotExist(err) { | |||
return models.ModelScheduleSucceed, nil | |||
} | |||
return models.ModelScheduleFailed, err | |||
} | |||
switch record.Status { | |||
case models.StorageUrchinScheduleWaiting: | |||
return models.ModelScheduleWaiting, nil | |||
case models.StorageUrchinScheduleProcessing: | |||
return models.ModelScheduleOperating, nil | |||
case models.StorageUrchinScheduleFailed: | |||
return models.ModelScheduleFailed, nil | |||
case models.StorageUrchinNoFile: | |||
return models.ModelScheduleSucceed, nil | |||
case models.StorageUrchinScheduleSucceed: | |||
moveStatus, err := GetMoveBucketStatus(record, job.JobName, job.VersionName) | |||
if err != nil { | |||
log.Error("GetMoveBucketStatus err.%v", err) | |||
return models.ModelScheduleFailed, err | |||
} | |||
switch moveStatus { | |||
case models.MoveBucketSucceed: | |||
return models.ModelScheduleSucceed, nil | |||
case models.MoveBucketOperating: | |||
return models.ModelScheduleOperating, nil | |||
case models.MoveBucketFailed: | |||
return models.ModelScheduleFailed, nil | |||
} | |||
} | |||
return models.ModelScheduleFailed, nil | |||
} | |||
func GetMoveBucketStatus(record *models.ScheduleRecord, jobName, versionName string) (int, error) { | |||
if record.ComputeSource == models.GPUResource || record.ComputeSource == models.GCUResource { | |||
return record.LocalOperateStatus, nil | |||
} | |||
if record.LocalOperateStatus != models.MoveBucketOperating { | |||
return record.LocalOperateStatus, nil | |||
} | |||
//由于NPU回传后还有异步的解压,所以对于进行中的状态需要进一步查询是否已解压结束 | |||
//判断方法是查询模型目录是否有文件 | |||
if IsNPUModelDirHasFile(jobName, versionName) { | |||
models.UpdateScheduleLocalOperateStatus(record, models.MoveBucketSucceed) | |||
return models.MoveBucketSucceed, nil | |||
} | |||
return record.LocalOperateStatus, nil | |||
} | |||
func IsNPUModelDirHasFile(jobName string, versionName string) bool { | |||
prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, versionName), "/") | |||
if !strings.HasSuffix(prefix, "/") { | |||
prefix += "/" | |||
} | |||
fileInfos, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, "") | |||
if err != nil { | |||
log.Info("IsNPUModelDirHasFile.get TrainJobListModel failed:", err) | |||
return false | |||
} | |||
if len(fileInfos) > 0 { | |||
return true | |||
} | |||
return len(fileInfos) > 0 | |||
} |
@@ -98,9 +98,11 @@ func SyncGrampusNotebookStatus(job *models.Cloudbrain) (*models.Cloudbrain, erro | |||
notification.NotifyChangeCloudbrainStatus(job, oldStatus) | |||
} | |||
if job.ComputeResource == models.NPUResource { | |||
if len(result.JobInfo.Tasks) > 0 { | |||
job.TrainUrl = result.JobInfo.Tasks[0].CodeUrl | |||
job.DataUrl = result.JobInfo.Tasks[0].DataUrl | |||
} | |||
} | |||
err = models.UpdateJob(job) | |||
if err != nil { | |||
log.Error("UpdateJob failed:", err) | |||
@@ -13,8 +13,6 @@ import ( | |||
"strconv" | |||
"strings" | |||
"code.gitea.io/gitea/modules/urfs_client/urchin" | |||
"code.gitea.io/gitea/modules/timeutil" | |||
"code.gitea.io/gitea/modules/notification" | |||
@@ -1103,9 +1101,6 @@ func SyncTaskStatus(task *models.Cloudbrain) error { | |||
task.CorrectCreateUnix() | |||
if oldStatus != task.Status { | |||
notification.NotifyChangeCloudbrainStatus(task, oldStatus) | |||
if models.IsTrainJobTerminal(task.Status) && len(result.JobInfo.Tasks[0].CenterID) == 1 { | |||
go urchin.GetAITaskOutPutBack(task.ID, task.JobName, result.JobInfo.Tasks[0].CenterID[0], task.ComputeResource) | |||
} | |||
} | |||
err = models.UpdateJob(task) | |||
if err != nil { | |||
@@ -65,7 +65,7 @@ | |||
{{if .IsSigned}} | |||
<a href="https://openi.pcl.ac.cn/zeizei/OpenI_Learning/issues/new" class="item" target="_blank"><i class="envelope icon"></i> {{.i18n.Tr "custom.foot.advice_feedback"}}</a> | |||
{{else}} | |||
<a href="{{AppSubUrl}}/user/login" class="item"><i class="envelope icon"></i> {{.i18n.Tr "custom.foot.advice_feedback"}}</a> | |||
<a href="{{AppSubUrl}}/user/login?redirect_to={{AppSubUrl}}/zeizei/OpenI_Learning/issues/new" class="item"><i class="envelope icon"></i> {{.i18n.Tr "custom.foot.advice_feedback"}}</a> | |||
{{end}} | |||
<a href="{{AppSubUrl}}/resource_desc" class="item" target="_blank"><i class="server icon"></i> {{.i18n.Tr "custom.resource_description"}}</a> | |||
@@ -50,7 +50,6 @@ | |||
{{.CsrfTokenHtml}} | |||
<input type="hidden" name="action" value="update"> | |||
<input type="hidden" id="ai_image_name" value="{{.image}}"> | |||
<input type="hidden" id="ai_model_version" name="model_version" value=""> | |||
<input type="hidden" id="failed_train_url" value="{{$.train_url}}"> | |||
<input type="hidden" id="failed_model_name" value="{{$.model_name}}"> | |||
@@ -702,6 +702,7 @@ | |||
type: 'POST', | |||
data: data, | |||
success: function (res) { | |||
const modelName = $('#formId #name').val(); | |||
$('input[name="engine_name"]').val(""); | |||
$('input[name="engine"]').val(""); | |||
$('input[name="jobId"]').val(""); | |||
@@ -710,7 +711,7 @@ | |||
var cityObj = $("#modelSelectedFile"); | |||
cityObj.attr("value", ""); | |||
document.getElementById("formId").reset(); | |||
location.href = `/${userName}/${repoPath}/modelmanage/show_model` | |||
location.href = `/${userName}/${repoPath}/modelmanage/model_readme_tmpl?name=${encodeURIComponent(modelName)}` | |||
$('.ui.modal.second').modal('hide') | |||
}, | |||
error: function (xhr) { | |||
@@ -94,7 +94,7 @@ | |||
{{if eq .ComputeResource "CPU/GPU"}} | |||
<a class="item run_info" data-tab="five{{$k}}" data-version="{{.VersionName}}">{{$.i18n.Tr "repo.cloudbrain.runinfo"}}</a> | |||
{{end}} | |||
<a class="item load-model-file" data-tab="third{{$k}}" data-download-flag="{{$.canDownload}}" data-path="{{$.RepoLink}}/modelarts/train-job/{{.JobID}}/model_list" data-version="{{.VersionName}}" data-parents="" data-filename="" data-init="init" >{{$.i18n.Tr "repo.model_download"}}</a> | |||
<a class="item load-model-file" data-tab="third{{$k}}" data-can-reschedule="{{$.canReschedule}}" data-retry-path="{{$.RepoLink}}/cloudbrain/train-job/{{.JobID}}/model/reschedule" data-download-flag="{{$.canDownload}}" data-path="{{$.RepoLink}}/modelarts/train-job/{{.JobID}}/model_list" data-version="{{.VersionName}}" data-parents="" data-filename="" data-init="init" >{{$.i18n.Tr "repo.model_download"}}</a> | |||
</div> | |||
<div class="ui tab active" data-tab="first{{$k}}"> | |||
<div style="padding-top: 10px;"> | |||
@@ -648,11 +648,9 @@ | |||
} | |||
let dirKey="isOnlyDir--:&"; | |||
function loadSelectedModelFile(trainJob){ | |||
console.log("trainJob=" + trainJob); | |||
$('#choice_file').dropdown('clear') | |||
$("#model-file").empty() | |||
if(trainJob ==null || trainJob ==""){ | |||
console.log("trainJob is null"); | |||
}else{ | |||
let type = trainJob.Type; | |||
if(type == 2){ | |||
@@ -785,6 +783,7 @@ | |||
type: 'POST', | |||
data: data, | |||
success: function (res) { | |||
const modelName = $('#formId #name').val(); | |||
$('input[name="engine_name"]').val(""); | |||
$('input[name="engine"]').val(""); | |||
$('input[name="jobId"]').val(""); | |||
@@ -793,7 +792,7 @@ | |||
var cityObj = $("#modelSelectedFile"); | |||
cityObj.attr("value", ""); | |||
document.getElementById("formId").reset(); | |||
location.href = `/${userName}/${repoPath}/modelmanage/show_model` | |||
location.href = `/${userName}/${repoPath}/modelmanage/model_readme_tmpl?name=${encodeURIComponent(modelName)}` | |||
$('.ui.modal.second').modal('hide') | |||
}, | |||
error: function (xhr) { | |||
@@ -121,7 +121,19 @@ | |||
<!-- 模型版本 --> | |||
<!-- href="{{$.RepoLink}}/modelmanage/model_readme_tmpl?name={{.ModelName}}" --> | |||
<div class="three wide column text center padding0" style="display: flex;width: 17% !important;"> | |||
<a id="{{.JobName}}" class="goto_modelmanage nowrap" title="{{.ModelName}}" href="javascript:void(0);" data-variation="inverted" data-position="top center" data-content="{{$.i18n.Tr "repo.modelarts.infer_job.tooltip"}}" data-jobname={{.JobName}} data-modelname={{.ModelName}} data-version={{.ModelVersion}} data-repopath="{{$.RepoLink}}">{{.ModelName}} </a> <span style="font-size: 12px;">{{.ModelVersion}}</span> | |||
<a id="{{.JobName}}" class="goto_modelmanage nowrap" title="{{.ModelName}}" href="javascript:void(0);" | |||
data-variation="inverted" | |||
data-position="top center" | |||
data-content="{{$.i18n.Tr "repo.modelarts.infer_job.model_cant_see"}}" | |||
data-jobname="{{.JobName}}" | |||
data-modelid="{{.ModelId}}" | |||
data-modelname="{{.ModelName}}" | |||
data-version="{{.ModelVersion}}" | |||
data-repopath="{{$.RepoLink}}" | |||
data-modelrepoownername="{{.ModelRepoOwnerName}}" | |||
data-modelreponame="{{.ModelRepoName}}" | |||
> | |||
{{.ModelName}} </a> <span style="font-size: 12px;">{{.ModelVersion}}</span> | |||
</div> | |||
<!-- 任务状态 --> | |||
<div class="two wide column text center padding0" style="width: 10.5% !important;"> | |||
@@ -49,7 +49,6 @@ | |||
<input type="hidden" name="action" value="update"> | |||
<input type="hidden" id="ai_engine_name" name="engine_names" value=""> | |||
<input type="hidden" id="ai_flaver_name" name="flaver_names" value=""> | |||
<input type="hidden" id="ai_model_version" name="model_version" value=""> | |||
<input type="hidden" id="failed_train_url" value="{{$.train_url}}"> | |||
<input type="hidden" id="failed_model_name" value="{{$.model_name}}"> | |||
@@ -788,6 +788,7 @@ | |||
type: 'POST', | |||
data: data, | |||
success: function (res) { | |||
const modelName = $('#formId #name').val(); | |||
$('input[name="engine_name"]').val(""); | |||
$('input[name="engine"]').val(""); | |||
$('input[name="jobId"]').val(""); | |||
@@ -796,7 +797,7 @@ | |||
var cityObj = $("#modelSelectedFile"); | |||
cityObj.attr("value", ""); | |||
document.getElementById("formId").reset(); | |||
location.href = `/${userName}/${repoPath}/modelmanage/show_model` | |||
location.href = `/${userName}/${repoPath}/modelmanage/model_readme_tmpl?name=${encodeURIComponent(modelName)}` | |||
$('.ui.modal.second').modal('hide') | |||
}, | |||
error: function (xhr) { | |||
@@ -1,5 +1,6 @@ | |||
{{template "base/head_home" .}} | |||
<link rel="stylesheet" href="{{StaticUrlPrefix}}/css/vp-model-settings.css?v={{MD5 AppVer}}" /> | |||
<script>window.REPO_IS_PRIVATE = {{$.Repository.IsPrivate}};</script> | |||
<div id="__vue-root"></div> | |||
<script src="{{StaticUrlPrefix}}/js/vp-model-settings.js?v={{MD5 AppVer}}"></script> | |||
{{template "base/footer" .}} |
@@ -1,4 +1,3 @@ | |||
<div class="{{TabSizeClass .Editorconfig .FileName}} non-diff-file-content gallery"> | |||
<h4 class="file-header ui top attached header"> | |||
<div class="file-header-left"> | |||
@@ -134,8 +133,6 @@ function submitDeleteForm() { | |||
$("#delete-file-form").submit() | |||
} | |||
} | |||
const baseUrls = {}; | |||
const justDomain = /^[^:]+:\/*[^/]*$/; | |||
const protocol = /^([^:]+:)[\s\S]*$/; | |||
@@ -212,4 +209,5 @@ function showNoteBook(){ | |||
} | |||
showNoteBook() | |||
</script> |
@@ -134,11 +134,9 @@ | |||
<div class="space-around" > | |||
<a class="op-btn" | |||
v-show="scope.row.modelType == 1" | |||
:href="url + 'model_setting?type=1&name=' + encodeURIComponent(scope.row.name) + '&id=' + scope.row.id + '&back=' + encodeURIComponent(curHref)" | |||
:class="{ disabled: !scope.row.isCanOper }" | |||
>{{ i18n.modify }}</a> | |||
<a class="op-btn" v-show="scope.row.modelType != 1" style="color:transparent;cursor:default;" >{{ i18n.modify }}</a> | |||
<a class="op-btn" style="color: #13c28d;" v-show="repoIsPrivate == false && scope.row.isPrivate==true && scope.row.isCanOper" @click=" | |||
modifyModelStatus(scope.row.id, scope.row.cName, scope.row.rowKey,false) | |||
">{{ i18n.modelaccess_setpublic }}</a> | |||
@@ -56,21 +56,27 @@ | |||
<div slot="content" class="multiple-wrap"> {{ data.description }}</div> | |||
<span class="model-title model-nowrap"> | |||
<div class="model_flex"> | |||
<span style="flex: inherit" class="model-nowrap">{{ node.label }}</span> | |||
<span style="flex: inherit" class="model-nowrap">{{ node.label }}<span | |||
v-if="(data.version != '0.0.1' || data.new != 1)" class="model-version"> | |||
{{ data.version }} </span> | |||
</span> | |||
<img v-if="data.recommend == 1" style="margin-left: 0.4rem" src="/img/jian.svg" /> | |||
</div> | |||
</span> | |||
</el-tooltip> | |||
<span v-else class="model-title model-nowrap"> | |||
<div class="model_flex"> | |||
<span style="flex: inherit" class="model-nowrap">{{ node.label }}</span> | |||
<span style="flex: inherit" class="model-nowrap">{{ node.label }}<span | |||
v-if="(data.version != '0.0.1' || data.new != 1)" class="model-version"> | |||
{{ data.version }} </span> | |||
</span> | |||
<img v-if="data.recommend == 1" style="margin-left: 0.4rem" src="/img/jian.svg" /> | |||
</div> | |||
</span> | |||
<span class="model-repolink model-nowrap" @click.stop="return false;"> | |||
<a :href="'/' + data.repoOwnerName + '/' + data.repoName + '/modelmanage/model_readme_tmpl?name=' + data.name" | |||
target="_blank"> | |||
{{ data.repoOwnerName }}/{{ data.repoName }} | |||
{{ data.repoOwnerName }}/{{ data.repoDisplayName }} | |||
</a> | |||
</span> | |||
</span> | |||
@@ -213,10 +219,16 @@ export default { | |||
const children = []; | |||
dataI.parent = true; | |||
dataI.disabled = true; | |||
_children.forEach(item => { | |||
item.ModTimeNum = new Date(item.ModTime).getTime(); | |||
}) | |||
_children.sort((a, b) => a.FileName.localeCompare(b.FileName)); | |||
_children.sort((a, b) => b.ModTimeNum - a.ModTimeNum); | |||
for (let j = 0, jLen = _children.length; j < jLen; j++) { | |||
const file = _children[j]; | |||
if (file.IsDir) continue; | |||
const arr = file.FileName.split('.'); | |||
if (!file.IsDir && !supportCheckPointFileExt.includes(arr[arr.length - 1])) continue; | |||
if (!supportCheckPointFileExt.includes(arr[arr.length - 1])) continue; | |||
file._modelID = dataI.id; | |||
file._modelName = dataI.name; | |||
file._modelVersion = dataI.version; | |||
@@ -250,6 +262,7 @@ export default { | |||
q: this.dlgSearchValue.trim(), | |||
page: this.dlgPage, | |||
needModelFile: true, | |||
notNeedEmpty: true, | |||
}; | |||
if (params.queryType == 2 || params.queryType == 4) { | |||
params.orderBy = 'created_unix'; | |||
@@ -665,4 +678,14 @@ export default { | |||
display: flex; | |||
align-items: center; | |||
} | |||
.model-version { | |||
margin-left: 4px; | |||
border-radius: 4px; | |||
color: rgba(16, 16, 16, 0.8); | |||
border-radius: 4px; | |||
font-size: 12px; | |||
background: rgba(220, 220, 220, 0.8); | |||
padding: 1px 3px; | |||
} | |||
</style> |
@@ -565,6 +565,7 @@ export default async function initCloudrainSow() { | |||
activeTab.trigger('click'); | |||
} | |||
// | |||
$(".content-pad").on("click", ".load-model-file", function () { | |||
let downloadFlag = $(this).data("download-flag") || ""; | |||
let gpuFlag = $(this).data("gpu-flag") || ""; | |||
@@ -573,9 +574,12 @@ export default async function initCloudrainSow() { | |||
let filename = $(this).data("filename"); | |||
let init = $(this).data("init") || ""; | |||
let path = $(this).data("path"); | |||
let retryPath = `/api/v1/repos${$(this).data("retry-path")}`; | |||
const rescheduleFlag = $(this).data("can-reschedule") || ""; | |||
$(`#dir_list${version_name}`).empty(); | |||
let url = `/api/v1/repos${path}?version_name=${version_name}&parentDir=${parents}`; | |||
$.get(url, (data) => { | |||
if (data.StatusOK == 0) { // 成功 0 | |||
if (data.Dirs) { | |||
data.Dirs.length !==0 && $(`#${version_name}-result-down`).show() | |||
@@ -595,6 +599,7 @@ export default async function initCloudrainSow() { | |||
$(`#file_breadcrumb${version_name}`).append(htmlBread); | |||
} else { | |||
renderBrend( | |||
this, | |||
path, | |||
version_name, | |||
parents, | |||
@@ -628,12 +633,23 @@ export default async function initCloudrainSow() { | |||
</div>`); | |||
} else if (data.StatusOK == 2) { // 失败 2 | |||
$(`#file_breadcrumb${version_name}`).empty(); | |||
if (rescheduleFlag) { | |||
$(`#dir_list${version_name}`).html(`<div style="height:200px;display:flex;justify-content:center;align-items:center;font-size:14px;color:rgb(16, 16, 16);"> | |||
<div style="display:flex;justify-content:center;align-items:center;height:24px;width:24px;margin-right:5px;"> | |||
<svg xmlns="http://www.w3.org/2000/svg" class="styles__StyledSVGIconPathComponent-sc-16fsqc8-0 iKfgJk svg-icon-path-icon fill" viewBox="64 64 896 896" width="16" height="16"><defs data-reactroot=""></defs><g><path d="M464 720a48 48 0 1 0 96 0 48 48 0 1 0-96 0zm16-304v184c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8V416c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8zm475.7 440l-416-720c-6.2-10.7-16.9-16-27.7-16s-21.6 5.3-27.7 16l-416 720C56 877.4 71.4 904 96 904h832c24.6 0 40-26.6 27.7-48zm-783.5-27.9L512 239.9l339.8 588.2H172.2z"></path></g></svg> | |||
</div> | |||
<span>${i18n['file_sync_fail']}</span> | |||
<a href="javascript:void(0)" id="retry_result" style='text-decoration: underline;margin-left:0.5rem'>${i18n['retrieve_results']}</a> | |||
</div>`); | |||
} | |||
else { | |||
$(`#dir_list${version_name}`).html(`<div style="height:200px;display:flex;justify-content:center;align-items:center;font-size:14px;color:rgb(16, 16, 16);"> | |||
<div style="display:flex;justify-content:center;align-items:center;height:24px;width:24px;margin-right:5px;"> | |||
<svg xmlns="http://www.w3.org/2000/svg" class="styles__StyledSVGIconPathComponent-sc-16fsqc8-0 iKfgJk svg-icon-path-icon fill" viewBox="64 64 896 896" width="16" height="16"><defs data-reactroot=""></defs><g><path d="M464 720a48 48 0 1 0 96 0 48 48 0 1 0-96 0zm16-304v184c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8V416c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8zm475.7 440l-416-720c-6.2-10.7-16.9-16-27.7-16s-21.6 5.3-27.7 16l-416 720C56 877.4 71.4 904 96 904h832c24.6 0 40-26.6 27.7-48zm-783.5-27.9L512 239.9l339.8 588.2H172.2z"></path></g></svg> | |||
</div> | |||
<span>${i18n['file_sync_fail']}</span> | |||
</div>`); | |||
} | |||
} else if (data.StatusOK == 3) { // 等待同步 3 | |||
$(`#file_breadcrumb${version_name}`).empty(); | |||
$(`#dir_list${version_name}`).html(`<div style="height:200px;display:flex;justify-content:center;align-items:center;font-size:14px;color:rgb(16, 16, 16);"> | |||
@@ -651,10 +667,21 @@ export default async function initCloudrainSow() { | |||
<span>${i18n['no_file_to_download']}</span> | |||
</div>`); | |||
} | |||
$('#retry_result').on('click', function () { | |||
$.post(retryPath, (data) => { | |||
if (data.code === 0) { | |||
$('.load-model-file').trigger('click'); | |||
} | |||
}).fail(function (err) { | |||
console.log(err); | |||
}); | |||
}) | |||
}).fail(function (err) { | |||
console.log(err, version_name); | |||
}); | |||
}); | |||
function renderSize(value) { | |||
if (null == value || value == "") { | |||
return "0 Bytes"; | |||
@@ -678,6 +705,7 @@ export default async function initCloudrainSow() { | |||
return size + unitArr[index]; | |||
} | |||
function renderBrend( | |||
that, | |||
path, | |||
version_name, | |||
parents, | |||
@@ -711,15 +739,9 @@ export default async function initCloudrainSow() { | |||
} else { | |||
$(`input[name=model${version_name}]`).val(parents); | |||
$(`input[name=modelback${version_name}]`).val(filename); | |||
let selectEle = $(`#file_breadcrumb${version_name} a.section`).filter( | |||
(index, item) => { | |||
return item.text == filename; | |||
} | |||
); | |||
selectEle.nextAll().remove(); | |||
selectEle.after("<div class='divider'> / </div>"); | |||
selectEle.replaceWith(`<div class='active section'>${filename}</div>`); | |||
$(that).nextAll().remove(); | |||
$(that).after("<div class='divider'> / </div>"); | |||
$(that).replaceWith(`<div class='active section'>${filename}</div>`); | |||
} | |||
} | |||
@@ -404,31 +404,33 @@ export default async function initCloudrain() { | |||
const versionName = this.dataset.version; | |||
stopVersion(versionName, ID, repoPath); | |||
}); | |||
function getModelInfo(repoPath, modelName, versionName, jobName) { | |||
function getModelInfo(repoPath, modelId, modelName, versionName, jobName) { | |||
$.get( | |||
`${repoPath}/modelmanage/show_model_info_api?name=${modelName}`, | |||
`/api/v1/repos${repoPath}/modelmanage/query_model_byId?id=${modelId}`, | |||
(data) => { | |||
if (data.length === 0) { | |||
$(`#${jobName}`).popup("toggle"); | |||
if (data && data.id == modelId) { | |||
location.href = `${repoPath}/modelmanage/model_readme_tmpl?name=${data.name}`; | |||
} else { | |||
let versionData = data.filter((item) => { | |||
return item.version === versionName; | |||
}); | |||
if (versionData.length == 0) { | |||
$(`#${jobName}`).popup("toggle"); | |||
} else { | |||
location.href = `${repoPath}/modelmanage/model_readme_tmpl?name=${modelName}`; | |||
} | |||
} | |||
} | |||
); | |||
).catch(err => { | |||
console.log(err); | |||
$(`#${jobName}`).popup("toggle"); | |||
}); | |||
} | |||
$(".goto_modelmanage").click(function () { | |||
const repoPath = this.dataset.repopath; | |||
let repoPath = this.dataset.repopath; | |||
const modelId = this.dataset.modelid; | |||
const modelName = this.dataset.modelname; | |||
const versionName = this.dataset.version; | |||
const jobName = this.dataset.jobname; | |||
getModelInfo(repoPath, modelName, versionName, jobName); | |||
const modelRepoOwnerName = this.dataset.modelrepoownername; | |||
const modelRepoName = this.dataset.modelreponame; | |||
if (modelRepoOwnerName && modelRepoName) { | |||
repoPath = `/${modelRepoOwnerName}/${modelRepoName}` | |||
} | |||
getModelInfo(repoPath, modelId, modelName, versionName, jobName); | |||
}); | |||
function debugAgain(ID, debugUrl, redirect_to) { | |||
if ($("#" + ID + "-text").text() === "RUNNING") { | |||
@@ -75,6 +75,7 @@ export const i18nVue = { | |||
file_sync_fail:"文件同步失败", | |||
no_file_to_download:"没有文件可以下载,稍后再来看看", | |||
task_not_finished: "任务还未结束,稍后再来看看", | |||
retrieve_results: "重新获取结果", | |||
local:"本地", | |||
online:"线上", | |||
modify:"修改", | |||
@@ -231,6 +232,7 @@ export const i18nVue = { | |||
file_sync_fail:"File synchronization failed", | |||
no_file_to_download:"No files can be downloaded", | |||
task_not_finished: "Task not finished yet, please wait", | |||
retrieve_results: "Retrieve results", | |||
local:"Local", | |||
online:"Online", | |||
modify:"Modify", | |||
@@ -89,6 +89,48 @@ const commentMDEditors = {}; | |||
// Silence fomantic's error logging when tabs are used without a target content element | |||
$.fn.tab.settings.silent = true; | |||
$(document).ready(function(){ | |||
if ($('.file-view.markdown').length && $('.file-view.markdown').length>0) { | |||
setTimeout(()=>{ | |||
const eleList =document.querySelectorAll('div.anchor-wrap .anchor') | |||
const navList = document.querySelectorAll('.markdown_toc .catalog-li') | |||
const toggleIcon = document.getElementsByClassName('toggle-container') | |||
toggleIcon[0].addEventListener('click',function(){ | |||
if(!this.parentElement.classList.contains('hidden')){ | |||
this.parentElement.classList.add('hidden') | |||
this.firstChild.classList.replace('ri-arrow-drop-left-line','ri-arrow-drop-right-line') | |||
}else{ | |||
this.parentElement.classList.remove('hidden') | |||
this.firstChild.classList.replace('ri-arrow-drop-right-line','ri-arrow-drop-left-line') | |||
} | |||
}) | |||
navList[0].classList.add('active') | |||
let timeout = null; | |||
$(window).on('scroll', () => { | |||
if(timeout !== null) clearTimeout(timeout); | |||
timeout = setTimeout(function(){ | |||
let flag = false | |||
eleList.forEach((k,index)=>{ | |||
if(isInviewPort(k) && !flag){ | |||
flag = true | |||
navList.forEach((ele)=>{ | |||
ele.classList.remove('active') | |||
}) | |||
navList[index].classList.add('active') | |||
} | |||
}) | |||
},50) | |||
}) | |||
},0) | |||
} | |||
}) | |||
function isInviewPort(element){ | |||
const viewWidth = window.innerWidth || document.documentElement.clientWidth || '' | |||
const viewHeight = window.innerHeight || document.documentElement.clientHeight || '' | |||
const {top,right,bottom,left} = element.getBoundingClientRect() | |||
return (top>=0 && left>=0 && right<viewWidth && bottom<=viewHeight) | |||
} | |||
function initCommentPreviewTab($form) { | |||
const $tabMenu = $form.find(".tabular.menu"); | |||
$tabMenu.find(".item").tab(); | |||
@@ -1,5 +1,5 @@ | |||
.markdown:not(code) { | |||
overflow: hidden; | |||
// overflow: hidden; | |||
font-size: 16px; | |||
line-height: 1.6 !important; | |||
word-wrap: break-word; | |||
@@ -8,9 +8,9 @@ | |||
padding: 3em; | |||
} | |||
&.file-view { | |||
padding: 2em 2em 2em !important; | |||
} | |||
// &.file-view { | |||
// padding: 2em 2em 2em !important; | |||
// } | |||
> *:first-child { | |||
margin-top: 0 !important; | |||
@@ -524,3 +524,93 @@ | |||
user-select: none; | |||
} | |||
} | |||
.markdown_catalog{ | |||
position: sticky; | |||
top: 1px; | |||
float: left; | |||
} | |||
.markdown_catalog.hidden .scroll-container{ | |||
width: 0px !important; | |||
} | |||
.markdown_catalog .scroll-container { | |||
overflow: hidden; | |||
width: 200px; | |||
min-height: calc(50vh); | |||
-webkit-transition: width .3s ease; | |||
transition: width .3s ease; | |||
} | |||
.markdown_catalog .toggle-container{ | |||
position: absolute; | |||
width: 12px; | |||
height: 56px; | |||
top: 50%; | |||
left: 100%; | |||
background: url("/img/holder.svg"); | |||
-webkit-transform: translate(0, -50%); | |||
transform: translate(0, -50%); | |||
cursor: pointer; | |||
} | |||
.markdown_catalog .toggle-container .icon{ | |||
display: inline-block; | |||
margin-top: 19px; | |||
margin-left: -2px; | |||
color: #B7BDC8; | |||
} | |||
.markdown_catalog .scroll-container .container { | |||
padding: 16px; | |||
width: inherit; | |||
max-height: calc(100vh - 1px); | |||
overflow-x: hidden; | |||
overflow-y: auto; | |||
-webkit-transition: margin-left .3s ease; | |||
transition: margin-left .3s ease; | |||
} | |||
.markdown_catalog .container>.markdown_toc { | |||
padding-left: 0; | |||
font-size: 14px; | |||
} | |||
.markdown_catalog .container .markdown_toc { | |||
margin: 0; | |||
padding-left: 16px; | |||
font-size: 13px; | |||
font-weight: 400; | |||
line-height: 28px; | |||
} | |||
.markdown_catalog .container .catalog-li { | |||
list-style: initial; | |||
list-style-position: inside; | |||
overflow: hidden; | |||
white-space: nowrap; | |||
text-overflow: ellipsis; | |||
} | |||
.markdown_catalog .container .no-catalog-li{ | |||
list-style: none; | |||
} | |||
.markdown_catalog .container .catalog-li.active { | |||
color: #2185d0;; | |||
font-weight: 600; | |||
} | |||
.markdown_catalog .container .catalog-li.active a { | |||
color: #2185d0;; | |||
} | |||
.markdown_catalog .container .catalog-li a { | |||
padding: 6px 0; | |||
outline: none; | |||
color: #40485b; | |||
} | |||
.markdown-content { | |||
min-height: calc(50vh); | |||
border-left: 1px solid #dce3e8; | |||
padding: 1rem 2rem 1rem 2rem; | |||
overflow: auto; | |||
line-height: 1.6; | |||
font-size: 16px; | |||
-webkit-text-size-adjust: 100%; | |||
word-wrap: break-word; | |||
} | |||
html { | |||
scroll-behavior: smooth; | |||
} |
@@ -120,7 +120,7 @@ export const setCompleteMultipart = (data) => { | |||
// data: { userName, repoName, context, text } | |||
export const getMarkdownPreview = (data) => { | |||
return service({ | |||
url: `/api/v1/repos/${data.userName}/${data.repoName}/markdown`, | |||
url: `/api/v1/repos/${data.repoOwnerName}/${data.repoName}/markdown`, | |||
method: 'post', | |||
headers: { 'Content-type': 'application/x-www-form-urlencoded' }, | |||
params: {}, | |||
@@ -10,7 +10,7 @@ | |||
<script> | |||
export default { | |||
name: "NotFind", | |||
name: "NotFound", | |||
props: { | |||
visible: { type: Boolean, default: false }, | |||
}, |
@@ -206,6 +206,7 @@ export default { | |||
q: this.dlgSearchValue.trim(), | |||
page: this.dlgPage, | |||
needModelFile: true, | |||
notNeedEmpty: true, | |||
}; | |||
this.dlgLoading = true; | |||
getModelList(params).then(res => { | |||
@@ -246,6 +246,7 @@ const en = { | |||
importLocalModel: 'Import Local Model', | |||
importOnlineModel: 'Import Online Model', | |||
modelInfo: 'Model information', | |||
modelBriefIntro: 'Model brief introduction', | |||
trainingInfo: 'Training information', | |||
modifyModelInfo: 'Modify model information', | |||
modelFiles: 'Model files', | |||
@@ -262,6 +262,7 @@ const zh = { | |||
importLocalModel: '导入本地模型', | |||
importOnlineModel: '导入线上模型', | |||
modelInfo: '模型信息', | |||
modelBriefIntro: '模型简介', | |||
trainingInfo: '训练相关信息', | |||
modifyModelInfo: '修改模型信息', | |||
modelFiles: '模型文件', | |||
@@ -2,12 +2,12 @@ | |||
<div v-loading="loading"> | |||
<div class="dataset_head_wrap"> | |||
<el-checkbox v-model="checked" style="padding: 0.5rem 1rem;" @change="handleCheckedChange">{{$t('datasets.platform_recommendations')}}</el-checkbox> | |||
<el-dropdown trigger="click" style="cursor: pointer;"> | |||
<el-dropdown trigger="click" style="cursor: pointer;" size="default"> | |||
<span class="el-dropdown-link"> | |||
{{$t('datasets.sort')}}<i class="el-icon-caret-bottom el-icon--right"></i> | |||
</span> | |||
<el-dropdown-menu slot="dropdown"> | |||
<el-dropdown-item :class="{'active':item.active}" v-for="item in sortList" :key="item.name" @click.native="handleSort(item)">{{$t('datasets.'+item.name)}}</el-dropdown-item> | |||
<el-dropdown-item :class="sortSelect == item.name ? 'active' : ''" v-for="item in sortList" :key="item.name" @click.native="handleSort(item)">{{$t('datasets.'+item.name)}}</el-dropdown-item> | |||
</el-dropdown-menu> | |||
</el-dropdown> | |||
</div> | |||
@@ -53,11 +53,11 @@ | |||
:src="`/user/avatar/${item.User.Name}/-1`"> | |||
</a> | |||
<span class="dataset_extra_time">{{item.CreatedUnix | DateTransfer}}</span> | |||
<span class="dataset_extra_link":title="$t('datasets.downloadtimes')"> | |||
<span class="dataset_extra_link" :title="$t('datasets.citations')"> | |||
<i class="ri-link"></i> | |||
<span class="dataset_extra_content">{{item.UseCount}}</span> | |||
</span> | |||
<span class="dataset_extra_download" :title="$t('datasets.citations')"> | |||
<span class="dataset_extra_download" :title="$t('datasets.downloadtimes')"> | |||
<i class="ri-download-line"></i> | |||
<span class="dataset_extra_content">{{item.DownloadTimes}}</span> | |||
</span> | |||
@@ -115,7 +115,15 @@ export default { | |||
licenseValue:{ | |||
type:String, | |||
default:'' | |||
} | |||
}, | |||
sortValue:{ | |||
type:String, | |||
default:'' | |||
}, | |||
recommendValue:{ | |||
type:Boolean, | |||
default:false | |||
}, | |||
}, | |||
data() { | |||
return { | |||
@@ -132,12 +140,11 @@ export default { | |||
license:'' | |||
}, | |||
checked:false, | |||
sortSelect: 'default', | |||
sortList:[ | |||
{name:'default',active:true}, | |||
{name:'latest',active:false}, | |||
{name:'oldest',active:false}, | |||
{name:'recentupdate',active:false}, | |||
{name:'leastupdate',active:false}, | |||
{name:'downloadtimes',active:false}, | |||
{name:'moststars',active:false}, | |||
{name:'mostusecount',active:false}, | |||
@@ -147,37 +154,71 @@ export default { | |||
}; | |||
}, | |||
watch: { | |||
dataGet(newVal) { | |||
this.params.page = 1 | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
}, | |||
searchValue(newVal){ | |||
if (!newVal) { | |||
this.params.page = 1 | |||
this.params.q = newVal | |||
this.getDataList(this.dataGet) | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
} | |||
}, | |||
searchFlag(newVal){ | |||
this.params.page = 1 | |||
this.params.q = this.searchValue | |||
this.getDataList(this.dataGet) | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
}, | |||
categoryValue(val){ | |||
this.params.category = val | |||
this.params.page = 1 | |||
this.getDataList(this.dataGet) | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
}, | |||
taskValue(val){ | |||
this.params.task = val | |||
this.params.page = 1 | |||
this.getDataList(this.dataGet) | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
}, | |||
licenseValue(val){ | |||
this.params.license = val | |||
this.params.page = 1 | |||
this.getDataList(this.dataGet) | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
}, | |||
sortValue(value) { | |||
this.sortSelect = value; | |||
this.params.page = 1 | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
}, | |||
recommendValue(value) { | |||
this.checked = value; | |||
this.params.page = 1 | |||
this.$nextTick(() => { | |||
this.getDataList() | |||
}); | |||
}, | |||
}, | |||
methods: { | |||
getDataList(dataType){ | |||
let url = `/explore/${dataType}` | |||
getDataList(){ | |||
this.params.q = this.searchValue | |||
this.params.category = this.categoryValue | |||
this.params.task = this.taskValue | |||
this.params.license = this.licenseValue | |||
this.params.sort = this.sortValue | |||
this.params.recommend = this.recommendValue | |||
this.sortSelect = this.sortValue | |||
this.checked = this.recommendValue | |||
const dataType = this.dataGet | |||
const url = `/explore/${dataType}` | |||
this.loading=true | |||
getDatasets(url,this.params).then((res)=>{ | |||
if(res.data.result_code==='0'){ | |||
@@ -249,22 +290,15 @@ export default { | |||
}) | |||
}, | |||
handleCheckedChange(val){ | |||
this.params.recommend = val | |||
this.getDataList(this.dataGet) | |||
this.$emit('changeRecommend', val); | |||
}, | |||
handleSort(item){ | |||
this.sortList.forEach(element => { | |||
element.active = false | |||
}); | |||
item.active = true | |||
this.params.sort=item.name | |||
this.getDataList(this.dataGet) | |||
this.$emit('changeSort', item.name); | |||
}, | |||
chooseLabel(item,type){ | |||
const data = {name:item,active:false,type:type} | |||
this.$emit('getLabel',data) | |||
}, | |||
}, | |||
filters:{ | |||
@@ -293,7 +327,6 @@ export default { | |||
font-size: 1rem; | |||
} | |||
.el-dropdown-menu__item { | |||
padding: 2px 1rem; | |||
font-size: 1rem; | |||
} | |||
.el-dropdown-menu__item.active{ | |||
@@ -56,7 +56,7 @@ | |||
</div> | |||
<div class="ui sixteen wide mobile sixteen wide tablet twelve wide computer column"> | |||
<div class="ui sixteen wide column"> | |||
<el-tabs v-model="activeName" @tab-click="handleClick"> | |||
<el-tabs v-model="activeName"> | |||
<el-tab-pane :label="$t('datasets.publick_dataset')" name="public_datasets"> | |||
<div v-if="activeName==='public_datasets'"> | |||
<public-dataset | |||
@@ -67,7 +67,11 @@ | |||
:categoryValue="categoryValue" | |||
:taskValue="taskValue" | |||
:licenseValue="licenseValue" | |||
@getLabel="getChildLabel"> | |||
:sortValue="sortValue" | |||
:recommendValue="recommendValue" | |||
@getLabel="getChildLabel" | |||
@changeSort="changeSort" | |||
@changeRecommend="changeRecommend"> | |||
</public-dataset> | |||
</div> | |||
</el-tab-pane> | |||
@@ -80,7 +84,12 @@ | |||
:searchFlag="searchFlag" | |||
:categoryValue="categoryValue" | |||
:taskValue="taskValue" | |||
:licenseValue="licenseValue"> | |||
:licenseValue="licenseValue" | |||
:sortValue="sortValue" | |||
:recommendValue="recommendValue" | |||
@getLabel="getChildLabel" | |||
@changeSort="changeSort" | |||
@changeRecommend="changeRecommend"> | |||
</public-dataset> | |||
</div> | |||
</el-tab-pane> | |||
@@ -93,7 +102,12 @@ | |||
:searchFlag="searchFlag" | |||
:categoryValue="categoryValue" | |||
:taskValue="taskValue" | |||
:licenseValue="licenseValue"> | |||
:licenseValue="licenseValue" | |||
:sortValue="sortValue" | |||
:recommendValue="recommendValue" | |||
@getLabel="getChildLabel" | |||
@changeSort="changeSort" | |||
@changeRecommend="changeRecommend"> | |||
</public-dataset> | |||
</div> | |||
</el-tab-pane> | |||
@@ -125,19 +139,21 @@ export default { | |||
categoryValue:'', | |||
taskValue:'', | |||
licenseValue:'', | |||
sortValue: 'default', | |||
recommendValue: false, | |||
activeName: 'public_datasets', | |||
isSigned:'false', | |||
searchValue:'', | |||
searchFlag:false | |||
searchFlag:false, | |||
} | |||
}, | |||
computed: { | |||
}, | |||
computed: { }, | |||
methods: { | |||
handleClick(tab, event) { | |||
this.searchValue = '' | |||
this.clearAllSelctLeft() | |||
changeSort(value) { | |||
this.sortValue = value; | |||
}, | |||
changeRecommend(value) { | |||
this.recommendValue = value; | |||
}, | |||
selectCategory(item){ | |||
this.Category.forEach(element => { | |||
@@ -18,11 +18,13 @@ | |||
<div class="sub-title"> | |||
<span> | |||
<span>{{ $t('modelManage.ownerRepository') }}:</span> | |||
<a :href="`/${userName}/${repoName}/modelmanage/show_model`">{{ userName + '/' + repoName }}</a> | |||
<a :href="`/${repoOwnerName}/${repoName}/modelmanage/show_model`"> | |||
{{ repoOwnerName + '/' + (model.repoDisplayName || repoName) }} | |||
</a> | |||
</span> | |||
<span> | |||
<span>{{ $t('modelManage.creator') }}:</span> | |||
<a :href="`/${userName}`">{{ userName }}</a> | |||
<a :href="`/${model.userName}`">{{ model.userName }}</a> | |||
</span> | |||
<span> | |||
<span>{{ $t('datasets.downloadtimes') }}:</span> | |||
@@ -36,24 +38,27 @@ | |||
<div class="tabs-wrap"> | |||
<div class="tabs"> | |||
<div class="tabs-l"> | |||
<a :href="`/${userName}/${repoName}/modelmanage/model_readme_tmpl?name=${encodeURIComponent(modelName)}`"> | |||
<a | |||
:href="`/${repoOwnerName}/${repoName}/modelmanage/model_readme_tmpl?name=${encodeURIComponent(modelName)}`"> | |||
<div class="tab" :class="tab == 'intro' ? 'focus' : ''"> | |||
<i class="ri-send-plane-2-line"></i><span>{{ $t('modelManage.modelIntroduction') }}</span> | |||
</div> | |||
</a> | |||
<a :href="`/${userName}/${repoName}/modelmanage/model_filelist_tmpl?name=${encodeURIComponent(modelName)}`"> | |||
<a | |||
:href="`/${repoOwnerName}/${repoName}/modelmanage/model_filelist_tmpl?name=${encodeURIComponent(modelName)}`"> | |||
<div class="tab" :class="tab == 'files' ? 'focus' : ''"> | |||
<i class="ri-bar-chart-horizontal-line"></i><span>{{ $t('modelManage.modelFiles') }}</span> | |||
</div> | |||
</a> | |||
<a :href="`/${userName}/${repoName}/modelmanage/model_evolution_map?name=${encodeURIComponent(modelName)}`"> | |||
<a | |||
:href="`/${repoOwnerName}/${repoName}/modelmanage/model_evolution_map?name=${encodeURIComponent(modelName)}`"> | |||
<div class="tab" :class="tab == 'graph' ? 'focus' : ''"> | |||
<i class="ri-equalizer-line"></i><span>{{ $t('modelManage.modelEvolutionMap') }}</span> | |||
</div> | |||
</a> | |||
</div> | |||
<div class="tabs-r"> | |||
<a :href="`/${userName}/${repoName}/modelmanage/model_setting?name=${encodeURIComponent(modelName)}`"> | |||
<a :href="`/${repoOwnerName}/${repoName}/modelmanage/model_setting?name=${encodeURIComponent(modelName)}`"> | |||
<div class="tab" :class="tab == 'settings' ? 'focus' : ''" v-if="model && model.isCanOper"> | |||
<i class="ri-settings-2-line"></i><span>{{ $t('modelManage.settings') }}</span> | |||
</div> | |||
@@ -72,7 +77,7 @@ export default { | |||
name: "ModelHeader", | |||
props: { | |||
model: { type: Object, default: () => ({}) }, | |||
userName: { type: String, default: '' }, | |||
repoOwnerName: { type: String, default: '' }, | |||
repoName: { type: String, default: '' }, | |||
modelName: { type: String, default: '' }, | |||
tab: { type: String, default: 'intro' } | |||
@@ -98,12 +103,15 @@ export default { | |||
if (res.data.code == '0') { | |||
this.isCollected = !this.isCollected; | |||
this.collectedCount = this.collectedCount + (this.isCollected ? 1 : -1); | |||
} | |||
if (res.data.code == '401') { | |||
this.$message.success(this.isCollected ? this.$t('datasets.starSuccess') : this.$t('datasets.unstarSuccess')); | |||
} else if (res.data.code == '401') { | |||
window.location.href = `/user/login?redirect_to=${encodeURIComponent(window.location.href)}`; | |||
} else { | |||
this.$message.error(res.data.msg); | |||
} | |||
}).catch(err => { | |||
console.log(err); | |||
this.$message.error(err); | |||
this.isSetting = false; | |||
}); | |||
} | |||
@@ -1,10 +1,11 @@ | |||
<template> | |||
<div> | |||
<div v-if="emptyPage" style="padding-top:50px"> | |||
<NotFind></NotFind> | |||
<NotFound></NotFound> | |||
</div> | |||
<div v-else> | |||
<ModelHeader :tab="'files'" :userName="userName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
<ModelHeader :tab="'files'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName" | |||
:model="modelData"> | |||
</ModelHeader> | |||
<div class="ui container"> | |||
<div class="header"> | |||
@@ -89,7 +90,7 @@ | |||
<script> | |||
import ModelHeader from '../components/ModelHeader.vue'; | |||
import NotFind from '~/components/NotFind.vue'; | |||
import NotFound from '~/components/NotFound.vue'; | |||
import { getModelInfoByName, getModelFiles, deleteModelFile } from '~/apis/modules/modelmanage'; | |||
import { getUrlSearchParams, getListValueWithKey, transFileSize, renderSpecStr } from '~/utils'; | |||
import { MODEL_ENGINES } from '~/const'; | |||
@@ -101,7 +102,7 @@ export default { | |||
emptyPage: false, | |||
modelName: '', | |||
userName: location.pathname.split('/')[1], | |||
repoOwnerName: location.pathname.split('/')[1], | |||
repoName: location.pathname.split('/')[2], | |||
repoUrl: location.pathname.split('/').slice(0, 3).join('/'), | |||
modelData: {}, | |||
@@ -129,7 +130,7 @@ export default { | |||
filePath: [], | |||
}; | |||
}, | |||
components: { ModelHeader, NotFind }, | |||
components: { ModelHeader, NotFound }, | |||
methods: { | |||
getDirFiles(dir) { | |||
dir = dir.length ? dir.slice(1) : ''; | |||
@@ -142,7 +143,8 @@ export default { | |||
list.forEach(item => { | |||
item.SizeShow = item.IsDir ? '' : transFileSize(item.Size); | |||
item.ModTimeNum = new Date(item.ModTime).getTime(); | |||
}) | |||
}); | |||
list.sort((a, b) => a.FileName.localeCompare(b.FileName)); | |||
list.sort((a, b) => b.ModTimeNum - a.ModTimeNum); | |||
this.filesList = list; | |||
this.$refs['tableRef']?.clearSort(); | |||
@@ -164,8 +166,14 @@ export default { | |||
const dir = this.filePath.map((item) => item.path).join('/'); | |||
this.getDirFiles(dir); | |||
}, | |||
changeVersion(version, noFileRefresh) { | |||
changeVersion(version) { | |||
this.curVersion = version; | |||
this.updateModelInfo(); | |||
this._changeVersion(version); | |||
}, | |||
_changeVersion(version, noFileRefresh) { | |||
const data = this.modelList.filter((model) => model.version == version)[0]; | |||
this.modelData = data; | |||
this.modelType = data.modelType; | |||
this.canOperate = data.isCanOper; | |||
this.canDownload = data.isCanDownload; | |||
@@ -252,7 +260,7 @@ export default { | |||
const list = res.data || []; | |||
this.modelList = list; | |||
const noFileRefresh = true; | |||
this.changeVersion(this.curVersion, noFileRefresh); | |||
this._changeVersion(this.curVersion, noFileRefresh); | |||
}).catch(err => { | |||
console.log(err); | |||
}); | |||
@@ -273,8 +281,7 @@ export default { | |||
this.modelList = list; | |||
if (list && list.length) { | |||
const data = list[0]; | |||
this.modelData = data; | |||
this.changeVersion(data.version); | |||
this._changeVersion(data.version); | |||
} else { | |||
this.emptyPage = true; | |||
} | |||
@@ -1,10 +1,10 @@ | |||
<template> | |||
<div> | |||
<div v-if="emptyPage" style="padding-top:50px"> | |||
<NotFind></NotFind> | |||
<NotFound></NotFound> | |||
</div> | |||
<div v-else> | |||
<ModelHeader :tab="'files'" :userName="userName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
<ModelHeader :tab="'files'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
</ModelHeader> | |||
<div class="ui container"> | |||
<div class="header"> | |||
@@ -87,7 +87,7 @@ | |||
<script> | |||
import ModelHeader from '../components/ModelHeader.vue'; | |||
import NotFind from '~/components/NotFind.vue'; | |||
import NotFound from '~/components/NotFound.vue'; | |||
import 'dropzone/dist/dropzone.css'; | |||
import Dropzone from 'dropzone'; | |||
import SparkMD5 from "spark-md5"; | |||
@@ -107,7 +107,7 @@ export default { | |||
emptyPage: false, | |||
modelName: '', | |||
userName: location.pathname.split('/')[1], | |||
repoOwnerName: location.pathname.split('/')[1], | |||
repoName: location.pathname.split('/')[2], | |||
modelData: {}, | |||
@@ -135,7 +135,7 @@ export default { | |||
uploading: false, | |||
}; | |||
}, | |||
components: { ModelHeader, NotFind }, | |||
components: { ModelHeader, NotFound }, | |||
methods: { | |||
initModelData() { | |||
const urlParams = getUrlSearchParams(); | |||
@@ -1,10 +1,10 @@ | |||
<template> | |||
<div> | |||
<div v-if="emptyPage" style="padding-top:50px"> | |||
<NotFind></NotFind> | |||
<NotFound></NotFound> | |||
</div> | |||
<div v-else> | |||
<ModelHeader :tab="'graph'" :userName="userName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
<ModelHeader :tab="'graph'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
</ModelHeader> | |||
<div class="ui container"> | |||
<div ref="graphContainerRef" class="graph-container"></div> | |||
@@ -15,7 +15,7 @@ | |||
<script> | |||
import ModelHeader from '../components/ModelHeader.vue'; | |||
import NotFind from '~/components/NotFind.vue'; | |||
import NotFound from '~/components/NotFound.vue'; | |||
import { getModelInfoByName, getModelEvolutionMap } from '~/apis/modules/modelmanage'; | |||
import { getUrlSearchParams } from '~/utils'; | |||
import { ModelGraph } from './model-graph'; | |||
@@ -29,26 +29,26 @@ export default { | |||
emptyPage: false, | |||
modelName: '', | |||
userName: location.pathname.split('/')[1], | |||
repoOwnerName: location.pathname.split('/')[1], | |||
repoName: location.pathname.split('/')[2], | |||
modelData: {}, | |||
}; | |||
}, | |||
components: { ModelHeader, NotFind }, | |||
components: { ModelHeader, NotFound }, | |||
methods: {}, | |||
beforeMount() { | |||
const urlParams = getUrlSearchParams(); | |||
if (urlParams.name) { | |||
this.modelName = urlParams.name; | |||
this.loading = true; | |||
getModelInfoByName({ repo: `/${this.userName}/${this.repoName}`, name: this.modelName }).then(res => { | |||
getModelInfoByName({ repo: `/${this.repoOwnerName}/${this.repoName}`, name: this.modelName }).then(res => { | |||
const data = res.data; | |||
this.loading = false; | |||
if (data && data.length) { | |||
const model = data[0]; | |||
this.modelData = model; | |||
getModelEvolutionMap({ | |||
repo: `/${this.userName}/${this.repoName}`, | |||
repo: `/${this.repoOwnerName}/${this.repoName}`, | |||
id: model.id, | |||
}).then(res => { | |||
const data = res.data; | |||
@@ -67,7 +67,7 @@ export default { | |||
} | |||
if (node.Type == 0) { | |||
node.type = 'repo'; | |||
node.name = node.RepoOwnerName + ' / ' + `<span style="font-weight:bold">${node.RepoName}</span>`; | |||
node.name = node.RepoOwnerName + ' / ' + `<span style="font-weight:bold">${node.RepoDisplayName}</span>`; | |||
node.link = `/${node.RepoOwnerName}/${node.RepoName}`; | |||
} | |||
node.children = node.Next || []; | |||
@@ -255,8 +255,9 @@ ModelGraph.prototype.showModelNodeInfo = function (nodeEl, nodeData, showOrHide) | |||
let modelObj = { ...model }; | |||
const trainTaskInfo = model.trainTaskInfo ? JSON.parse(model.trainTaskInfo) : ''; | |||
if (trainTaskInfo) { | |||
trainTaskInfo.DisplayJobName = trainTaskInfo.DisplayJobName == undefined ? '' : trainTaskInfo.DisplayJobName; | |||
const taskType = trainTaskInfo.Type; | |||
let taskUrl = location.href.split('modelmanage')[0]; | |||
let taskUrl = `/${model.repoOwnerName}/${model.repoName}/`; | |||
if (taskType == 0) { | |||
taskUrl = taskUrl + 'cloudbrain/train-job/' + trainTaskInfo.JobID; | |||
} else if (taskType == 1) { | |||
@@ -270,7 +271,7 @@ ModelGraph.prototype.showModelNodeInfo = function (nodeEl, nodeData, showOrHide) | |||
} catch (e) { | |||
specObj = trainTaskInfo.FlavorName; | |||
} | |||
modelObj.displayJobNameHtml = `<a target="_blank" href="${taskUrl}" title="${trainTaskInfo.DisplayJobName}">${trainTaskInfo.DisplayJobName}</a>`; | |||
modelObj.displayJobNameHtml = `<a target="_blank" href="${trainTaskInfo.DisplayJobName ? taskUrl : 'javascript:;'}" title="${trainTaskInfo.DisplayJobName}">${trainTaskInfo.DisplayJobName}</a>`; | |||
modelObj.trainJobDuration = trainTaskInfo.TrainJobDuration; | |||
modelObj.sepcStr = typeof specObj == 'object' ? renderSpecStr(specObj, false) : specObj; | |||
} | |||
@@ -317,7 +318,7 @@ ModelGraph.prototype.showModelNodeInfo = function (nodeEl, nodeData, showOrHide) | |||
</div>`; | |||
const posInfo = nodeEl.getBoundingClientRect(); | |||
const winW = window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth; | |||
showInfoEl.style.top = Math.max(posInfo.top - 278 + 6, 20) + 'px'; | |||
showInfoEl.style.top = Math.max(posInfo.top - 258 + 6, 20) + 'px'; | |||
showInfoEl.style.left = Math.min(posInfo.left - 6 + posInfo.width, winW - 250 - 20) + 'px'; | |||
document.querySelector('body').append(showInfoEl); | |||
const mouseEnter = function () { | |||
@@ -1,10 +1,10 @@ | |||
<template> | |||
<div> | |||
<div v-if="emptyPage" style="padding-top:50px"> | |||
<NotFind></NotFind> | |||
<NotFound></NotFound> | |||
</div> | |||
<div v-else> | |||
<ModelHeader :tab="'intro'" :userName="userName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
<ModelHeader :tab="'intro'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
</ModelHeader> | |||
<div class="ui container content"> | |||
<div class="content-l" v-loading="loading"> | |||
@@ -77,7 +77,7 @@ | |||
</div> | |||
</div> | |||
<div class="content-r" v-show="!editing"> | |||
<div class="title">{{ $t('modelManage.briefIntroduction') }}</div> | |||
<div class="title">{{ $t('modelManage.modelBriefIntro') }}</div> | |||
<div class="descr">{{ modelData.description }}</div> | |||
<div class="labels"> | |||
<span class="label" v-for="(item, index) in labels" :key="index">{{ item }}</span> | |||
@@ -119,7 +119,7 @@ | |||
<script> | |||
import ModelHeader from '../components/ModelHeader.vue'; | |||
import NotFind from '~/components/NotFind.vue'; | |||
import NotFound from '~/components/NotFound.vue'; | |||
import { getUrlSearchParams, setWebpackPublicPath, getListValueWithKey, transFileSize } from '~/utils'; | |||
import { getModelInfoByName, getMarkdownPreview, getModelIntroduce, setModelIntroduce } from '~/apis/modules/modelmanage'; | |||
import { MODEL_ENGINES } from '~/const'; | |||
@@ -131,7 +131,7 @@ export default { | |||
emptyPage: false, | |||
modelName: '', | |||
userName: location.pathname.split('/')[1], | |||
repoOwnerName: location.pathname.split('/')[1], | |||
repoName: location.pathname.split('/')[2], | |||
loading: false, | |||
modelData: {}, | |||
@@ -158,7 +158,7 @@ export default { | |||
submitLoading: false, | |||
}; | |||
}, | |||
components: { ModelHeader, NotFind }, | |||
components: { ModelHeader, NotFound }, | |||
methods: { | |||
createIntro() { | |||
this.content = this.emptyDefaultContent; | |||
@@ -169,7 +169,7 @@ export default { | |||
this.editTab = tab; | |||
if (tab == 'preview') { | |||
getMarkdownPreview({ | |||
userName: this.userName, | |||
repoOwnerName: this.repoOwnerName, | |||
repoName: this.repoName, | |||
content: '', | |||
text: this.editContent, | |||
@@ -185,7 +185,7 @@ export default { | |||
} | |||
}, | |||
addLabel() { | |||
window.location.href = `/${this.userName}/${this.repoName}/modelmanage/model_setting?name=${this.modelName}`; | |||
window.location.href = `/${this.repoOwnerName}/${this.repoName}/modelmanage/model_setting?name=${this.modelName}`; | |||
}, | |||
toggleEdit(state) { | |||
if (state) { | |||
@@ -235,7 +235,7 @@ export default { | |||
getIntroInfo() { | |||
if (!this.modelData.id) return; | |||
this.loading = true; | |||
getModelIntroduce({ repo: `/${this.userName}/${this.repoName}`, id: this.modelData.id }).then(res => { | |||
getModelIntroduce({ repo: `/${this.repoOwnerName}/${this.repoName}`, id: this.modelData.id }).then(res => { | |||
const data = res.data; | |||
if (data && data.code == 0) { | |||
if (data.isExistMDFile == 'false') { | |||
@@ -266,7 +266,7 @@ export default { | |||
} | |||
this.submitLoading = true; | |||
setModelIntroduce({ | |||
repo: `/${this.userName}/${this.repoName}`, | |||
repo: `/${this.repoOwnerName}/${this.repoName}`, | |||
id: this.modelData.id, | |||
content: this.editContent, | |||
}).then(res => { | |||
@@ -297,17 +297,18 @@ export default { | |||
if (urlParams.name) { | |||
this.modelName = urlParams.name; | |||
this.loading = true; | |||
getModelInfoByName({ repo: `/${this.userName}/${this.repoName}`, name: this.modelName }).then(res => { | |||
getModelInfoByName({ repo: `/${this.repoOwnerName}/${this.repoName}`, name: this.modelName }).then(res => { | |||
const data = res.data; | |||
this.loading = false; | |||
if (data && data.length) { | |||
const model = data[0]; | |||
const trainTaskInfo = model.trainTaskInfo ? JSON.parse(model.trainTaskInfo) : ''; | |||
if (trainTaskInfo) { | |||
trainTaskInfo.DisplayJobName = trainTaskInfo.DisplayJobName == undefined ? '' : trainTaskInfo.DisplayJobName; | |||
const taskType = trainTaskInfo.Type; | |||
const versionName = trainTaskInfo.VersionName; | |||
const versionHtml = versionName ? `<span class="append-txt" title="${versionName}">${versionName}</span>` : ''; | |||
let taskUrl = location.href.split('modelmanage')[0]; | |||
let taskUrl = `/${model.repoOwnerName}/${model.repoName}/`; | |||
if (taskType == 0) { | |||
taskUrl = taskUrl + 'cloudbrain/train-job/' + trainTaskInfo.JobID; | |||
} else if (taskType == 1) { | |||
@@ -315,7 +316,7 @@ export default { | |||
} else if (taskType == 2) { | |||
taskUrl = taskUrl + 'grampus/train-job/' + trainTaskInfo.JobID; | |||
} | |||
model.displayJobNameHtml = `<a target="_blank" href="${taskUrl}" title="${trainTaskInfo.DisplayJobName}">${trainTaskInfo.DisplayJobName}</a>${versionHtml}`; | |||
model.displayJobNameHtml = `<a target="_blank" href="${trainTaskInfo.DisplayJobName ? taskUrl : 'javascript:;'}" title="${trainTaskInfo.DisplayJobName}">${trainTaskInfo.DisplayJobName}</a>${versionHtml}`; | |||
} | |||
this.modelData = { | |||
...model, | |||
@@ -324,7 +325,7 @@ export default { | |||
createTimeStr: formatDate(new Date(model.createdUnix * 1000), 'yyyy-MM-dd HH:mm:ss'), | |||
isPrivateStr: model.isPrivate ? this.$t('modelManage.modelAccessPrivate') : this.$t('modelManage.modelAccessPublic'), | |||
}; | |||
this.labels = model.label ? model.label.split(' ') : []; | |||
this.labels = model.label ? model.label.trim().split(/\s+/) : []; | |||
this.canEdit = model.isCanOper; | |||
this.getIntroInfo(); | |||
} else { | |||
@@ -68,7 +68,7 @@ | |||
</div> | |||
</div> | |||
<div class="row" style="align-items:flex-start;"> | |||
<div class="r-title"><label>{{ $t('modelManage.modelDescr') }}</label></div> | |||
<div class="r-title"><label>{{ $t('modelManage.modelBriefIntro') }}</label></div> | |||
<div class="r-content"> | |||
<el-input type="textarea" :maxLength="255" size="medium" v-model="state.description" :rows="3" | |||
:placeholder="$t('modelManage.modelDescrInputTips')"> | |||
@@ -152,6 +152,7 @@ export default { | |||
submintApi({ | |||
repo: location.pathname.split('/').slice(0, 3).join('/'), | |||
...this.state, | |||
label: this.state.label.split(/\s+/).join(' ').trim(), | |||
}).then(res => { | |||
res = res.data; | |||
if (res && res.code == '0') { | |||
@@ -1,10 +1,11 @@ | |||
<template> | |||
<div> | |||
<div v-if="emptyPage" style="padding-top:50px"> | |||
<NotFind></NotFind> | |||
<NotFound></NotFound> | |||
</div> | |||
<div v-else> | |||
<ModelHeader :tab="'settings'" :userName="userName" :repoName="repoName" :modelName="modelName" :model="modelData"> | |||
<ModelHeader :tab="'settings'" :repoOwnerName="repoOwnerName" :repoName="repoName" :modelName="modelName" | |||
:model="modelData"> | |||
</ModelHeader> | |||
<div class="ui container content-wrap"> | |||
<div class="header"> | |||
@@ -51,7 +52,7 @@ | |||
</div> | |||
</div> | |||
<div class="row" style="align-items:flex-start;"> | |||
<div class="r-title"><label>{{ $t('modelManage.modelDescr') }}</label></div> | |||
<div class="r-title"><label>{{ $t('modelManage.modelBriefIntro') }}</label></div> | |||
<div class="r-content"> | |||
<el-input type="textarea" :maxLength="255" size="medium" v-model="state.description" :rows="3" | |||
:placeholder="$t('modelManage.modelDescrInputTips')"> | |||
@@ -74,7 +75,7 @@ | |||
<script> | |||
import ModelHeader from '../components/ModelHeader.vue'; | |||
import NotFind from '~/components/NotFind.vue'; | |||
import NotFound from '~/components/NotFound.vue'; | |||
import { getUrlSearchParams } from '~/utils'; | |||
import { saveLocalModel, getModelInfoByName, modifyModel } from '~/apis/modules/modelmanage'; | |||
import { MODEL_ENGINES } from '~/const'; | |||
@@ -90,7 +91,7 @@ export default { | |||
backUrl: '', | |||
modelName: '', | |||
userName: location.pathname.split('/')[1], | |||
repoOwnerName: location.pathname.split('/')[1], | |||
repoName: location.pathname.split('/')[2], | |||
modelData: {}, | |||
loading: false, | |||
@@ -106,10 +107,10 @@ export default { | |||
nameErr: false, | |||
isShowVersion: false, | |||
engineList: MODEL_ENGINES, | |||
repoIsPrivate: false, | |||
repoIsPrivate: REPOISPRIVATE, | |||
}; | |||
}, | |||
components: { ModelHeader, NotFind }, | |||
components: { ModelHeader, NotFound }, | |||
methods: { | |||
checkName() { | |||
this.nameErr = !this.state.name; | |||
@@ -144,7 +145,8 @@ export default { | |||
modifyModel({ | |||
repo: location.pathname.split('/').slice(0, 3).join('/'), | |||
...this.state, | |||
isPrivate: this.state.isPrivate == 1 ? true : false, | |||
label: this.state.label.split(/\s+/).join(' ').trim(), | |||
isPrivate: (this.repoIsPrivate || this.state.isPrivate == 1) ? true : false, | |||
}).then(res => { | |||
res = res.data; | |||
if (res && res.code == '0') { | |||
@@ -41,6 +41,7 @@ export default { | |||
components: {}, | |||
data() { | |||
return { | |||
isLogin: false, | |||
tabList: [{ | |||
key: '1', | |||
label: this.$t('modelObj.model_public'), | |||
@@ -54,6 +55,12 @@ export default { | |||
sortList: [{ | |||
key: '', | |||
label: this.$t('datasets.default'), | |||
}, { | |||
key: 'created_unix', | |||
label: this.$t('datasets.latest'), | |||
}, { | |||
key: 'updated_unix', | |||
label: this.$t('datasets.recentupdate'), | |||
}, { | |||
key: 'download_count', | |||
label: this.$t('datasets.downloadtimes'), | |||
@@ -63,9 +70,6 @@ export default { | |||
}, { | |||
key: 'reference_count', | |||
label: this.$t('datasets.mostusecount'), | |||
}, { | |||
key: 'created_unix', | |||
label: this.$t('datasets.latest'), | |||
}], | |||
conds: { | |||
tab: 'public', | |||
@@ -104,7 +108,12 @@ export default { | |||
deep: true, | |||
}, | |||
}, | |||
beforeMount() { }, | |||
beforeMount() { | |||
this.isLogin = !!document.querySelector('meta[name="_uid"]'); | |||
if (!this.isLogin) { | |||
this.tabList.splice(1, Infinity); | |||
} | |||
}, | |||
mounted() { }, | |||
}; | |||
</script> | |||
@@ -118,7 +127,7 @@ export default { | |||
align-items: center; | |||
.tab-item { | |||
margin-right: 16px; | |||
margin-right: 32px; | |||
padding: 4px 0; | |||
text-align: center; | |||
font-size: 14px; | |||
@@ -138,7 +147,8 @@ export default { | |||
display: flex; | |||
align-items: center; | |||
justify-content: space-between; | |||
margin-top: 16px; | |||
margin-top: 20px; | |||
margin-bottom: 8px; | |||
.only-recommend-c { | |||
margin-right: 22px; | |||
@@ -1,14 +1,15 @@ | |||
<template> | |||
<div class="item"> | |||
<a class="item" target="_blank" | |||
:href="`/${data.repoOwnerName}/${data.repoName}/modelmanage/model_readme_tmpl?name=${data.name}`"> | |||
<div class="item-top"> | |||
<div class="title-c"> | |||
<div class="title"> | |||
<a target="_blank" | |||
:href="`/${data.repoOwnerName}/${data.repoName}/modelmanage/model_readme_tmpl?name=${data.name}`"> | |||
<span> | |||
<span :title="data.name">{{ data.name }}</span> | |||
</a> | |||
</span> | |||
<img v-show="data.recommend == 1" src="/img/jian.svg" style="margin-left:4px"> | |||
</div> | |||
<div class="fav-c" @click="changeFav(data)"> | |||
<div class="fav-c" :class="condition.tab == 2 ? 'fav-disabled' : ''" @click.prevent.stop="changeFav(data)"> | |||
<i v-if="!isCollected" class="heart outline icon" :title="$t('star')"></i> | |||
<i v-if="isCollected" class="heart icon" :title="$t('unStar')"></i> | |||
<span>{{ collectedCount }}</span> | |||
@@ -18,6 +19,7 @@ | |||
<a v-for="(item, index) in data.labels" :key="index" class="label">{{ item }}</a> | |||
</div> | |||
<div class="descr" :title="data.description"> {{ data.description }} </div> | |||
</div> | |||
<div class="footer"> | |||
<div class="footer-l"> | |||
<a :href="`/${data.userName}`" class="avatar-c"> | |||
@@ -37,7 +39,7 @@ | |||
<span class="item-engine" style="margin-left:5px;"> {{ data.engineName || data.engine }} </span> | |||
</div> | |||
</div> | |||
</div> | |||
</a> | |||
</template> | |||
<script> | |||
@@ -46,6 +48,7 @@ import { setModelFav } from '~/apis/modules/modelsquare'; | |||
export default { | |||
name: "ModelItem", | |||
props: { | |||
condition: { type: Object, default: () => ({}) }, | |||
data: { type: Object, default: () => ({}) }, | |||
}, | |||
components: {}, | |||
@@ -58,6 +61,7 @@ export default { | |||
}, | |||
methods: { | |||
changeFav(item) { | |||
if (this.condition.tab == 2) return; | |||
if (this.isSetting) return; | |||
this.isSetting = true; | |||
setModelFav({ | |||
@@ -68,12 +72,16 @@ export default { | |||
if (res.data.code == '0') { | |||
this.isCollected = !this.isCollected; | |||
this.collectedCount = this.collectedCount + (this.isCollected ? 1 : -1); | |||
} | |||
if (res.data.code == '401') { | |||
this.$message.success(this.isCollected ? this.$t('datasets.starSuccess') : this.$t('datasets.unstarSuccess')); | |||
this.$emit('changeFav'); | |||
} else if (res.data.code == '401') { | |||
window.location.href = `/user/login?redirect_to=${encodeURIComponent(window.location.href)}`; | |||
} else { | |||
this.$message.error(res.data.msg); | |||
} | |||
}).catch(err => { | |||
console.log(err); | |||
this.$message.error(err); | |||
this.isSetting = false; | |||
}); | |||
} | |||
@@ -95,6 +103,10 @@ export default { | |||
overflow: hidden; | |||
padding: 15px; | |||
background: transparent; | |||
display: flex; | |||
justify-content: space-between; | |||
flex-direction: column; | |||
height: 100%; | |||
&:hover { | |||
border-color: rgba(204, 204, 255, 0.6); | |||
@@ -145,10 +157,15 @@ export default { | |||
span { | |||
color: rgb(16, 16, 16); | |||
} | |||
&.fav-disabled { | |||
i { | |||
cursor: default; | |||
} | |||
} | |||
} | |||
.descr { | |||
height: 40px; | |||
margin-top: 8px; | |||
font-size: 12px; | |||
font-weight: 300; | |||
@@ -167,7 +184,6 @@ export default { | |||
text-overflow: ellipsis; | |||
white-space: nowrap; | |||
margin-top: 8px; | |||
height: 20px; | |||
.label { | |||
color: rgba(16, 16, 16, 0.8); | |||
@@ -2,7 +2,7 @@ | |||
<div class="list-container"> | |||
<div class="list-item-container" v-loading="loading"> | |||
<div class="item-container" v-for="(item, index) in list" :key="item.ID"> | |||
<ModelItem :data="item"></ModelItem> | |||
<ModelItem :data="item" :condition="condition" @changeFav="changeFav"></ModelItem> | |||
</div> | |||
<div v-show="(!list.length && !loading)" class="no-data"> | |||
<div class="item-empty"> | |||
@@ -37,7 +37,7 @@ export default { | |||
return { | |||
loading: false, | |||
list: [], | |||
iPageSizes: [30, 50], | |||
iPageSizes: [30], | |||
iPageSize: 30, | |||
iPage: 1, | |||
total: 0, | |||
@@ -55,6 +55,7 @@ export default { | |||
label: this.condition.label, | |||
page: this.condition.page, | |||
pageSize: this.condition.pageSize, | |||
notNeedEmpty: true, | |||
}).then(res => { | |||
res = res.data; | |||
this.loading = false; | |||
@@ -62,7 +63,7 @@ export default { | |||
this.list = (res.data || []).map(item => { | |||
return { | |||
...item, | |||
labels: item.label ? item.label.split(' ') : [], | |||
labels: item.label ? item.label.trim().split(/\s+/) : [], | |||
engineName: getListValueWithKey(MODEL_ENGINES, item.engine.toString()), | |||
createTimeStr: formatDate(new Date(item.createdUnix * 1000), 'yyyy-MM-dd'), | |||
} | |||
@@ -77,6 +78,11 @@ export default { | |||
search() { | |||
this.getListData(); | |||
}, | |||
changeFav() { | |||
if (this.condition.tab == '3') { | |||
this.getListData(); | |||
} | |||
}, | |||
currentChange(page) { | |||
this.iPage = page; | |||
this.$emit('changeCondition', { | |||
@@ -1,10 +1,16 @@ | |||
<template> | |||
<div> | |||
<div class="search-bar-wrap"> | |||
<div class="search-bar"> | |||
<input type="text" v-model="condition.q" :placeholder="$t('modelObj.model_search')" | |||
<div class="explore repositories"> | |||
<div class="repos--seach"> | |||
<div class="ui container"> | |||
<div class="search-bar-wrap ui two column centered grid"> | |||
<div class="search-bar fourteen wide mobile ten wide tablet ten wide computer column ui form ignore-dirty"> | |||
<div class="ui fluid action input"> | |||
<input type="text" v-model="condition.q" :placeholder="$t('modelObj.model_search')" :autofocus="true" | |||
@keyup.enter="conditionChange" /> | |||
<button @click="conditionChange">{{ $t('repos.search') }}</button> | |||
<button class="ui green button" @click="conditionChange">{{ $t('repos.search') }}</button> | |||
</div> | |||
</div> | |||
</div> | |||
</div> | |||
</div> | |||
<div class="ui container"> | |||
@@ -40,7 +46,7 @@ export default { | |||
page: 1, | |||
pageSize: 30, | |||
}, | |||
pageSizes: [30, 50], | |||
pageSizes: [30], | |||
}; | |||
}, | |||
components: { ModelFilters, ModelCondition, ModelList }, | |||
@@ -61,7 +67,7 @@ export default { | |||
`&label=${encodeURIComponent(this.condition.label)}` + | |||
`&page=${encodeURIComponent(this.condition.page)}` + | |||
`&pageSize=${encodeURIComponent(this.condition.pageSize)}`; | |||
} | |||
}, | |||
}, | |||
beforeMount() { | |||
const urlParams = getUrlSearchParams(); | |||
@@ -84,50 +90,9 @@ export default { | |||
</script> | |||
<style scoped lang="less"> | |||
.search-bar-wrap { | |||
height: 88px; | |||
display: flex; | |||
align-items: center; | |||
justify-content: center; | |||
background: #f5f5f6; | |||
.search-bar { | |||
height: 40px; | |||
font-size: 14px; | |||
text-align: left; | |||
display: flex; | |||
input { | |||
width: 602px; | |||
height: 100%; | |||
padding: 0 8px; | |||
outline: none; | |||
border-color: #bbbbbb; | |||
border-width: 1px; | |||
border-style: solid; | |||
border-radius: 5px 0px 0px 5px; | |||
&:focus { | |||
border-color: #85b7d9; | |||
-webkit-box-shadow: 0 0 0 0 rgba(34, 36, 38, .35) inset; | |||
box-shadow: 0 0 0 0 rgba(34, 36, 38, .35) inset | |||
} | |||
} | |||
button { | |||
padding: 0 20px; | |||
height: 40px; | |||
color: #ffffff; | |||
border-radius: 0px 4px 4px 0px; | |||
background: #5bb973; | |||
border: none; | |||
cursor: pointer; | |||
} | |||
} | |||
} | |||
.content { | |||
display: flex; | |||
margin-top: -6px; | |||
.content-l { | |||
flex: 1; | |||
Dear OpenI User
Thank you for your continuous support to the Openl Qizhi Community AI Collaboration Platform. In order to protect your usage rights and ensure network security, we updated the Openl Qizhi Community AI Collaboration Platform Usage Agreement in January 2024. The updated agreement specifies that users are prohibited from using intranet penetration tools. After you click "Agree and continue", you can continue to use our services. Thank you for your cooperation and understanding.
For more agreement content, please refer to the《Openl Qizhi Community AI Collaboration Platform Usage Agreement》