diff --git a/custom/public/img/logo-footer.svg b/custom/public/img/logo-footer.svg index 3be0e5b18c..f226958e33 100644 --- a/custom/public/img/logo-footer.svg +++ b/custom/public/img/logo-footer.svg @@ -1 +1,183 @@ - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + 新一代人工智能开源开放平台 + + + + + + + + + diff --git a/entity/ai_task.go b/entity/ai_task.go index da78e29abf..1ffe8b9e21 100644 --- a/entity/ai_task.go +++ b/entity/ai_task.go @@ -2,8 +2,6 @@ package entity import ( "archive/zip" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/storage" "encoding/json" "fmt" "io" @@ -11,6 +9,10 @@ import ( "strings" "sync" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/role" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" @@ -105,6 +107,7 @@ type AITaskDetailInfo struct { ID int64 `json:"id"` JobID string `json:"job_id"` Status string `json:"status"` + DetailedStatus string `json:"detailed_status"` JobType string `json:"job_type"` Cluster string `json:"cluster"` DisplayJobName string `json:"display_job_name"` @@ -151,7 +154,7 @@ func (a *AITaskDetailInfo) Tr(language string) { } func (a *AITaskDetailInfo) TryToRemoveDatasets(currentUser *models.User) { - if currentUser == nil || a.UserId == 0 || (!currentUser.IsAdmin && currentUser.ID != a.UserId) { + if currentUser == nil || a.UserId == 0 || (!currentUser.IsAdmin && currentUser.ID != a.UserId && !role.UserHasRole(currentUser.ID, models.MonitorAdmin)) { a.DatasetList = []*models.DatasetDownload{} } } @@ -198,10 +201,19 @@ type GetAITaskCreationInfoReq struct { IsOnlineType bool } +type GetAITaskCreationImageInfoReq struct { + JobType models.JobType + ClusterType ClusterType + ComputeSource *models.ComputeSource + Spec *models.Specification + UserID int64 +} + type AITaskBriefInfo struct { ID int64 `json:"id"` JobType string `json:"job_type"` Status string `json:"status"` + DetailedStatus string `json:"detailed_status"` DisplayJobName string `json:"display_job_name"` CreatedUnix timeutil.TimeStamp `json:"created_unix"` StartTime timeutil.TimeStamp `json:"start_time"` @@ -239,6 +251,7 @@ func ConvertCloudbrainToAITaskBriefInfo(task *models.Cloudbrain) *AITaskBriefInf ID: task.ID, JobType: task.JobType, Status: task.Status, + DetailedStatus: task.DetailedStatus, DisplayJobName: task.DisplayJobName, CreatedUnix: task.CreatedUnix, FormattedDuration: task.TrainJobDuration, diff --git a/entity/cluster.go b/entity/cluster.go index ce65cc669e..70667178f1 100644 --- a/entity/cluster.go +++ b/entity/cluster.go @@ -63,19 +63,20 @@ type CreateTrainTaskRequest struct { } type QueryTaskResponse struct { - StartedAt timeutil.TimeStamp `json:"started_at"` - CompletedAt timeutil.TimeStamp `json:"completed_at"` - JobId string `json:"job_id"` - Status string `json:"status"` - Url string `json:"url"` - Token string `json:"token"` - CenterId string `json:"center_id"` - CenterName string `json:"center_name"` - CodeUrl string `json:"code_url"` - DataUrl string `json:"data_url"` - ContainerIP string `json:"container_ip"` - ContainerID string `json:"container_id"` - VersionId int64 `json:"version_id"` + StartedAt timeutil.TimeStamp `json:"started_at"` + CompletedAt timeutil.TimeStamp `json:"completed_at"` + JobId string `json:"job_id"` + Status string `json:"status"` + DetailedStatus string `json:"detailed_status"` + Url string `json:"url"` + Token string `json:"token"` + CenterId string `json:"center_id"` + CenterName string `json:"center_name"` + CodeUrl string `json:"code_url"` + DataUrl string `json:"data_url"` + ContainerIP string `json:"container_ip"` + ContainerID string `json:"container_id"` + VersionId int64 `json:"version_id"` } func ConvertGrampusNotebookResponse(job models.GrampusNotebookInfo) *QueryTaskResponse { @@ -100,16 +101,17 @@ func ConvertGrampusNotebookResponse(job models.GrampusNotebookInfo) *QueryTaskRe dataUrl = t.DataUrl } return &QueryTaskResponse{ - StartedAt: timeutil.TimeStamp(job.StartedAt), - CompletedAt: timeutil.TimeStamp(job.CompletedAt), - Status: job.Status, - CenterId: centerId, - CenterName: centerName, - Url: url, - Token: token, - JobId: job.JobID, - CodeUrl: codeUrl, - DataUrl: dataUrl, + StartedAt: timeutil.TimeStamp(job.StartedAt), + CompletedAt: timeutil.TimeStamp(job.CompletedAt), + Status: job.Status, + DetailedStatus: job.DetailedStatus, + CenterId: centerId, + CenterName: centerName, + Url: url, + Token: token, + JobId: job.JobID, + CodeUrl: codeUrl, + DataUrl: dataUrl, } } func ConvertGrampusTrainResponse(job models.GrampusJobInfo) *QueryTaskResponse { diff --git a/entity/creation.go b/entity/creation.go index 1b6a822065..1add1003e9 100644 --- a/entity/creation.go +++ b/entity/creation.go @@ -21,6 +21,11 @@ type CreationRequiredInfo struct { AllowedWorkerNum []int `json:"allowed_worker_num"` } +type ImageRequiredInfo struct { + Images []ClusterImage `json:"images"` + CanUseAllImages bool `json:"can_use_all_images"` +} + type AITaskCreationConfig struct { DatasetMaxSize int `json:"dataset_max_size"` } diff --git a/manager/client/grampus/grampus.go b/manager/client/grampus/grampus.go index 7b15ddc16e..47f08bde2c 100644 --- a/manager/client/grampus/grampus.go +++ b/manager/client/grampus/grampus.go @@ -30,7 +30,8 @@ const ( urlGetImages = urlOpenApiV1 + "image" urlNotebookJob = urlOpenApiV1 + "notebook" - errorIllegalToken = 1005 + errorIllegalToken = 1005 + errorCannotStopCreatingJob = 5008 ) type GetTokenParams struct { @@ -421,6 +422,9 @@ sendjob: if result.ErrorCode != 0 { log.Error("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) + if result.ErrorCode == errorCannotStopCreatingJob { + return &result, &models.ErrCannotStopCreatingGrampusJob{} + } return &result, fmt.Errorf("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) } diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 46669d1e1c..a087a2b9e8 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -236,8 +236,9 @@ type Cloudbrain struct { JobName string DisplayJobName string Status string - UserID int64 `xorm:"INDEX NOT NULL"` - RepoID int64 `xorm:"INDEX NOT NULL"` + DetailedStatus string `xorm:"DEFAULT '-'"` + UserID int64 `xorm:"INDEX NOT NULL"` + RepoID int64 `xorm:"INDEX NOT NULL"` SubTaskName string ContainerID string ContainerIp string @@ -1593,6 +1594,7 @@ type DatasetDownload struct { DatasetDownloadLink string `json:"dataset_download_link"` RepositoryLink string `json:"repository_link"` IsDelete bool `json:"is_delete"` + Size int64 `json:"size"` } type ModelDownload struct { @@ -1839,31 +1841,33 @@ type GrampusResult struct { } type GrampusJobInfo struct { - StartedAt int64 `json:"startedAt"` - RunSec int64 `json:"runSec"` - CompletedAt int64 `json:"completedAt"` - CreatedAt int64 `json:"createdAt"` - UpdatedAt int64 `json:"updatedAt"` - Desc string `json:"desc"` - JobID string `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - UserID string `json:"userId"` - Tasks []GrampusTasks `json:"tasks"` + StartedAt int64 `json:"startedAt"` + RunSec int64 `json:"runSec"` + CompletedAt int64 `json:"completedAt"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` + Desc string `json:"desc"` + JobID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + DetailedStatus string `json:"detailedStatus"` + UserID string `json:"userId"` + Tasks []GrampusTasks `json:"tasks"` } type GrampusNotebookInfo struct { - StartedAt int64 `json:"startedAt"` - RunSec int64 `json:"runSec"` - CompletedAt int64 `json:"completedAt"` - CreatedAt int64 `json:"createdAt"` - UpdatedAt int64 `json:"updatedAt"` - Desc string `json:"desc"` - JobID string `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - UserID string `json:"userId"` - Tasks []GrampusNotebookTask `json:"tasks"` + StartedAt int64 `json:"startedAt"` + RunSec int64 `json:"runSec"` + CompletedAt int64 `json:"completedAt"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` + Desc string `json:"desc"` + JobID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + DetailedStatus string `json:"detailedStatus"` + UserID string `json:"userId"` + Tasks []GrampusNotebookTask `json:"tasks"` } type Center struct { ID string `json:"id"` @@ -1924,13 +1928,17 @@ type GetGrampusAiCentersResult struct { Infos []GrampusAiCenter `json:"aiCenterInfos"` TotalSize int `json:"totalSize"` } - +type AICenterImage struct { + AICenterID string `json:"aiCenterId"` + ImageUrl string `json:"imageUrl"` +} type GrampusImage struct { - CreatedAt int64 `json:"createdAt"` - UpdatedAt int64 `json:"updatedAt"` - ID string `json:"id"` - Name string `json:"name"` - ProcessorType string `json:"processorType"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` + ID string `json:"id"` + Name string `json:"name"` + ProcessorType string `json:"processorType"` + AICenterImage []AICenterImage `json:"aiCenterImages"` } type GetGrampusImagesResult struct { diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index cf968725df..4ac3d4c738 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -15,6 +15,7 @@ type TaskDetail struct { JobName string `json:"JobName"` DisplayJobName string `json:"DisplayJobName"` Status string `json:"Status"` + DetailedStatus string `json:"DetailedStatus"` JobType string `json:"JobType"` CreatedUnix timeutil.TimeStamp `json:"CreatedUnix"` WaitTime string `json:"WaitTime"` @@ -103,6 +104,137 @@ type CloudbrainAllDuration struct { Count int `xorm:"count"` } +type XPUInfoBase struct { + ID int64 `xorm:"pk autoincr"` + CardType string + CardTypeShow string + ResourceType string + Company string + AccessTime string +} + +type XPUInfoStatistic struct { + ID int64 `xorm:"pk autoincr"` + InfoID int64 `xorm:"index"` + Type int + UsedDuration int64 + UsedCardHour int64 + UserCount int64 + TaskCount int64 + UpdatedUnix int64 +} + +type XPUInfoStatisticExtendBase struct { + CardType string + CardTypeShow string + ResourceType string + Company string + AccessTime string + UpdatedUnix int64 + UsedCardHour int64 + UserCount int64 + TaskCount int64 +} + +func (XPUInfoStatisticExtendBase) TableName() string { + return "xpu_info_statistic" +} + +type XPUInfoStatisticShow struct { + CardType string `json:"card_type"` + ResourceType string `json:"resource_type"` + Company string `json:"company"` + AccessTime string `json:"access_time"` + Count int64 `json:"count"` + UpdatedUnix int64 `json:"updated_unix"` +} + +const TypeAllDays = 0 +const TypeSevenDays = 1 +const TypeThirtyDays = 2 + +var XpuInfoCategories = map[string]string{"card": "used_card_hour", "user": "user_count", "task": "task_count"} +var XpuInfoType = map[string]int{"all": TypeAllDays, "7": TypeSevenDays, "30": TypeThirtyDays} + +func GetXPUInfos() ([]*XPUInfoBase, error) { + infos := make([]*XPUInfoBase, 0) + err := xStatistic.Find(&infos) + return infos, err +} + +func GetXPUStatisticInfos(dataType int, category string) ([]*XPUInfoStatisticShow, error) { + statistics := make([]XPUInfoStatisticExtendBase, 0) + + s := xStatistic.Where("type=?", dataType). + Join("INNER", "xpu_info_base", "xpu_info_base.id = xpu_info_statistic.info_id") + err := s.OrderBy("xpu_info_statistic." + XpuInfoCategories[category] + " desc").Find(&statistics) + + if err != nil { + return nil, err + } + var result = make([]*XPUInfoStatisticShow, 0) + for _, statistic := range statistics { + item := &XPUInfoStatisticShow{ + CardType: statistic.CardTypeShow, + ResourceType: statistic.ResourceType, + Company: statistic.Company, + AccessTime: statistic.AccessTime, + UpdatedUnix: statistic.UpdatedUnix, + } + + if category == "card" { + item.Count = statistic.UsedCardHour + } else if category == "user" { + item.Count = statistic.UserCount + } else { + item.Count = statistic.TaskCount + } + result = append(result, item) + } + return result, nil +} + +func UpdateOrInsertXPUInfoStatistic(bean *XPUInfoStatistic) error { + + xpuInfoIndb := new(XPUInfoStatistic) + has, err := xStatistic.Where("info_id = ? and type=?", bean.InfoID, bean.Type).Get(xpuInfoIndb) + + if err != nil { + return err + } + + if has { + _, err = xStatistic.ID(xpuInfoIndb.ID).Cols("used_duration", "user_count", "used_card_hour", "task_count", "updated_unix").Update(bean) + + } else { + _, err = xStatistic.Insert(bean) + } + return err + +} + +func HasTotalTypeXPUInfoStatisticRecord() bool { + xpuInfo := new(XPUInfoStatistic) + total, _ := xStatistic.Where("type =?", TypeAllDays).Count(xpuInfo) + return total > 0 +} + +func FindTotalTypeXPUInfoStatisticRecords() ([]XPUInfoStatistic, error) { + + xpuInfos := make([]XPUInfoStatistic, 0) + + err := xStatistic.Where("type =?", TypeAllDays).Find(&xpuInfos) + return xpuInfos, err +} +func GetTotalTypeLatestUpdateUnix() int64 { + var updated []int64 + xStatistic.Table("xpu_info_statistic").Cols("updated_unix").Find(&updated) + if len(updated) > 0 { + return updated[0] + } + return 0 +} + func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(distinct user_id) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + diff --git a/models/error.go b/models/error.go index 8fa2c37521..0f887517cb 100755 --- a/models/error.go +++ b/models/error.go @@ -2082,3 +2082,15 @@ func IsErrPretrainModelNotExist(err error) bool { func (err ErrPretrainModelNotExist) Error() string { return fmt.Sprintf("pretrain model is not exists") } + +type ErrCannotStopCreatingGrampusJob struct { +} + +func IsErrCannotStopCreatingGrampusJob(err error) bool { + _, ok := err.(ErrCannotStopCreatingGrampusJob) + return ok +} + +func (err ErrCannotStopCreatingGrampusJob) Error() string { + return fmt.Sprintf("job is creating, can not be stopped") +} diff --git a/models/models.go b/models/models.go index 29f75e12f8..1a6a63ff50 100755 --- a/models/models.go +++ b/models/models.go @@ -200,6 +200,8 @@ func init() { new(ModelApp), new(LlmChat), new(LlmChatVisit), + new(XPUInfoBase), + new(XPUInfoStatistic), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo_permission.go b/models/repo_permission.go index 2061f9770d..b0c6fc355c 100644 --- a/models/repo_permission.go +++ b/models/repo_permission.go @@ -188,7 +188,6 @@ func getUserRepoPermission(e Engine, repo *Repository, user *User) (perm Permiss } perm.Units = repo.Units - // anonymous visit public repo if user == nil { perm.AccessMode = AccessModeRead diff --git a/models/resource_specification.go b/models/resource_specification.go index b74adfb3d6..5e493ee81a 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -297,6 +297,23 @@ func (s *Specification) ToShowString() string { return specName } +func GetAvailableCenterIdsByASpec(ID int64) ([]string, error) { + spec, err := GetResourceSpecification(&ResourceSpecification{ + ID: ID}) + if err != nil { + return []string{}, err + } + var queueIds []int64 + err = x.Table("resource_specification").Cols("queue_id").Where("status=? and source_spec_id=?", SpecOnShelf, spec.SourceSpecId).Find(&queueIds) + if err != nil || len(queueIds) == 0 { + return []string{}, err + } + var centerIds []string + err = x.Table("resource_queue").Cols("ai_center_code").In("id", queueIds).Find(¢erIds) + return centerIds, err + +} + type GetAvailableCenterIdOpts struct { UserId int64 JobType JobType diff --git a/models/unit.go b/models/unit.go index 681400bf77..72a15293ad 100755 --- a/models/unit.go +++ b/models/unit.go @@ -60,6 +60,8 @@ func (u UnitType) String() string { return "UnitTypeBlockChain" case UnitTypeModelManage: return "UnitTypeModelManage" + case UnitTypeHPC: + return "UnitTypeHPC" } return fmt.Sprintf("Unknown UnitType %d", u) } @@ -297,6 +299,14 @@ var ( 8, } + UnitHPC = Unit{ + UnitTypeHPC, + "repo.hpc", + "/supercompute", + "repo.hpc.desc", + 9, + } + // Units contains all the units Units = map[UnitType]Unit{ UnitTypeCode: UnitCode, @@ -310,6 +320,7 @@ var ( UnitTypeCloudBrain: UnitCloudBrain, UnitTypeBlockChain: UnitBlockChain, UnitTypeModelManage: UnitModelManage, + UnitTypeHPC: UnitHPC, } ) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 78dc42aea5..4ed94241ac 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -120,7 +120,7 @@ func CanModifyJob(ctx *context.Context, job *models.Cloudbrain) bool { return isAdminOrJobCreater(ctx, job, nil) } func CanDownloadJob(ctx *context.Context, job *models.Cloudbrain) bool { - return isAdminOrJobCreater(ctx, job, nil) + return isAdminOrOwnerOrJobCreater(ctx, job, nil) } func isAdminOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 39d6c21a72..4737a41020 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -230,6 +230,17 @@ func registerHandleSummaryStatistic() { return nil }) } +func registerHandleCardStatistic() { + RegisterTaskFatal("handle_card_statistic", &BaseConfig{ + Enabled: setting.CardStatistic.Enabled, + RunAtStart: setting.CardStatistic.RunAtStart, + Schedule: setting.CardStatistic.Cron, + }, func(ctx context.Context, _ *models.User, _ Config) error { + cloudbrainService.CardStatistic() + return nil + }) + +} func registerHandleOrgStatistic() { RegisterTaskFatal("handle_org_statistic", &BaseConfig{ @@ -411,4 +422,5 @@ func initBasicTasks() { registerSyncFinetunePanguDeployStatus() registerFinetunePanguServiceCreateQueue() + registerHandleCardStatistic() } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 545997d299..b5bc7a1e7e 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -659,6 +659,11 @@ var ( Cron string RunAtStart bool }{} + CardStatistic = struct { + Enabled bool + Cron string + RunAtStart bool + }{} C2NetInfos *C2NetSqInfos CenterInfos *AiCenterInfos @@ -1785,6 +1790,7 @@ func NewContext() { getModelAppConfig() getClearStrategy() getNotebookStrategy() + getCardStatistic() NewScreenMapConfig() } @@ -1904,6 +1910,14 @@ func getNotebookStrategy() { NotebookStrategy.RunAtStart = sec.Key("RUN_AT_START").MustBool(false) } +func getCardStatistic() { + + sec := Cfg.Section("card_statistic") + CardStatistic.Enabled = sec.Key("ENABLED").MustBool(false) + CardStatistic.Cron = sec.Key("CRON").MustString("@daily") + CardStatistic.RunAtStart = sec.Key("RUN_AT_START").MustBool(false) +} + func GetGrampusConfig() { sec := Cfg.Section("grampus") diff --git a/modules/storage/minio_ext.go b/modules/storage/minio_ext.go index 216c2611a6..9cc56face3 100755 --- a/modules/storage/minio_ext.go +++ b/modules/storage/minio_ext.go @@ -175,53 +175,62 @@ func GetOneLevelAllObjectUnderDirMinio(bucket string, prefixRootPath string, rel Prefix += "/" } log.Info("bucket=" + bucket + " Prefix=" + Prefix) - output, err := core.ListObjects(bucket, Prefix, "", "", 1000) + marker := "" fileInfos := make([]FileInfo, 0) prefixLen := len(Prefix) fileMap := make(map[string]bool, 0) - if err == nil { - for _, val := range output.Contents { - - log.Info("val key=" + val.Key) - var isDir bool - var fileName string - if val.Key == Prefix { - continue - } - fileName = val.Key[prefixLen:] - log.Info("fileName =" + fileName) - files := strings.Split(fileName, "/") - if fileMap[files[0]] { - continue - } else { - fileMap[files[0]] = true + index := 1 + for { + output, err := core.ListObjects(bucket, Prefix, marker, "", 1000) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + log.Info("val key=" + val.Key) + var isDir bool + var fileName string + if val.Key == Prefix { + continue + } + fileName = val.Key[prefixLen:] + log.Info("fileName =" + fileName) + files := strings.Split(fileName, "/") + if fileMap[files[0]] { + continue + } else { + fileMap[files[0]] = true + } + ParenDir := relativePath + fileName = files[0] + if len(files) > 1 { + isDir = true + ParenDir += fileName + "/" + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: ParenDir, + } + fileInfos = append(fileInfos, fileInfo) } - ParenDir := relativePath - fileName = files[0] - if len(files) > 1 { - isDir = true - ParenDir += fileName + "/" + if output.IsTruncated { + marker = output.NextMarker } else { - isDir = false + break } - fileInfo := FileInfo{ - ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), - FileName: fileName, - Size: val.Size, - IsDir: isDir, - ParenDir: ParenDir, - } - fileInfos = append(fileInfos, fileInfo) - } - return fileInfos, err - } else { + } else { - log.Error("Message:%s", err.Error()) + log.Error("Message:%s", err.Error()) - return nil, err + return nil, err + } } - + return fileInfos, err } func MinioGetFilesSize(bucketName string, Files []string) int64 { @@ -422,3 +431,134 @@ func MinioCheckAndGetFileSize(srcBucket string, key string) (bool, int64) { } return true, meta.Size } + +func GetDirsListMinio(bucket string, prefixRootPath string, prefix string, dirLevel int) ([]string, error) { + _, core, err := getClients() + if err != nil { + log.Error("getClients failed:", err.Error()) + return nil, err + } + minioPrefix := prefixRootPath + prefix + if !strings.HasSuffix(minioPrefix, "/") { + minioPrefix += "/" + } + prefixLen := len(minioPrefix) + delimiter := "" + marker := "" + index := 1 + fileMap := make(map[string]bool, 0) + log.Info("buckent=" + bucket + " minioPrefix=" + minioPrefix) + for { + output, err := core.ListObjects(bucket, minioPrefix, marker, delimiter, 1000) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + fileName := val.Key[prefixLen:] + files := strings.Split(fileName, "/") + currentLevel := 0 + for { + if currentLevel >= dirLevel || currentLevel >= (len(files)-1) { + break + } + path := getPathFromPathArrays(files, currentLevel) + if !fileMap[path] { + fileMap[path] = true + } + currentLevel = currentLevel + 1 + } + } + if output.IsTruncated { + marker = output.NextMarker + } else { + break + } + } else { + log.Info("list error." + err.Error()) + return nil, err + } + } + result := make([]string, 0) + for k, _ := range fileMap { + result = append(result, k) + } + return result, nil +} + +func GetDirsSomeFileMinio(bucket string, prefixRootPath string, relativePath string, marker string, pageSize int) ([]FileInfo, string, error) { + _, core, err := getClients() + if err != nil { + log.Error("getClients failed:", err.Error()) + return nil, "", err + } + + Prefix := prefixRootPath + relativePath + if !strings.HasSuffix(Prefix, "/") { + Prefix += "/" + } + log.Info("bucket=" + bucket + " Prefix=" + Prefix) + fileInfos := make([]FileInfo, 0) + + fileMap := make(map[string]bool, 0) + index := 1 + for { + output, err := core.ListObjects(bucket, Prefix, marker, "", 1000) + if err == nil { + log.Info("Page:%d\n", index) + index++ + isBreak := false + for _, val := range output.Contents { + //log.Info("val key=" + val.Key) + var isDir bool + var fileName string + if val.Key == Prefix { + continue + } + fileNameRune := []rune(val.Key) + prefixRune := []rune(Prefix) + + fileName = string(fileNameRune[len(prefixRune):]) + files := strings.Split(fileName, "/") + if fileMap[files[0]] { + continue + } else { + fileMap[files[0]] = true + } + ParenDir := relativePath + fileName = files[0] + if len(files) > 1 { + isDir = true + ParenDir += fileName + "/" + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: ParenDir, + } + fileInfos = append(fileInfos, fileInfo) + if len(fileInfos) >= pageSize { + marker = val.Key + isBreak = true + break + } + } + if isBreak { + break + } + if output.IsTruncated { + marker = output.NextMarker + } else { + marker = "" + break + } + } else { + log.Error("Message:%s", err.Error()) + return nil, "", err + } + } + return fileInfos, marker, err +} diff --git a/modules/storage/obs.go b/modules/storage/obs.go index de5c7892ae..5fcaad6ff1 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -742,7 +742,7 @@ func PutStringToObs(bucket, key string, fileContent string) error { } func PutReaderToObs(bucket, key string, reader io.Reader) error { - log.Info("PutStringToObs bucket=" + bucket + " key=" + key) + log.Info("PutReaderToObs bucket=" + bucket + " key=" + key) input := &obs.PutObjectInput{} input.Bucket = bucket input.Key = key @@ -755,3 +755,150 @@ func PutReaderToObs(bucket, key string, reader io.Reader) error { } return err } + +func GetDirsList(bucket string, prefixRootPath string, prefix string, dirLevel int) ([]string, error) { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + input.Prefix = prefixRootPath + prefix + if !strings.HasSuffix(input.Prefix, "/") { + input.Prefix += "/" + } + prefixLen := len(input.Prefix) + fileMap := make(map[string]bool, 0) + index := 1 + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n input.Prefix=v%", index, input.Prefix) + log.Info("input.Prefix=" + input.Prefix) + index++ + for _, val := range output.Contents { + fileName := val.Key[prefixLen:] + files := strings.Split(fileName, "/") + currentLevel := 0 + for { + if currentLevel >= dirLevel || currentLevel >= (len(files)-1) { + break + } + path := getPathFromPathArrays(files, currentLevel) + if !fileMap[path] { + fileMap[path] = true + } + currentLevel = currentLevel + 1 + } + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } + } + result := make([]string, 0) + for k, _ := range fileMap { + result = append(result, k) + } + return result, nil +} + +func getPathFromPathArrays(files []string, currentLevel int) string { + from := 0 + re := "" + for { + re += files[from] + if from >= currentLevel { + break + } + from = from + 1 + re += "/" + } + return re +} + +func GetDirsSomeFile(bucket string, prefixRootPath string, prefix string, marker string, pageSize int) ([]FileInfo, string, error) { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + input.Prefix = prefixRootPath + prefix + if !strings.HasSuffix(input.Prefix, "/") { + input.Prefix += "/" + } + input.Marker = marker + fileInfos := make([]FileInfo, 0) + //prefixLen := len(input.Prefix) + fileMap := make(map[string]bool, 0) + index := 1 + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d input.Prefix=v%", index, input.Prefix) + log.Info("input.Prefix=" + input.Prefix) + index++ + isBreak := false + for _, val := range output.Contents { + var isDir bool + var fileName string + //log.Info("val.Key=" + val.Key) + if val.Key == input.Prefix { + continue + } + fileNameRune := []rune(val.Key) + prefixRune := []rune(input.Prefix) + + fileName = string(fileNameRune[len(prefixRune):]) + //log.Info("fileName=" + fileName) + //val.Key[prefixLen:] + files := strings.Split(fileName, "/") + if fileMap[files[0]] { + continue + } else { + fileMap[files[0]] = true + } + ParenDir := prefix + fileName = files[0] + if len(files) > 1 { + isDir = true + ParenDir += fileName + "/" + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: ParenDir, + } + fileInfos = append(fileInfos, fileInfo) + if len(fileInfos) >= pageSize { + marker = val.Key + log.Info("marker 1= " + marker) + isBreak = true + break + } + } + if isBreak { + break + } + if output.IsTruncated { + input.Marker = output.NextMarker + marker = output.NextMarker + log.Info("marker 2= " + marker) + } else { + marker = "" + log.Info("marker 3= " + marker) + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, "", err + } + } + return fileInfos, marker, nil +} diff --git a/modules/urfs_client/objectstorage/mocks/objectstorage_mock.go b/modules/urfs_client/objectstorage/mocks/objectstorage_mock.go index 0b09403dd9..baa34f437e 100644 --- a/modules/urfs_client/objectstorage/mocks/objectstorage_mock.go +++ b/modules/urfs_client/objectstorage/mocks/objectstorage_mock.go @@ -1,9 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. // Source: objectstorage.go +// Package mocks is a generated GoMock package. package mocks - -import ( - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cd407539b1..e3784634ac 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -335,6 +335,7 @@ robot = Robot federated_learning = Federated learning data_mining = Data mining RISC-V_development = RISC-V development +domestic_computing_power = Domestic computing power [auth] create_new_account = Register Account @@ -864,6 +865,12 @@ download_copy=Copy URL create_dataset_fail=Failed to create dataset. query_dataset_fail=Failed to query dataset. edit_attachment_fail=Failed to update description. +file_list=File list +file_not_support_preview=File does not support preview +please_select_file_preview=Please select a file for preview +see_more=See more +query_timeout_tips=The server is processing it. Drink a glass of water and come back to take a look~ + show_dataset= Dataset edit_dataset= Edit Dataset update_dataset= Update Dataset @@ -1002,6 +1009,7 @@ go_new_dataset= to create the dataset export_tips = Only zip/tar.gz type result file export is supported, and the exported file can finally be viewed on the Datasets tab of the current project. [repo] +attachmentfilesizetobig=The file size exceeds 2000 lines or 2MB, and preview is not supported. owner = Owner repo_name = Repository Name repo_name_helper = Good repository names use short, memorable and unique keywords. @@ -2507,6 +2515,8 @@ imagetopic.format_prompt = Topics can be up to 35 characters long. use_repo_agreement=I promise that the content of this warehouse does not violate any national laws and regulations. During the use of the warehouse, I will abide by the OpenI community management regulations and platform usage rules, and will not conduct malicious attacks, mining, or any other illegal or disruptive platform order. Information release and related behaviors. For more information please refer to openi_use_agreement=OpenI Openi Community Platform Use Agreement. reuse_last_result = Reuse last result +migratingData=Data migration in progress +centerPending=Queuing in sub centers [org] org_name_holder = Organization Name org_full_name_holder = Organization Full Name @@ -3469,6 +3479,8 @@ dataset_number_over_limit = The dataset count exceed the limit result_cleared=The files of the task have been cleared, can not restart or retrain any more, please create a new task instead model_not_exist=The model in the task does not exist or has been deleted too_many_notebook=A user can have up to 5 debug tasks, please try again after delete some debug tasks. +can_not_stop_creating_job=AI task is creating, can not be stopped. +no_center_match=Can not match a AI center, please select other specification. [common_error] system_error = System error.Please try again later diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 440727613c..acb847ed5b 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -338,6 +338,7 @@ robot = 机器人 federated_learning = 联邦学习 data_mining = 数据挖掘 RISC-V_development = RISC-V开发 +domestic_computing_power = 国产算力 [auth] create_new_account=注册帐号 @@ -864,6 +865,11 @@ create_dataset=创建数据集 create_dataset_fail=创建数据集失败。 query_dataset_fail=查询数据集失败。 edit_attachment_fail=修改描述失败。 +file_list=文件列表 +file_not_support_preview=文件暂不支持预览 +please_select_file_preview=请选择文件进行预览 +see_more=查看更多 +query_timeout_tips=服务器正在处理，喝杯水回来再看看~ reference_dataset_fail=关联数据集失败，请稍后再试。 cancel_reference_dataset_fail=取消关联数据集失败，请稍后再试。 @@ -1007,6 +1013,7 @@ go_new_dataset=去创建数据集 export_tips = 仅支持 zip/tar.gz 类型的结果文件导出，导出的文件最终可以在当前项目的数据集页签下查看。 [repo] +attachmentfilesizetobig=文件大小超过2000行或者超过2MB，不支持预览。 owner=拥有者 repo_name=项目名称 repo_name_helper=好的存储库名称使用简短、深刻和独特的关键字。 @@ -2525,6 +2532,9 @@ imagetopic.format_prompt=标签长度不得超过35个字符 use_repo_agreement=我承诺此仓内容不违反任何国家法律法规，仓库使用过程中遵守OpenI启智社区管理规定和平台使用规则，不进行恶意攻击、挖矿等任何违法或扰乱平台秩序的信息发布和相关行为。更多信息请参考 openi_use_agreement=OpenI启智社区平台使用协议 reuse_last_result = 复用上次结果 +migratingData=数据迁移中 +centerPending=分中心排队中 + [org] org_name_holder=组织名称 org_full_name_holder=组织全名 @@ -3492,6 +3502,8 @@ dataset_number_over_limit = 选择的数据集文件数量超出限制 result_cleared=源任务的文件已被清理，无法再次调试或复用训练结果，请新建任务。 model_not_exist=选择的预训练模型不存在或者已被删除 too_many_notebook=每个用户最多只能创建5个调试任务，请删除历史任务再新建。 +can_not_stop_creating_job=任务正在创建中，请等创建完成后再尝试停止。 +no_center_match=没有可分配的中心，请选择其他规格。 [common_error] diff --git a/public/img/domestic/ascend.svg b/public/img/domestic/ascend.svg new file mode 100644 index 0000000000..ec7cab0f0e --- /dev/null +++ b/public/img/domestic/ascend.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/domestic/cambricon.svg b/public/img/domestic/cambricon.svg new file mode 100644 index 0000000000..caf27cd653 --- /dev/null +++ b/public/img/domestic/cambricon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/domestic/enflame.svg b/public/img/domestic/enflame.svg new file mode 100644 index 0000000000..37b8d761fa --- /dev/null +++ b/public/img/domestic/enflame.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/domestic/hygon.png b/public/img/domestic/hygon.png new file mode 100644 index 0000000000000000000000000000000000000000..c2ae3c39d5a21fc8acc4ecff35d1284e5b64d873 GIT binary patch literal 2837 zcmX|C2{_d27aoeDs5H4E!Wc7y8Os=h7<;x!3}w4glI@~&kt{8iWNw#~E6b%Mp%~1@ zSi4zTlx!t~EQxXp(X~eZGkTu?^PKr+=A8Gu=Y7xjn`crS9jqn9WyKK)goLe)B@N!k z;6>QF2~Od0k{SZB3E@a}v9dlxtx`n)KPvaDFwrgN?&ip|CY1Lk>X)zCsnI*BVmT7>yb|!&)O#7zzP;Y7dwI7%(;D19e7b)jDJ* z%z^-NG!F7W4lvLr(2v7cYmq=?t+5$XRUI-M0+FjjVyR$RSUd<|t81~bf)<$0R@bZ{ z8T{14(a>U|Ft7^f0`CBW475P}RKix%k`jH?jnb%BpZdg0YaBQPS^5+2qL`+*EROH5Tkslmbq^guhbz)(QKlOWg(kPwK#AW*e& zHBbToSYs1(01V>mnOrNA8Pc8O{nQLvLA0x9yv}79(mc< zogB$74yC*0y7X1t_2%3tQm9LO{mAS6n?1Z=D<*c*`&~+0oBd+Sr@M-YT zixLx$%TctTydAAx1tz?XjAv0ai|cha4N(2O&n;um{krYw9SRv4`3|8D2f7RKm9~3g z3-t85&+m+|jjnx?(0<#Y&8RIw-mN2}r!JJ~6Q*30n$^zj4fjp*&9rAWUnueTyDPaW z>|kj5>%|_%W&}dC)z;F?CE(o`_j6G$Ry&*QC8woqY|hF|ZxuRZvsFoGF5U5-Yl
9j?HdM(;O^KM5Krxqxp7E%uK^#M~*AE55{ zw;0=))uN*MV~5ysVRLT9Ug-cfg1>@sttW@NRLd4@U(8Qb=8;pU-SrBIV>qdPqE>O( z)y<0!G9`KEF}nx_ZwxB3uY}i}rjh6pZC3Za|7(9Ecy6%V4Cp~a{2gc#Oynl{Rcq*44ei-jg{50Hkp>=auu&-=lc_H@|FO+O}WZr8? zgE2*jO+zty8Ij0o#5Sgwy+e3XmuMRL3d*d1C=Jk1zl zN}rOiU+A3Yah2>lzqgIV39YCZZQ6R%S30WA-6^tbs|@x{5m&6n-_ffjy4^9rJ;vkm z4@;{0zuSeSZd1EP$2NWa!q4=-Q`z?N?A%=NSM*A%@a(dUg_GEaZ#%up+E0kmS*7LKp@A)~ z@9^gPv>v79V+qd16eQo{I^U3B$<)a1Yhjkcw*+N8@ zd0TV#RBT;>@RN{zGrD$7_VO>vo9iNSS$=aU=i+y9tHpWK**XQwr8gsx=SgUW<_%oV z?!%o`lQFghW_y%eAvYecdB*4w z_E?k;a{1_Pw;R1=f_Rdag%woq2kS7BzCvD5L^FXr-h!Znr=}DK3sm{ziq~hQ&|$y>p`NBX)jM#I{{64QKhnQgc-GQ8a^CMXTfl9}rwsRNEvb+W6K+f?N7AfA7ISx_ zbiJrl``B%y4$-(Z&XJCuSahIm$!R2g;a=!(GWJ<~?j0uy&zB7vPwSp0+@hU)Q2$?P zhMDXY^lTodIsT%$b1e74n$MPqBz4U9E7r>=dR9g0fm5R-rO7)5OHREqr26O_f;^SL z2>N@GHXs);STfpmR$+p&=0Qy$9#~S9R@>D-P$%zYEW-C3kFpF}qh`;DW!(Mr*4k*% z>mgs(oL+x+i?>i?{%E0t$eM^-L+j&R-(xRa+q59g9kdFGd4b;(6z3R~y>_bStnZ`x zGuKn;#k&T(JS*)jxmjUz@90B}H$Z>kNf9($T1l~=KD8ms6)xhQ_}l&^?% zPHm3L%d~}^`kU565+*JMOFXG;5~uHPU)phPM7*AM;m<3P#OUAuT|rKrP;B9l&h@lt z2?da4btU*Fh-nqhkDjF0EWdfDZ;B4-e~zBA7~PhQn~>4l>Ru$9`RRA3*bBn(64FJ1 z>(WZ&{)a}6W`^ahZ5e1xSX<9tdOVPqp5uAc&7=4JAuE+L#EC+q>z&bV1Kk0Np?8*- zlz%;nva&Po8uX+tRp^K(_0jOOOnLzqP_{Lx|#LebxzZrm%48l Tl|Bpq?Fd^d2TO+e@%aA%bnOWX literal 0 HcmV?d00001 diff --git a/public/img/domestic/iluvatar.svg b/public/img/domestic/iluvatar.svg new file mode 100644 index 0000000000..922523b102 --- /dev/null +++ b/public/img/domestic/iluvatar.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/domestic/metax.svg b/public/img/domestic/metax.svg new file mode 100644 index 0000000000..5587fbece6 --- /dev/null +++ b/public/img/domestic/metax.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/self/dataset_preview.js b/public/self/dataset_preview.js index 81620e1a0d..506523d816 100644 --- a/public/self/dataset_preview.js +++ b/public/self/dataset_preview.js @@ -1,754 +1,340 @@ -var img=new Image(); - -var ip = getIp(); - - - -var token = getCookie("_csrf"); - -canvas = document.getElementById("myCanvas"); -context = canvas.getContext("2d"); -// canvas.width = document.getElementById("myCanvas").offsetWidth; -// canvas.height = document.getElementById("myCanvas").offsetWidth/1280*720; - -canvas.width = document.getElementById("win_canvas").offsetWidth; -canvas.height = document.getElementById("win_canvas").offsetHeight; - - - - -var color_dict = {"car":"#0099CC", "person":"#FF99CC","point":"#00cc00","pointselected":"red"}; -var color_person = {"0":"#13c90c","1":"#fc0707","2":"#FF99CC","3":"#fceb07"}; - -var rects=[]; -var masks=[]; -var pointShapes =[]; - -var fileindex =0; -var lastindex=false; -var labeltastresult; +(function () { + function DatasetPreview() { + this.ajaxTimeOut = 1000 * 60; + this.datasetData = DATASET_DATA; + this.supportImgReg = /(\.jpg|\.jpeg|\.png|\.gif|\.bmp)$/i; + this.supportTxtReg = /(\.txt|\.xml|\.html|\.json|\.py|\.sh|\.md|\.csv|\.log|\.js|\.css|\.ipynb)$/i; + this.data = { + filePath: [{ path: '', name: this.datasetData.path[0], ParenDir: '', marker: '' }], + dirMarkerChildrenMap: {}, + fileContentMap: {}, + currentPage: '', + }; + this.currentFileList = []; + this.currentFile = ''; + this.pageSize = 100; + this.getPathChildren(); + this.eventInit(); + } -var pageSize = 12; -var tableData; -var tablePageData; -var dataset_id = $('#hide_uuidid').val(); -var dbdatasetid = dataset_id; -var textContent; -var labelInfo; + DatasetPreview.prototype.renderFilePath = function () { + var container = $('#file_path_container').empty(); + var filePath = this.data.filePath; + for (let i = 0, iLen = filePath.length; i < iLen; i++) { + var path = filePath[i]; + var pathEle; + if (i == iLen - 1) { + pathEle = $(`
+ ${path.name}
/
+
`); + } else { + pathEle = $(`
+ ${path.name}
/
+
`); + } + pathEle.data('data', path); + container.append(pathEle); + } + }; -page(0,pageSize); + DatasetPreview.prototype.prevDirs = function () { + this.data.filePath.pop(); + this.getPathChildren(); + }; + DatasetPreview.prototype.nextDirs = function (pathObj) { + this.data.filePath.push({ + ...pathObj, + path: pathObj.FileName, + name: pathObj.FileName, + }); + this.getPathChildren(); + }; -function getCookie(name) -{ - var arr,reg=new RegExp("(^| )"+name+"=([^;]*)(;|$)"); - - if(arr=document.cookie.match(reg)) - - return unescape(arr[2]); - else - return null; -} + DatasetPreview.prototype.showGetPathChildrenTimeout = function () { + $('#myCanvas_div .tabpannel.ui.form .query-timeout').show(); + $('#filelist').hide(); + } -function list(current,pageSize){ - $.ajax({ - type:"GET", - url:ip + "/gitea-dateset-item-page", - headers: { - authorization:token, - }, - dataType:"json", - data:{ - 'datasetId':dbdatasetid, - 'startPage':current, - 'pageSize':pageSize}, - async:false, - success:function(json){ - tablePageData = json; - tableData = json.data; - labeltastresult = tableData; - fileindex=0; - if(lastindex){ - fileindex = pageSize - 1; - } - - }, - error:function(response) { - - + DatasetPreview.prototype.getPathChildrenMore = function () { + var self = this; + var lastDir = this.data.filePath[this.data.filePath.length - 1]; + var path = this.data.filePath.map((item) => item.path).join('/'); + if (!lastDir) return; + $('#myCanvas_div .tabpannel.ui.form').addClass('loading'); + $.ajax({ + type: "get", + url: `/api/v1/attachments/get_dir`, + dataType: "json", + timeout: this.ajaxTimeOut, + data: { + _csrf: this.datasetData.csrf, + uuid: this.datasetData.uuid, + marker: lastDir.marker, + pageSize: this.pageSize, + prefix: lastDir.ParenDir ? '/' + lastDir.ParenDir.replace(/^\//, '') : '', + }, + success: function (res) { + $('#myCanvas_div .tabpannel.ui.form').removeClass('loading'); + if (res.result_code == 0) { + var result = res.data.sort((a, b) => { + var a1 = a.IsDir ? 1 : 0; + var b1 = b.IsDir ? 1 : 0; + return b1 - a1; + }); + var fileList = []; + for (let i = 0, iLen = result.length; i < iLen; i++) { + const file = result[i]; + if (self.currentFileList.findIndex(itm => `${itm.ParenDir}${itm.FileName}` == `${file.ParenDir}${file.FileName}`) >= 0) { + continue; + } + fileList.push(file); + } + self.data.dirMarkerChildrenMap[path + '-' + lastDir.marker] = result; + self.currentFileList = self.currentFileList.concat(fileList); + lastDir.marker = res.marker; + self.renderFileListAdd(fileList, result.length); + } else { + console.log(res); } - }); -} - - + }, + error: function (err) { + $('#myCanvas_div .tabpannel.ui.form').removeClass('loading'); + console.log(err); + if (err.statusText == 'timeout') { + self.showGetPathChildrenTimeout(); + } + } + }); + } -function getTextContent(uuid,filename){ - $.ajax({ - type:"GET", - url:ip + "/getgiteatext", - headers: { - authorization:token, - }, - dataType:"text", - data:{ - 'uuid':uuid, - 'filename':filename - }, - async:false, - success:function(json){ - textContent = json; - + DatasetPreview.prototype.getPathChildren = function () { + this.renderFilePath(); + var lastDir = this.data.filePath[this.data.filePath.length - 1]; + if (!lastDir) return; + var path = this.data.filePath.map((item) => item.path).join('/'); + var cache = this.data.dirMarkerChildrenMap[path + '-' + lastDir.marker]; + var self = this; + if (false && cache) { + self.currentFileList = cache; + this.renderFileList(cache); + } else { + $('#myCanvas_div .tabpannel.ui.form').addClass('loading'); + $.ajax({ + type: "get", + url: `/api/v1/attachments/get_dir`, + dataType: "json", + timeout: this.ajaxTimeOut, + data: { + _csrf: this.datasetData.csrf, + uuid: this.datasetData.uuid, + marker: '', + pageSize: this.pageSize, + prefix: lastDir.ParenDir ? '/' + lastDir.ParenDir.replace(/^\//, '') : '', + }, + success: function (res) { + $('#myCanvas_div .tabpannel.ui.form').removeClass('loading'); + if (res.result_code == 0) { + var result = res.data.sort((a, b) => { + var a1 = a.IsDir ? 1 : 0; + var b1 = b.IsDir ? 1 : 0; + return b1 - a1; + }); + self.data.dirMarkerChildrenMap[path + '-' + lastDir.marker] = result; + lastDir.marker = res.marker; + self.currentFileList = result; + self.currentFile = ''; + self.renderFileList(result); + } else { + console.log(res); + } }, - error:function(response) { - + error: function (err) { + $('#myCanvas_div .tabpannel.ui.form').removeClass('loading'); + console.log(err); + if (err.statusText == 'timeout') { + self.showGetPathChildrenTimeout(); + } } - }); -} - - - -/* -function previewDataSetFile(uuid,filename){ - console.log('uuid=' + uuid + " filename=" + filename); - loadimg(uuid,filename); -} - -function loadimg(uuid,filename){ - img.src = ip + "/getgiteaimage?uuid=" + uuid + "&filename=" + filename; - var fname = filename.substring(filename.lastIndexOf('/') + 1); - $("#filename").text(fname); -} -*/ - -function loadimg(){ - var length = labeltastresult[fileindex].pic_image_field.length; - -  if(labeltastresult[fileindex].pic_image_field.substring(length - 5).toLowerCase() == ".json"  -     || labeltastresult[fileindex].pic_image_field.substring(length - 4).toLowerCase() == ".xml" -     || labeltastresult[fileindex].pic_image_field.substring(length - 4).toLowerCase() == ".txt" -     || labeltastresult[fileindex].pic_image_field.substring(length - 4).toLowerCase() == ".csv" -     || labeltastresult[fileindex].pic_image_field.substring(length - 3).toLowerCase() == ".md" -     || labeltastresult[fileindex].pic_image_field.substring(length - 3).toLowerCase() == ".py" -     || labeltastresult[fileindex].pic_image_field.substring(length - 3).toLowerCase() == ".sh"){ - - //文本 - canvas.style.display="none"; - document.getElementById("textcontent").style.display="block"; - getTextContent(dataset_id,labeltastresult[fileindex].pic_image_field); - $('#textcontent').height(canvas.height-40) - $("#textcontent").text(textContent); - }else{ - if(labeltastresult[fileindex].pic_image_field.substring(length - 5).toLowerCase() == ".jpeg"  -    || labeltastresult[fileindex].pic_image_field.substring(length - 4).toLowerCase() == ".jpg" -    || labeltastresult[fileindex].pic_image_field.substring(length - 4).toLowerCase() == ".bmp" -    || labeltastresult[fileindex].pic_image_field.substring(length - 4).toLowerCase() == ".gif" -    || labeltastresult[fileindex].pic_image_field.substring(length - 4).toLowerCase() == ".png"){ - canvas.style.display="block"; - document.getElementById("textcontent").style.display="none"; - img.src = ip + "/getgiteaimage?uuid=" + dataset_id + "&filename=" + labeltastresult[fileindex].pic_image_field; - }else{ - canvas.style.display="none"; - document.getElementById("textcontent").style.display="block"; - $('#textcontent').height(canvas.height) - $("#textcontent").text("暂不支持预览"); - - } - } - - - var fname = tableData[fileindex].pic_image_field.substring(tableData[fileindex].pic_image_field.lastIndexOf('/') + 1); - $("#filename").text(fname); -} - -img.onload = function(){ - - - canvas.width = document.getElementById("win_canvas").offsetWidth; - canvas.height = document.getElementById("win_canvas").offsetHeight-40; - //调整画布大小 - // if ((img.width/img.height)<(canvas.width/canvas.height)){ - // canvas.width=canvas.height * img.width / img.height; - // } - // else{ - // canvas.height=canvas.width * img.height / img.width; - // } - drawimage(); -} - -function isEmpty(str){ - if(typeof str == "undefined" || str == null || str == ""){ - return true; - } - return false; -} - -function drawimage() { - - parse_labelinfo(labeltastresult[fileindex].label_info); - - // 清除画布，准备绘制 - context.clearRect(0, 0, canvas.width, canvas.heigth); - // modal_context.cleararc - - context.drawImage(img,0,0,img.width,img.height,0,0,canvas.width, canvas.height); - - for(var i=0; i + + ${file.FileName} + `); + fileEl.data('data', file); + domC.append(fileEl); } - this.mouseonpoint = false; - this.mouseonrect = false; - this.isSelected = false; - this.id =""; //标识 - this.blurred=false;//模糊不清的; 记不清的; 难以区分的; 模棱两可的 - this.goodIllumination = true; //照明 - this.frontview = true;//正面图 -}; - - - -function maskar(x0,y0,type){ - this.type = type; - this.points = [new point(x0,y0)]; - this.finish = false; - this.mouseonpoint = false; - this.mouseonmask = false; - this.isSelected = false; - this.getX1Y1 = function(){return [this.points[0].x,this.points[0].y]} - this.getBound = function(){ - mlen = this.points.length; - var minX = 999999999, minY = 999999999, maxX = -1, maxY = -1; - for (var i = 0; i < mlen; i ++){ - if(minX > this.points[i].x){ - minX = this.points[i].x; - } - if(maxX < this.points[i].x){ - maxX = this.points[i].x; - } - if(minY > this.points[i].y){ - minY = this.points[i].y; - } - if(maxY < this.points[i].y){ - maxY = this.points[i].y; - } + var lastDir = this.data.filePath[this.data.filePath.length - 1]; + if (!lastDir) return; + if (length >= this.pageSize && lastDir.marker) { + domC.append(`
${$('#lang-seemore').text()}
`); } - return [minX, minY, maxX, maxY]; - } - - this.id =""; //标识 - this.blurred=false;//模糊不清的; 记不清的; 难以区分的; 模棱两可的 - this.goodIllumination = true; //照明 - this.frontview = true;//正面图 -} - -function getCanvasLocationX(num){ - return Math.round(num * canvas.width/parseInt(img.width)); -} - -function getCanvasLocationY(num){ - return Math.round(num * canvas.height/parseInt(img.height)); -} - - -function page(current,pageSize){ - list(current,pageSize); - showfilelist(); - breadFiles(); - loadimg(); - setPage(tablePageData,pageSize); -} - - -getLabelInfo(dataset_id); -showlabelflist(); - - -function nextPage(){ - var current = $('#displayPage1').text(); - - page(current,pageSize); -} - -function prePage(){ - var current =$('#displayPage1').text(); - - if(current > 1){ - - page(current - 2,pageSize); - } -} - -function goPage(){ - var goNum = $('#goNum').val(); + }; - var pageTotal = $("#totalNum").text(); - var pageNum = parseInt(pageTotal/pageSize); - if(pageTotal%pageSize!=0){ - pageNum += 1; - }else { - pageNum = pageNum; + DatasetPreview.prototype.renderFileList = function (files) { + var domC = $('#filelist').empty(); + var findFile = false; + for (let i = 0, iLen = files.length; i < iLen; i++) { + const file = files[i]; + const fileEl = $(`
+ + ${file.FileName} +
`); + fileEl.data('data', file); + domC.append(fileEl); } - if (goNum<=0){ - alert("请输入大于0的数值"); + if (!findFile) { + this.currentFile = null; + this.renderPleseSelectFile(); } - else if(goNum<=pageNum){ - page(goNum - 1,pageSize); + var lastDir = this.data.filePath[this.data.filePath.length - 1]; + if (!lastDir) return; + if (files.length >= this.pageSize && lastDir.marker) { + domC.append(`
${$('#lang-seemore').text()}
`); } - else{ - alert("不能超出总页码！"); - } -} + }; -$("#goNum").keydown(function (e) { - if (e.keyCode == 13) { - goPage(); + DatasetPreview.prototype.renderPreview = function (file) { + var fileName = file.FileName; + if (this.supportImgReg.test(fileName)) { + console.log('renderImage'); + $('#textcontent').hide(); + $('#win_canvas .select-file').hide(); + $('#win_canvas .not-support').hide(); + $('#imgcontent').attr('src', `/api/v1/attachments/get_image_content?uuid=${this.datasetData.uuid}&filePath=${file.ParenDir + file.FileName}&type=${this.datasetData.type}&_csrf=${this.datasetData.csrf}`).show(); + } else if (this.supportTxtReg.test(fileName)) { + console.log('getFileText'); + this.getFileText(file); + } else { + console.log('not support'); + this.renderNotSupport(); } -}); - - -function setPage(pageData,pageSize){ - - if (isEmpty(pageData)){ - return; - } - var startIndex = pageData.current * pageSize; - if(pageData.total > 0){ - startIndex = startIndex + 1; - } - if(startIndex < 10){ - $('#startIndex').text(" " + (startIndex)); - }else{ - $('#startIndex').text(startIndex); - } - var endIndex = pageData.current * pageSize + pageData.data.length; - if(endIndex < 10){ - $('#endIndex').text(" " + (endIndex)); - }else{ - $('#endIndex').text(endIndex); - } - - $('#totalNum').text(pageData.total); - $('#displayPage1').text(pageData.current + 1); - - - - if(pageData.current == 0){ - - $('#prePage').removeAttr("href"); - $('#prePage').attr('style','color:#f5f5f6;'); - } - else{ - $('#prePage').attr("href","javascript:prePage()"); - $('#prePage').attr('style','color:#000;'); - } - - if((pageData.current + 1) * pageSize >= pageData.total){ - - $('#nextPage').removeAttr("href"); - $('#nextPage').attr('style','color:#f5f5f6;') - } - else{ - $('#nextPage').attr("href","javascript:nextPage()"); - $('#nextPage').attr('style','color:#000;'); - } + }; - var pageTotal = pageData.total; - var pageNum = parseInt(pageTotal/pageSize); - if(pageTotal%pageSize!=0){ - pageNum += 1; - }else { - pageNum = pageNum; + DatasetPreview.prototype.renderNotSupport = function () { + $('#imgcontent').hide(); + $('#textcontent').hide(); + $('#win_canvas .select-file').hide(); + $('#win_canvas .not-support').show(); } - $("#totalPageNum").text(pageNum); -} -function clickfilelist(index){ - fileindex=index; - loadimg(); - //drawimage(); - breadFiles() - showfilelist(); -} - -function clickNext(){ - if(fileindex= tablePageData.total){ - return; - } - nextPage(); + DatasetPreview.prototype.renderPleseSelectFile = function () { + $('#imgcontent').hide(); + $('#textcontent').hide(); + $('#win_canvas .not-support').hide(); + $('#win_canvas .select-file').show(); } - -} - - -function next(){ - if(fileindex0) {fileindex=fileindex-1;} - loadimg(); - //drawimage(); - breadFiles(); - showfilelist(); -} - - -// function showfilelist(){ -// var htmlstr=""; -// for (var i=0;i"+ fname+ ""; -// }; -// document.getElementById("filelist").innerHTML=htmlstr; -// } + DatasetPreview.prototype.renderText = function (content) { + $('#imgcontent').hide(); + $('#win_canvas .select-file').hide(); + $('#win_canvas .not-support').hide(); + $('#textcontent').empty().text(content).show(); + }; -function showfilelist(){ - // var filename_title = $('a.section:first').text() - // filename_title = filename_title.substring(0,filename_title.lastIndexOf('.')) - var filename_index = labeltastresult[0].pic_image_field.indexOf("/",70); - filename_title = labeltastresult[0].pic_image_field.substring(filename_index + 1); - filename_title = filename_title.substring(0,filename_title.indexOf('/')) - var htmlstr = ''; - // htmlstr += '
'; - htmlstr += '
' - htmlstr += '' - htmlstr += '
' - htmlstr += '
'+filename_title+'
' - htmlstr += '
' - - for (var i=0;i 70){ - var tmpIndex = labeltastresult[i].pic_image_field.indexOf("/",70); - //console.log(tmpIndex) - if(tmpIndex != -1){ - fname = labeltastresult[i].pic_image_field.substring(tmpIndex + 1); - fname = fname.substring(fname.indexOf('/')+1); + DatasetPreview.prototype.getFileText = function (file) { + var cache = this.data.fileContentMap[file.ParenDir + file.FileName]; + var self = this; + if (false && cache) { + self.renderText(cache); + } else { + $.ajax({ + type: "get", + url: "/api/v1/attachments/get_txt_content", + headers: { authorization: this.datasetData.csrf, }, + dataType: "json", + data: { + _csrf: this.datasetData.csrf, + uuid: this.datasetData.uuid, + type: this.datasetData.type, + filePath: file.ParenDir + file.FileName, + }, + success: function (res) { + self.data.fileContentMap[file.ParenDir + file.FileName] = res.data; + const data = res.data || []; + if (res.result_code == 0) { + self.renderText(data.join('')); + } else if (res.result_code == -1) { + self.renderText(res.msg); + } else { + self.renderNotSupport(); + } + }, + error: function (err) { + console.log(err); + self.renderNotSupport(); } + }); } - - htmlstr += '
' - htmlstr += '
' - if(i==fileindex){ - htmlstr += '
'+fname+'
' - } - else{ - htmlstr += '
'+fname+'
' - } - htmlstr += '
' - htmlstr += '
' }; - htmlstr += '
' - htmlstr += '
' - htmlstr += '
' - document.getElementById("filelist").innerHTML=htmlstr; -} - - -function breadFiles(){ - - // for (var i=0;i 70){ - // var tmp_index = labeltastresult[i].pic_image_field.indexOf("/",70); - // if(tmp_index != -1){ - // fname_full_path = labeltastresult[i].pic_image_field.substring(tmp_index + 1); - // } - // var fname_path = fname_full_path.split('/') - // html_breadFile += '
'+fname_full_path+'.zip'+'
' - // for(var i=1;i' - // } - // } - // else{ - - - - // } - // } - var fname_full_path="" - var filename_title = $('a.section:first').text() - filename_title = filename_title.substring(0,filename_title.lastIndexOf('.')) - var tmp_index = tableData[fileindex].pic_image_field.indexOf("/",70); - if(tmp_index != -1){ - fname_full_path = tableData[fileindex].pic_image_field.substring(tmp_index + 1); + DatasetPreview.prototype.offsetPrview = function (offset) { + var fileList = this.currentFileList.filter(function (item) { return !item.IsDir }); + if (!fileList.length) { + this.renderPleseSelectFile(); + return; } - var fname_path = fname_full_path.split('/') - //console.log(fname_path) - // var filename_text = tableData[fileindex].pic_image_field.substring(tableData[fileindex].pic_image_field.lastIndexOf('/')+1) - var html_breadFile = '' - // var source_name = filename_title+'.zip' - // html_breadFile += '
'+source_name+'
' - // html_breadFile += '
/
' - html_breadFile += '
'+filename_title+'
' - html_breadFile += '
/
' - for (var i=0;i' - html_breadFile += '
/
' + var file = this.currentFile; + var index = -1; + var nextIndex = 0; + if (file) { + index = fileList.findIndex(function (item) { return (item.ParenDir + item.FileName) == (file.ParenDir + file.FileName); }); + index = Math.max(0, index); + nextIndex = (offset + index + fileList.length) % fileList.length; } - - document.getElementById("breadFile").innerHTML=html_breadFile - -} - -function showlabelflist(){ - if(!isEmpty(labelInfo)){ - var resLabelInfo = JSON.parse(labelInfo) - var textInfo = resLabelInfo["type"] - var html_labelFile = '' - for (var i=0;i${textInfo[i]}` - - - } - document.getElementById("labellist").innerHTML=html_labelFile - setColor() - } - else{ - return - } - -} -function setColor(){ - colorinfo1 = ['rgba(0, 199, 255, 0.4)','rgba(114, 46, 209, 0.4)','rgba(188, 100, 164, 0.4)','rgba(153, 204, 0, 0.4)','rgba(51, 204, 153, 0.4)','rgba(255, 204, 51, 0.4)', - 'rgba(71, 255, 71, 0.4)','rgba(255, 154, 71, 0.4)','rgba(71, 126, 255, 0.4)','rgba(71, 255, 255, 0.4)','rgba(255, 247, 71, 0.4)','rgba(196, 127, 255, 0.4)','rgba(233, 84, 100, 0.4)', - 'rgba(255, 71, 204, 0.4)',' rgba(43, 199, 160, 0.4)'] - colorinfo2 = ['#00C7FF','#722ED1','#BC64A4','#99CC00','#33CC99','#FFCC33','#47FF47','#FF9A47','#477EFF','#47FFFF','#FFF747','#C47FFF','#E95464','#FF47CC','#2BC7A0'] - var el_span = document.querySelectorAll("span.labelInfo") - for (var i=0;i { + var eleObj = $(item); + var data = eleObj.data('data'); + if ((data.ParenDir + data.FileName) == (self.currentFile.ParenDir + self.currentFile.FileName)) { + eleObj.addClass('active'); + self.renderPreview(data); + } + }) + }; -} -function getLabelInfo(uuid){ - $.ajax({ - type:"GET", - url:ip + "/getlabelinfo", - headers: { - authorization:token, - }, - dataType:"text", - data:{ - 'uuid':uuid - }, - async:false, - success:function(json){ - labelInfo = json; - }, - error:function(response) { - + DatasetPreview.prototype.eventInit = function () { + var self = this; + $('#filelist').on('click', '.file-item', function () { + var eleObj = $(this); + var file = eleObj.data('data'); + if (file.IsDir) { + self.nextDirs(file); + } else { + $('#filelist .file-item').removeClass('active'); + eleObj.addClass('active'); + self.currentFile = file; + self.renderPreview(file); } - }); + }).on('click', '.file-more', function () { + self.getPathChildrenMore(); + }); + $('#file_path_container').on('click', '.section', function () { + var eleObj = $(this).parent(); + var pathData = eleObj.data('data'); + var index = self.data.filePath.findIndex(function (item) { return item.path == pathData.path }); + var newPath = self.data.filePath.slice(0, index + 1); + self.data.filePath = newPath; + self.getPathChildren(); + }); + $('#myCanvas_div').on('click', '.prev_view_btn', function () { + self.offsetPrview(-1); + }).on('click', '.next_view_btn', function () { + self.offsetPrview(1); + }); + }; -} \ No newline at end of file + $(document).ready(function () { + window.DatasetPreviewController = new DatasetPreview(); + }) +})(); diff --git a/routers/ai_task/ai_task.go b/routers/ai_task/ai_task.go index d065b6321d..bbfbc095c3 100644 --- a/routers/ai_task/ai_task.go +++ b/routers/ai_task/ai_task.go @@ -2,6 +2,10 @@ package ai_task import ( "archive/zip" + "net/http" + "net/url" + "strings" + "code.gitea.io/gitea/entity" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/cloudbrain" @@ -13,9 +17,7 @@ import ( "code.gitea.io/gitea/routers/response" "code.gitea.io/gitea/services/ai_task_service/schedule" "code.gitea.io/gitea/services/ai_task_service/task" - "net/http" - "net/url" - "strings" + "code.gitea.io/gitea/services/cloudbrain/resource" ) func CreateAITask(ctx *context.Context, form entity.CreateReq) { @@ -385,6 +387,47 @@ func GetNodeInfo(ctx *context.Context) { ctx.JSON(http.StatusOK, response.OuterSuccessWithData(m)) } +func GetImageInfoBySelectedSpec(ctx *context.Context) { + jobType := ctx.Query("job_type") + + if models.JobType(jobType) == (models.JobTypeOnlineInference) { + jobType = string(models.JobTypeDebug) + } + log.Info("required jobType=" + jobType) + computeSourceName := ctx.Query("compute_source") + clusterType := ctx.Query("cluster_type") + + computeSource := models.GetComputeSourceInstance(computeSourceName) + specId := ctx.QueryInt64("spec_id") + hasInternet := ctx.QueryInt("has_internet") + + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ + JobType: models.JobType(jobType), + ComputeResource: computeSourceName, + Cluster: clusterType, + HasInternet: models.SpecInternetQuery(hasInternet), + }) + + if err != nil || spec == nil { + ctx.JSON(http.StatusOK, response.OuterTrBizError(response.SPEC_NOT_AVAILABLE, ctx)) + return + } + + result, bizerr := task.GetAvailableImageInfoBySpec(entity.GetAITaskCreationImageInfoReq{ + ClusterType: entity.ClusterType(clusterType), + ComputeSource: computeSource, + Spec: spec, + JobType: models.JobType(jobType), + UserID: ctx.User.ID, + }) + if bizerr != nil { + log.Error("GetAITaskImageCreationInfo error,err=%v", bizerr) + ctx.JSON(http.StatusOK, response.OuterTrBizError(bizerr, ctx)) + return + } + ctx.JSON(http.StatusOK, response.OuterSuccessWithData(result)) +} + func GetCreationRequiredInfo(ctx *context.Context) { jobType := ctx.Query("job_type") var isOnlineType bool diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c820a3b738..a7b6b9f277 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -59,10 +59,11 @@ package v1 import ( - "code.gitea.io/gitea/routers/reward/point" "net/http" "strings" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/services/memory" "code.gitea.io/gitea/routers/response" @@ -657,6 +658,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/restart", reqWeChatStandard(), reqRepoWriter(models.UnitTypeCloudBrain), reqAITaskInRepo(), reqAdminOrAITaskCreator(), ai_task.RestartAITask) m.Get("/debug_url", reqWeChatStandard(), reqRepoWriter(models.UnitTypeCloudBrain), reqAITaskInRepo(), ai_task.GetNotebookUrl) m.Get("/creation/required", reqWeChatStandard(), reqRepoWriter(models.UnitTypeCloudBrain), ai_task.GetCreationRequiredInfo) + m.Get("/creation/image_by_spec", reqWeChatStandard(), reqRepoWriter(models.UnitTypeCloudBrain), ai_task.GetImageInfoBySelectedSpec) m.Post("/output/reschedule", reqRepoWriter(models.UnitTypeCloudBrain), ai_task.RetryModelSchedule) }, reqToken(), context.RepoRef()) @@ -758,14 +760,17 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqToken()) m.Group("/attachments", func() { - m.Get("/:uuid", repo.GetAttachment) m.Get("/get_chunks", repo.GetSuccessChunks) m.Get("/new_multipart", repo.NewMultipart) m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) m.Post("/complete_multipart", repo.CompleteMultipart) - }, reqToken()) + m.Group("/attachments", func() { + m.Get("/get_dir", repo.GetAttachmentDir) + m.Get("/get_image_content", repo.GetAttachmentImageContent) + m.Get("/get_txt_content", repo.GetAttachmentTxtContent) + }) m.Group("/attachments/model", func() { m.Get("/get_chunks", repo.GetModelChunks) m.Get("/new_multipart", repo.NewModelMultipart) @@ -852,6 +857,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/cloudbrainboard/cloudbrain/resource_queues", repo.GetResourceQueues) m.Get("/cloudbrainboard/ai_center_overview", repo.GetCloubrainOverviewGroupByAiCenter) m.Get("/cloudbrainboard/location", cloudbrainService.GetCloudbrainLocationInfo) + m.Get("/cloudbrainboard/card_data", repo.GetCartStatisticData) m.Group("/cloudbrainboard", func() { m.Get("/downloadAll", repo.DownloadCloudBrainBoard) diff --git a/routers/api/v1/repo/attachments.go b/routers/api/v1/repo/attachments.go index 413b367685..e092dcc9a9 100644 --- a/routers/api/v1/repo/attachments.go +++ b/routers/api/v1/repo/attachments.go @@ -54,6 +54,18 @@ func checkDatasetPermission(ctx *context.APIContext) string { return "" } +func GetAttachmentDir(ctx *context.APIContext) { + routeRepo.GetDirSomeFiles(ctx.Context) +} + +func GetAttachmentImageContent(ctx *context.APIContext) { + routeRepo.GetImageContent(ctx.Context) +} + +func GetAttachmentTxtContent(ctx *context.APIContext) { + routeRepo.GetTxtContent(ctx.Context) +} + func NewMultipart(ctx *context.APIContext) { if errStr := checkDatasetPermission(ctx); errStr != "" { ctx.JSON(200, map[string]string{ diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index c5b2b5a88c..df1a6c1ca9 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -492,13 +492,14 @@ func GetCloudbrainTask(ctx *context.APIContext) { if job.IsNewAITask() { jobAfter, _ := task.UpdateCloudbrain(job) ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": jobAfter.JobName, - "JobStatus": jobAfter.Status, - "SubState": "", - "CreatedTime": jobAfter.CreatedUnix.Format("2006-01-02 15:04:05"), - "CompletedTime": jobAfter.UpdatedUnix.Format("2006-01-02 15:04:05"), - "JobDuration": jobAfter.TrainJobDuration, + "ID": ID, + "JobName": jobAfter.JobName, + "JobStatus": jobAfter.Status, + "DetailedStatus": jobAfter.DetailedStatus, + "SubState": "", + "CreatedTime": jobAfter.CreatedUnix.Format("2006-01-02 15:04:05"), + "CompletedTime": jobAfter.UpdatedUnix.Format("2006-01-02 15:04:05"), + "JobDuration": jobAfter.TrainJobDuration, }) return } @@ -507,13 +508,14 @@ func GetCloudbrainTask(ctx *context.APIContext) { routerRepo.GetAiSafetyTaskByJob(job) job, err = models.GetCloudbrainByID(ID) ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": job.JobName, - "JobStatus": job.Status, - "SubState": "", - "CreatedTime": job.CreatedUnix.Format("2006-01-02 15:04:05"), - "CompletedTime": job.UpdatedUnix.Format("2006-01-02 15:04:05"), - "JobDuration": job.TrainJobDuration, + "ID": ID, + "JobName": job.JobName, + "JobStatus": job.Status, + "DetailedStatus": job.DetailedStatus, + "SubState": "", + "CreatedTime": job.CreatedUnix.Format("2006-01-02 15:04:05"), + "CompletedTime": job.UpdatedUnix.Format("2006-01-02 15:04:05"), + "JobDuration": job.TrainJobDuration, }) } else { jobAfter, err := cloudbrainTask.SyncCloudBrainOneStatus(job) @@ -525,13 +527,14 @@ func GetCloudbrainTask(ctx *context.APIContext) { } ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": jobAfter.JobName, - "JobStatus": jobAfter.Status, - "SubState": "", - "CreatedTime": jobAfter.CreatedUnix.Format("2006-01-02 15:04:05"), - "CompletedTime": jobAfter.UpdatedUnix.Format("2006-01-02 15:04:05"), - "JobDuration": jobAfter.TrainJobDuration, + "ID": ID, + "JobName": jobAfter.JobName, + "JobStatus": jobAfter.Status, + "DetailedStatus": jobAfter.DetailedStatus, + "SubState": "", + "CreatedTime": jobAfter.CreatedUnix.Format("2006-01-02 15:04:05"), + "CompletedTime": jobAfter.UpdatedUnix.Format("2006-01-02 15:04:05"), + "JobDuration": jobAfter.TrainJobDuration, }) } } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index b0d1432228..430f0d22f3 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/setting" @@ -937,6 +939,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.JobName = ciTasks[i].JobName taskDetail.DisplayJobName = ciTasks[i].DisplayJobName taskDetail.Status = ciTasks[i].Status + taskDetail.DetailedStatus = ciTasks[i].DetailedStatus taskDetail.JobType = ciTasks[i].JobType taskDetail.CreatedUnix = ciTasks[i].Cloudbrain.CreatedUnix taskDetail.RunTime = ciTasks[i].Cloudbrain.TrainJobDuration @@ -959,7 +962,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.CardDuration = repo.GetCloudbrainCardDuration(ciTasks[i].Cloudbrain) taskDetail.WaitTime = repo.GetCloudbrainWaitTime(ciTasks[i].Cloudbrain) - if ciTasks[i].Cloudbrain.DeletedAt != nilTime || ciTasks[i].Repo == nil { + if ciTasks[i].Cloudbrain.DeletedAt != nilTime { taskDetail.IsDelete = true } else { taskDetail.IsDelete = false @@ -980,6 +983,28 @@ func GetCloudbrainsDetailData(ctx *context.Context) { }) } +func GetCartStatisticData(ctx *context.Context) { + + dataType := ctx.QueryTrim("type") + category := ctx.QueryTrim("category") + + if _, ok := models.XpuInfoType[dataType]; !ok { + ctx.Error(http.StatusBadRequest) + return + } + if _, ok := models.XpuInfoCategories[category]; !ok { + ctx.Error(http.StatusBadRequest) + return + } + + result, err := models.GetXPUStatisticInfos(models.XpuInfoType[dataType], category) + if err != nil { + log.Error("can not get statistic info", err) + ctx.JSON(http.StatusOK, response.OuterSuccessWithData([]*models.XPUInfoStatisticShow{})) + } + ctx.JSON(http.StatusOK, response.OuterSuccessWithData(result)) +} + func GetCloudbrainsCreateHoursData(ctx *context.Context) { recordCloudbrain, err := models.GetRecordBeginTime() if err != nil { diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 125c3fa0d8..9a13668a69 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -66,11 +66,12 @@ func GetModelArtsNotebook2(ctx *context.APIContext) { } ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": job.JobName, - "JobStatus": job.Status, - "JobDuration": job.TrainJobDuration, - "StartTime": job.StartTime, + "ID": ID, + "JobName": job.JobName, + "JobStatus": job.Status, + "JobDuration": job.TrainJobDuration, + "StartTime": job.StartTime, + "DetailedStatus": job.DetailedStatus, }) } @@ -143,11 +144,12 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { } aiCenterName = cloudbrainService.GetAiCenterShow(job.AiCenter, ctx.Context) ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobID": jobID, - "JobStatus": job.Status, - "JobDuration": job.TrainJobDuration, - "AiCenter": aiCenterName, - "StartTime": job.StartTime, + "JobID": jobID, + "JobStatus": job.Status, + "DetailedStatus": job.DetailedStatus, + "JobDuration": job.TrainJobDuration, + "AiCenter": aiCenterName, + "StartTime": job.StartTime, }) return } @@ -208,11 +210,12 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { } ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobID": jobID, - "JobStatus": job.Status, - "JobDuration": job.TrainJobDuration, - "AiCenter": aiCenterName, - "StartTime": job.StartTime, + "JobID": jobID, + "JobStatus": job.Status, + "JobDuration": job.TrainJobDuration, + "AiCenter": aiCenterName, + "StartTime": job.StartTime, + "DetailedStatus": job.DetailedStatus, }) } diff --git a/routers/home.go b/routers/home.go index 7a8a0de2af..3a113d47ad 100755 --- a/routers/home.go +++ b/routers/home.go @@ -56,7 +56,8 @@ const ( tplRepoSearch base.TplName = "explore/repos/search" tplRoshmci base.TplName = "explore/ros-hmci" - tplExploreCenterMap base.TplName = "explore/center_map" + tplExploreCenterMap base.TplName = "explore/center_map" + tplExploreDomestic base.TplName = "explore/domestic" ) // Home render home page @@ -820,6 +821,10 @@ func ExploreImages(ctx *context.Context) { ctx.HTML(200, tplExploreImages) } +func ExploreDomestic(ctx *context.Context) { + ctx.HTML(200, tplExploreDomestic) +} + func ExploreDataAnalysisUserTrend(ctx *context.Context) { ctx.Data["url_params"] = "UserTrend" ctx.HTML(200, tplExploreExploreDataAnalysis) diff --git a/routers/org/teams.go b/routers/org/teams.go index 17c424578b..9097aefe5e 100644 --- a/routers/org/teams.go +++ b/routers/org/teams.go @@ -6,6 +6,7 @@ package org import ( + "fmt" "net/http" "path" "strings" @@ -310,18 +311,37 @@ func EditTeamPost(ctx *context.Context, form auth.CreateTeamForm) { } t.Description = form.Description if t.Authorize < models.AccessModeOwner { - var units = make([]models.TeamUnit, 0, len(form.Units)) - for _, tp := range form.Units { - units = append(units, models.TeamUnit{ - OrgID: t.OrgID, - TeamID: t.ID, - Type: tp, - }) - } - err := models.UpdateTeamUnits(t, units) - if err != nil { - ctx.Error(http.StatusInternalServerError, "LoadIssue", err.Error()) - return + log.Info("t.Authorize < models.AccessModeOwner") + if t.Authorize < models.AccessModeAdmin { + var units = make([]models.TeamUnit, 0, len(form.Units)) + for _, tp := range form.Units { + units = append(units, models.TeamUnit{ + OrgID: t.OrgID, + TeamID: t.ID, + Type: tp, + }) + } + err := models.UpdateTeamUnits(t, units) + if err != nil { + ctx.Error(http.StatusInternalServerError, "LoadIssue", err.Error()) + return + } + } else { + log.Info("t.Authorize = models.AccessModeAdmin") + + var units = make([]models.TeamUnit, 0, len(models.AllRepoUnitTypes)) + for _, tp := range models.AllRepoUnitTypes { + units = append(units, models.TeamUnit{ + OrgID: t.OrgID, + TeamID: t.ID, + Type: tp, + }) + } + err := models.UpdateTeamUnits(t, units) + if err != nil { + ctx.Error(http.StatusInternalServerError, "LoadIssue", err.Error()) + return + } } } t.CanCreateOrgRepo = form.CanCreateOrgRepo @@ -335,7 +355,7 @@ func EditTeamPost(ctx *context.Context, form auth.CreateTeamForm) { ctx.RenderWithErr(ctx.Tr("form.team_no_units_error"), tplTeamNew, &form) return } - + log.Info("isAuthChanged=" + fmt.Sprint(isAuthChanged) + " isIncludeAllChanged=" + fmt.Sprint(isIncludeAllChanged)) if err := models.UpdateTeam(t, isAuthChanged, isIncludeAllChanged); err != nil { ctx.Data["Err_TeamName"] = true switch { diff --git a/routers/repo/attachment_dir.go b/routers/repo/attachment_dir.go new file mode 100644 index 0000000000..82e0074f82 --- /dev/null +++ b/routers/repo/attachment_dir.go @@ -0,0 +1,368 @@ +package repo + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" + "time" + + "golang.org/x/text/encoding/simplifiedchinese" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +type DirRedisCache struct { + DirList []storage.FileInfo + Marker string +} + +func GetDirSomeFiles(ctx *context.Context) { + uuid := ctx.Query("uuid") + marker := ctx.Query("marker") + pageSize := ctx.QueryInt("pageSize") + prefix := ctx.Query("prefix") + cacheKey := uuid + prefix + marker + " " + fmt.Sprint(pageSize) + //delimiter := ctx.Query("delimiter") + attach, err := models.GetAttachmentByUUID(uuid) + if err == nil { + cacheResult, err := getAttachmentDirFromCache(cacheKey) + if err == nil { + log.Info("load from cache.") + ctx.JSON(200, map[string]interface{}{ + "result_code": "0", + "data": cacheResult.DirList, + "marker": cacheResult.Marker, + }) + return + } + if attach.Type == models.TypeCloudBrainOne { + re, marker, err := storage.GetDirsSomeFileMinio(setting.Attachment.Minio.Bucket, + setting.Attachment.Minio.BasePath+ + models.AttachmentRelativePath(attach.UUID)+ + attach.UUID, prefix, marker, pageSize) + if err == nil { + setAttachmentDirToCache(cacheKey, DirRedisCache{ + DirList: re, + Marker: marker, + }) + ctx.JSON(200, map[string]interface{}{ + "result_code": "0", + "data": re, + "marker": marker, + }) + return + } + } else if attach.Type == models.TypeCloudBrainTwo { + re, marker, err := storage.GetDirsSomeFile(setting.Bucket, + setting.BasePath+ + models.AttachmentRelativePath(attach.UUID)+ + attach.UUID, prefix, marker, pageSize) + if err == nil { + setAttachmentDirToCache(cacheKey, DirRedisCache{ + DirList: re, + Marker: marker, + }) + ctx.JSON(200, map[string]interface{}{ + "result_code": "0", + "data": re, + "marker": marker, + }) + return + } + } + } + ctx.JSON(200, map[string]interface{}{ + "result_code": "-1", + "data": "", + }) +} + +func GetImageContent(ctx *context.Context) { + uuid := ctx.Query("uuid") + fileName := ctx.Query("filePath") + storageType := ctx.QueryInt("type") + if storageType == 0 { + getMinioImageContent(uuid, fileName, ctx) + } else { + if storageType == 1 { + getObsImageContent(uuid, fileName, ctx) + } + } +} + +func GetTxtContent(ctx *context.Context) { + uuid := ctx.Query("uuid") + fileName := ctx.Query("filePath") + storageType := ctx.QueryInt("type") + if storageType == 0 { + getMinioTxtContent(uuid, fileName, ctx) + } else { + if storageType == 1 { + getObsTxtContent(uuid, fileName, ctx) + } + } +} + +func getObsImageContent(uuid string, fileName string, ctx *context.Context) { + objectName := strings.TrimPrefix(path.Join(setting.BasePath+models.AttachmentRelativePath(uuid)+uuid, fileName), "/") + //log.Info("obs objectName=" + objectName) + body, err := storage.ObsDownloadAFile(setting.Bucket, objectName) + if err != nil { + log.Info("upload error.") + } else { + defer body.Close() + ctx.Resp.Header().Set("Content-Type", "image/jpg;charset=utf-8") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + } + if readErr != nil { + break + } + } + } +} + +func getObsTxtContent(uuid string, fileName string, ctx *context.Context) { + objectName := strings.TrimPrefix(path.Join(setting.BasePath+models.AttachmentRelativePath(uuid)+uuid, fileName), "/") + //log.Info("obs objectName=" + objectName) + body, err := storage.ObsDownloadAFile(setting.Bucket, objectName) + if err != nil { + log.Info("download error.") + ctx.JSON(200, map[string]interface{}{ + "result_code": "-1", + "msg": "Read file error.", + "data": "", + }) + return + } else { + readS3Body(body, ctx) + } +} + +func readS3Body(body io.ReadCloser, ctx *context.Context) { + defer body.Close() + result := make([]string, 0) + r := bufio.NewReader(body) + size := 0 + for { + line, err := r.ReadString('\n') + if err == io.EOF { + result = append(result, toUTF8(line)) + size += len(line) + if !toDealLine(result, line, size, ctx) { + return + } + break + } + if err != nil { + log.Info("read error,err=" + err.Error()) + break + } + result = append(result, toUTF8(line)) + size += len(line) + if !toDealLine(result, line, size, ctx) { + return + } + } + ctx.JSON(200, map[string]interface{}{ + "result_code": "0", + "msg": "", + "data": result, + }) +} + +func toDealLine(result []string, line string, size int, ctx *context.Context) bool { + log.Info("line count=" + fmt.Sprint(len(result)) + " size=" + fmt.Sprint(size)) + if len(result) > 2000 || size > 2*1024*1024 { + ctx.JSON(200, map[string]interface{}{ + "result_code": "-1", + "msg": ctx.Tr("repo.attachmentfilesizetobig"), + "data": "", + }) + return false + } + return true +} + +func toUTF8(line string) string { + lineBytes := []byte(line) + if isUtf8(lineBytes) { + return line + } + if isGBK(lineBytes) { + log.Info("this is gbk") + gbkBytes, err := simplifiedchinese.GBK.NewDecoder().Bytes(lineBytes) //gbk 转 utf-8 + if err == nil { + return string(gbkBytes) + } + } + return line +} + +func toGBK(line string) string { + lineBytes := []byte(line) + if !isGBK(lineBytes) { + gbkBytes, err := simplifiedchinese.GBK.NewEncoder().Bytes(lineBytes) //utf-8 转 gbk + if err == nil { + return string(gbkBytes) + } + } + return line +} + +func isGBK(data []byte) bool { + length := len(data) + var i int = 0 + for i < length { + //fmt.Printf("for %x\n", data[i]) + if data[i] <= 0xff { + //编码小于等于127,只有一个字节的编码，兼容ASCII吗 + i++ + continue + } else { + //大于127的使用双字节编码 + if data[i] >= 0x81 && + data[i] <= 0xfe && + data[i+1] >= 0x40 && + data[i+1] <= 0xfe && + data[i+1] != 0xf7 { + i += 2 + continue + } else { + return false + } + } + } + return true +} + +func getMinioImageContent(uuid string, fileName string, ctx *context.Context) { + objectName := strings.TrimPrefix(path.Join(setting.Attachment.Minio.BasePath+models.AttachmentRelativePath(uuid)+uuid, fileName), "/") + //log.Info("minio objectName=" + objectName) + body, err := storage.Attachments.DownloadAFile(setting.Attachment.Minio.Bucket, objectName) + if err != nil { + log.Info("download error.") + } else { + defer body.Close() + ctx.Resp.Header().Set("Content-Type", "image/jpg;charset=utf-8") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + } + if readErr != nil { + break + } + } + } +} + +func getMinioTxtContent(uuid string, fileName string, ctx *context.Context) { + objectName := strings.TrimPrefix(path.Join(setting.Attachment.Minio.BasePath+models.AttachmentRelativePath(uuid)+uuid, fileName), "/") + //log.Info("minio objectName=" + objectName) + body, err := storage.Attachments.DownloadAFile(setting.Attachment.Minio.Bucket, objectName) + if err != nil { + log.Info("download error.") + ctx.JSON(200, map[string]interface{}{ + "result_code": "-1", + "msg": "Read file error.", + "data": "", + }) + return + } else { + readS3Body(body, ctx) + } +} + +func preNUm(data byte) int { + str := fmt.Sprintf("%b", data) + var i int = 0 + for i < len(str) { + if str[i] != '1' { + break + } + i++ + } + return i +} + +func isUtf8(data []byte) bool { + for i := 0; i < len(data); { + if data[i]&0x80 == 0x00 { + // 0XXX_XXXX + i++ + continue + } else if num := preNUm(data[i]); num > 2 { + // 110X_XXXX 10XX_XXXX + // 1110_XXXX 10XX_XXXX 10XX_XXXX + // 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数 + i++ + for j := 0; j < num-1; j++ { + //判断后面的 num - 1 个字节是不是都是10开头 + if data[i]&0xc0 != 0x80 { + return false + } + i++ + } + } else { + //其他情况说明不是utf-8 + return false + } + } + return true +} + +func setAttachmentDirToCache(msgKey string, dirList DirRedisCache) { + msgMapJson, _ := json.Marshal(dirList) + redisValue := string(msgMapJson) + log.Info("set redis key=" + msgKey + " value=" + redisValue) + re, err := redis_client.Setex(msgKey, redisValue, 7*24*3600*time.Second) + if err == nil { + log.Info("re =" + fmt.Sprint(re)) + } else { + log.Info("set redis error:" + err.Error()) + } +} + +func getAttachmentDirFromCache(msgKey string) (*DirRedisCache, error) { + valueStr, err := redis_client.Get(msgKey) + //msgMap := make(map[string]string, 0) + fileInfos := &DirRedisCache{} + if err == nil { + if valueStr != "" { + err1 := json.Unmarshal([]byte(valueStr), fileInfos) + if err1 != nil { + log.Info("unmarshal json failed. " + err1.Error()) + return nil, err1 + } + } else { + return nil, errors.New("cache is empty.") + } + } else { + log.Info("Failed to load from reids. " + err.Error()) + return nil, err + } + return fileInfos, nil +} diff --git a/routers/repo/dir.go b/routers/repo/dir.go index 81549e76ac..fbd91bf0db 100755 --- a/routers/repo/dir.go +++ b/routers/repo/dir.go @@ -127,6 +127,7 @@ func DirIndex(ctx *context.Context) { ctx.Data["Path"] = dirArray ctx.Data["Dirs"] = true ctx.Data["Uuid"] = uuid + ctx.Data["Type"] = attachment.Type ctx.Data["PageIsDataset"] = true ctx.HTML(200, tplDirIndex) diff --git a/routers/repo/download.go b/routers/repo/download.go index f1b6e09b75..7a1c25bf85 100755 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -43,7 +43,7 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error { cs = "utf-8" } ctx.Resp.Header().Set("Content-Type", "text/plain; charset="+strings.ToLower(cs)) - } else if base.IsImageFile(buf) || base.IsPDFFile(buf) { + } else if base.IsImageFile(buf) { ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, name)) } else { ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index ab9961ffba..42e5583db3 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -1356,13 +1356,14 @@ func GetGrampusNotebook(ctx *context.APIContext) { } ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": jobAfter.JobName, - "JobStatus": jobAfter.Status, - "AiCenter": aiCenterName, - "CreatedTime": jobAfter.CreatedUnix.Format("2006-01-02 15:04:05"), - "CompletedTime": jobAfter.UpdatedUnix.Format("2006-01-02 15:04:05"), - "JobDuration": jobAfter.TrainJobDuration, + "ID": ID, + "JobName": jobAfter.JobName, + "JobStatus": jobAfter.Status, + "DetailedStatus": jobAfter.DetailedStatus, + "AiCenter": aiCenterName, + "CreatedTime": jobAfter.CreatedUnix.Format("2006-01-02 15:04:05"), + "CompletedTime": jobAfter.UpdatedUnix.Format("2006-01-02 15:04:05"), + "JobDuration": jobAfter.TrainJobDuration, }) } diff --git a/routers/response/response_list.go b/routers/response/response_list.go index e37673b269..9a3b0c671c 100644 --- a/routers/response/response_list.go +++ b/routers/response/response_list.go @@ -37,3 +37,5 @@ var BRANCH_NOT_EXISTS = &BizError{Code: 2020, DefaultMsg: "The branch does not e var MODEL_NUM_OVER_LIMIT = &BizError{Code: 2021, DefaultMsg: "The number of models exceeds the limit of 30", TrCode: "repo.debug.manage.model_num_over_limit"} var DATASET_NUMBER_OVER_LIMIT = &BizError{Code: 2022, DefaultMsg: "The dataset count exceed the limit", TrCode: "ai_task.dataset_number_over_limit"} var NOTEBOOK_EXCEED_MAX_NUM = &BizError{Code: 2023, DefaultMsg: "You can have up to 5 Debug Tasks, please try again after delete some tasks. ", TrCode: "ai_task.too_many_notebook"} +var CAN_NOT_STOP_CREATING_JOB = &BizError{Code: 2024, DefaultMsg: "AI task is creating, can not be stopped", TrCode: "ai_task.can_not_stop_creating_job"} +var NO_CENTER_MATCH = &BizError{Code: 2024, DefaultMsg: "", TrCode: "ai_task.no_center_match"} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 1c21f7c8fa..3938eb4ba2 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -431,6 +431,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/data_analysis/Overview", routers.ExploreDataAnalysisOverview) m.Get("/data_analysis/BrainAnalysis", routers.ExploreDataAnalysisBrainAnalysis) m.Get("/center_map", reqSignIn, routers.CenterMapUI) + m.Get("/domestic", routers.ExploreDomestic) }, ignSignIn) m.Combo("/install", routers.InstallInit).Get(routers.Install). diff --git a/services/ai_task_service/cluster/c2net.go b/services/ai_task_service/cluster/c2net.go index 581e4535f0..d099f2c20a 100644 --- a/services/ai_task_service/cluster/c2net.go +++ b/services/ai_task_service/cluster/c2net.go @@ -59,7 +59,7 @@ func (c C2NetClusterAdapter) CreateOnlineInfer(req entity.CreateNoteBookTaskRequ return convertGrampus2NoteBookRes(jobResult), nil } -func (c C2NetClusterAdapter) GetNotebookImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) { +func (c C2NetClusterAdapter) GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { processType := req.ComputeSource.FullName images, err := grampus.GetImages(processType, string(req.JobType)) if err != nil { @@ -69,15 +69,37 @@ func (c C2NetClusterAdapter) GetNotebookImages(req entity.GetImageReq) ([]entity if images == nil || images.Infos == nil || len(images.Infos) == 0 { return nil, true, err } - r := make([]entity.ClusterImage, len(images.Infos)) - for i, v := range images.Infos { - r[i] = ConvertGrampusImageToStandard(v) + + r := make([]entity.ClusterImage, 0) + for _, v := range images.Infos { + if hasIntersection(v.AICenterImage, centerId...) { + r = append(r, ConvertGrampusImageToStandard(v)) + } + } + if len(r) == 0 { + return nil, false, nil } + return r, false, nil } -func (c C2NetClusterAdapter) GetTrainImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) { - return c.GetNotebookImages(req) +func hasIntersection(imageCenterInfos []models.AICenterImage, centerId ...string) bool { + if len(centerId) == 0 || len(imageCenterInfos) == 0 { + //如果没传centerId或者查询的镜像不含可用中心信息，不进行判断，直接返回true + return true + } + for _, aicenterImage := range imageCenterInfos { + for _, centerCode := range centerId { + if aicenterImage.AICenterID == centerCode { + return true + } + } + } + return false +} + +func (c C2NetClusterAdapter) GetTrainImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { + return c.GetNotebookImages(req, centerId...) } func ConvertGrampusImageToStandard(image models.GrampusImage) entity.ClusterImage { @@ -658,12 +680,13 @@ func convertGrampusTrainJobResponse(job models.GrampusJobInfo) *entity.QueryTask centerName = task.CenterName[0] } return &entity.QueryTaskResponse{ - StartedAt: timeutil.TimeStamp(job.StartedAt), - CompletedAt: timeutil.TimeStamp(job.CompletedAt), - Status: job.Status, - CenterId: centerId, - CenterName: centerName, - JobId: job.JobID, + StartedAt: timeutil.TimeStamp(job.StartedAt), + CompletedAt: timeutil.TimeStamp(job.CompletedAt), + Status: job.Status, + DetailedStatus: job.DetailedStatus, + CenterId: centerId, + CenterName: centerName, + JobId: job.JobID, } } diff --git a/services/ai_task_service/cluster/cloudbrain_one.go b/services/ai_task_service/cluster/cloudbrain_one.go index f43b9902b8..cef48dd279 100644 --- a/services/ai_task_service/cluster/cloudbrain_one.go +++ b/services/ai_task_service/cluster/cloudbrain_one.go @@ -37,11 +37,11 @@ func (c CloudbrainOneClusterAdapter) CreateOnlineInfer(req entity.CreateNoteBook return nil, nil } -func (c CloudbrainOneClusterAdapter) GetNotebookImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) { +func (c CloudbrainOneClusterAdapter) GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { return nil, true, nil } -func (c CloudbrainOneClusterAdapter) GetTrainImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) { +func (c CloudbrainOneClusterAdapter) GetTrainImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { return c.GetNotebookImages(req) } diff --git a/services/ai_task_service/cluster/cloudbrain_two.go b/services/ai_task_service/cluster/cloudbrain_two.go index 6991504d12..95bbe77930 100644 --- a/services/ai_task_service/cluster/cloudbrain_two.go +++ b/services/ai_task_service/cluster/cloudbrain_two.go @@ -52,13 +52,19 @@ func (c CloudbrainTwoClusterAdapter) CreateNoteBook(req entity.CreateNoteBookTas WorkspaceID: "0", }) } else { + var poolId = poolInfos.PoolInfo[0].PoolId + for _, poolInfo := range poolInfos.PoolInfo { + if poolInfo.PoolName == t.Spec.QueueCode { + poolId = poolInfo.PoolId + } + } jobResult, err = cloudbrain_two.CreateNotebook2(models.CreateNotebook2Params{ JobName: req.Name, Description: req.Description, Flavor: t.Spec.SourceSpecId, Duration: t.AutoStopDuration, ImageID: t.ImageId, - PoolID: poolInfos.PoolInfo[0].PoolId, + PoolID: poolId, Feature: models.NotebookFeature, Volume: models.VolumeReq{ Capacity: setting.Capacity, @@ -82,7 +88,7 @@ func (c CloudbrainTwoClusterAdapter) CreateOnlineInfer(req entity.CreateNoteBook var cloudbrainTwoNotebookImages []entity.ClusterImage -func (c CloudbrainTwoClusterAdapter) GetNotebookImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) { +func (c CloudbrainTwoClusterAdapter) GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { if cloudbrainTwoNotebookImages == nil || len(cloudbrainTwoNotebookImages) == 0 { images := setting.StImageInfos.ImageInfo cloudbrainTwoNotebookImages = make([]entity.ClusterImage, len(images)) @@ -99,7 +105,7 @@ func (c CloudbrainTwoClusterAdapter) GetNotebookImages(req entity.GetImageReq) ( var cloudbrainTwoTrainImages []entity.ClusterImage -func (c CloudbrainTwoClusterAdapter) GetTrainImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) { +func (c CloudbrainTwoClusterAdapter) GetTrainImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) { if cloudbrainTwoTrainImages == nil || len(cloudbrainTwoTrainImages) == 0 { var versionInfos modelarts.VersionInfo if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { diff --git a/services/ai_task_service/cluster/cluster_base.go b/services/ai_task_service/cluster/cluster_base.go index ce66e16629..e877dfa2ad 100644 --- a/services/ai_task_service/cluster/cluster_base.go +++ b/services/ai_task_service/cluster/cluster_base.go @@ -48,7 +48,7 @@ type ClusterAdapter interface { GetResourceUsage(opts entity.ClusterResourceUsageOpts) (*entity.ResourceUsage, error) //GetImages return available list of clusters //The second parameter will return true if image is no limit - GetNotebookImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) - GetTrainImages(req entity.GetImageReq) ([]entity.ClusterImage, bool, error) + GetNotebookImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) + GetTrainImages(req entity.GetImageReq, centerId ...string) ([]entity.ClusterImage, bool, error) CreateOnlineInfer(req entity.CreateNoteBookTaskRequest) (*entity.CreateNoteBookTaskResponse, error) } diff --git a/services/ai_task_service/task/cloudbrain_one_notebook_task.go b/services/ai_task_service/task/cloudbrain_one_notebook_task.go index 701da0c47f..db1f67ca56 100644 --- a/services/ai_task_service/task/cloudbrain_one_notebook_task.go +++ b/services/ai_task_service/task/cloudbrain_one_notebook_task.go @@ -133,6 +133,16 @@ func (g CloudbrainOneNotebookTaskTemplate) CallCreationAPI(ctx *context.Creation return response.SYSTEM_ERROR } form := ctx.Request + + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } + req := entity.CreateNoteBookTaskRequest{ Name: form.JobName, Tasks: []entity.NoteBookTask{ @@ -147,12 +157,8 @@ func (g CloudbrainOneNotebookTaskTemplate) CallCreationAPI(ctx *context.Creation OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), AutoStopDuration: autoStopDurationMs, Capacity: setting.Capacity, - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), - Spec: ctx.Spec, + CenterID: centerIds, + Spec: ctx.Spec, }, }, } diff --git a/services/ai_task_service/task/cloudbrain_one_train_task.go b/services/ai_task_service/task/cloudbrain_one_train_task.go index 4a8716a71e..6d16e82d55 100644 --- a/services/ai_task_service/task/cloudbrain_one_train_task.go +++ b/services/ai_task_service/task/cloudbrain_one_train_task.go @@ -97,6 +97,14 @@ func (g CloudbrainOneTrainTaskTemplate) CallCreationAPI(ctx *context.CreationCon return response.SYSTEM_ERROR } form := ctx.Request + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } req := entity.CreateTrainTaskRequest{ Name: form.JobName, DisplayJobName: form.DisplayJobName, @@ -108,17 +116,13 @@ func (g CloudbrainOneTrainTaskTemplate) CallCreationAPI(ctx *context.CreationCon ImageUrl: strings.TrimSpace(form.ImageUrl), Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), Code: ctx.GetContainerDataArray(entity.ContainerCode), - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), - PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), - BootFile: form.BootFile, - OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), - LogPath: ctx.GetContainerDataArray(entity.ContainerLogPath), - Params: form.ParamArray, - Spec: ctx.Spec, + CenterID: centerIds, + PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), + BootFile: form.BootFile, + OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), + LogPath: ctx.GetContainerDataArray(entity.ContainerLogPath), + Params: form.ParamArray, + Spec: ctx.Spec, }, }, } @@ -142,6 +146,14 @@ func (g CloudbrainOneTrainTaskTemplate) CallRestartAPI(ctx *context.CreationCont return response.SYSTEM_ERROR } form := ctx.Request + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } req := entity.CreateTrainTaskRequest{ Name: form.JobName, DisplayJobName: form.DisplayJobName, @@ -153,16 +165,12 @@ func (g CloudbrainOneTrainTaskTemplate) CallRestartAPI(ctx *context.CreationCont ImageUrl: strings.TrimSpace(form.ImageUrl), Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), Code: ctx.GetContainerDataArray(entity.ContainerCode), - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), - PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), - BootFile: form.BootFile, - OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), - Params: form.ParamArray, - Spec: ctx.Spec, + CenterID: centerIds, + PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), + BootFile: form.BootFile, + OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), + Params: form.ParamArray, + Spec: ctx.Spec, }, }, } diff --git a/services/ai_task_service/task/cloudbrain_two_train_task.go b/services/ai_task_service/task/cloudbrain_two_train_task.go index 071c925658..57fa0f1e93 100644 --- a/services/ai_task_service/task/cloudbrain_two_train_task.go +++ b/services/ai_task_service/task/cloudbrain_two_train_task.go @@ -115,24 +115,28 @@ func (g CloudbrainTwoTrainTaskTemplate) CallCreationAPI(ctx *context.CreationCon } } form := ctx.Request + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } req := entity.CreateTrainTaskRequest{ Name: form.JobName, DisplayJobName: form.DisplayJobName, Description: form.Description, Tasks: []entity.TrainTask{ { - Name: form.JobName, - ResourceSpecId: ctx.Spec.SourceSpecId, - ImageId: form.ImageID, - ImageUrl: strings.TrimSpace(form.ImageUrl), - Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), - Code: ctx.GetContainerDataArray(entity.ContainerCode), - LogPath: ctx.GetContainerDataArray(entity.ContainerLogPath), - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), + Name: form.JobName, + ResourceSpecId: ctx.Spec.SourceSpecId, + ImageId: form.ImageID, + ImageUrl: strings.TrimSpace(form.ImageUrl), + Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), + Code: ctx.GetContainerDataArray(entity.ContainerCode), + LogPath: ctx.GetContainerDataArray(entity.ContainerLogPath), + CenterID: centerIds, PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), BootFile: form.BootFile, OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), diff --git a/services/ai_task_service/task/grampus_notebook_task.go b/services/ai_task_service/task/grampus_notebook_task.go index 3648735d92..85868fd149 100644 --- a/services/ai_task_service/task/grampus_notebook_task.go +++ b/services/ai_task_service/task/grampus_notebook_task.go @@ -199,6 +199,14 @@ func (g GrampusNoteBookTaskTemplate) CallCreationAPI(ctx *context.CreationContex return response.SYSTEM_ERROR } form := ctx.Request + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } imageUrl := strings.TrimSpace(form.ImageUrl) if form.ImageID != "" { imageUrl = "" @@ -215,12 +223,8 @@ func (g GrampusNoteBookTaskTemplate) CallCreationAPI(ctx *context.CreationContex Code: ctx.GetContainerDataArray(entity.ContainerCode), AutoStopDuration: autoStopDurationMs, Capacity: setting.Capacity, - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), - Spec: ctx.Spec, + CenterID: centerIds, + Spec: ctx.Spec, }, }, } diff --git a/services/ai_task_service/task/grampus_online_infer_task.go b/services/ai_task_service/task/grampus_online_infer_task.go index cbcfbf26ab..f65e00930a 100644 --- a/services/ai_task_service/task/grampus_online_infer_task.go +++ b/services/ai_task_service/task/grampus_online_infer_task.go @@ -95,6 +95,14 @@ func (g GrampusOnlineInferTaskTemplate) CallCreationAPI(ctx *context.CreationCon return response.SYSTEM_ERROR } form := ctx.Request + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } imageUrl := strings.TrimSpace(form.ImageUrl) if form.ImageID != "" { imageUrl = "" @@ -119,13 +127,9 @@ func (g GrampusOnlineInferTaskTemplate) CallCreationAPI(ctx *context.CreationCon OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), AutoStopDuration: -1, Capacity: setting.Capacity, - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), - Spec: ctx.Spec, - BootFile: ctx.Request.BootFile, + CenterID: centerIds, + Spec: ctx.Spec, + BootFile: ctx.Request.BootFile, }, }, } diff --git a/services/ai_task_service/task/grampus_train_task.go b/services/ai_task_service/task/grampus_train_task.go index fe47a1cfd1..7c676cbcdb 100644 --- a/services/ai_task_service/task/grampus_train_task.go +++ b/services/ai_task_service/task/grampus_train_task.go @@ -131,6 +131,14 @@ func (g GrampusTrainTaskTemplate) CallCreationAPI(ctx *context.CreationContext) return response.SYSTEM_ERROR } form := ctx.Request + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } imageUrl := strings.TrimSpace(form.ImageUrl) if form.ImageID != "" { imageUrl = "" @@ -140,17 +148,13 @@ func (g GrampusTrainTaskTemplate) CallCreationAPI(ctx *context.CreationContext) DisplayJobName: form.DisplayJobName, Tasks: []entity.TrainTask{ { - Name: form.JobName, - ResourceSpecId: ctx.Spec.SourceSpecId, - ImageId: form.ImageID, - ImageUrl: imageUrl, - Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), - Code: ctx.GetContainerDataArray(entity.ContainerCode), - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), + Name: form.JobName, + ResourceSpecId: ctx.Spec.SourceSpecId, + ImageId: form.ImageID, + ImageUrl: imageUrl, + Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), + Code: ctx.GetContainerDataArray(entity.ContainerCode), + CenterID: centerIds, PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), BootFile: form.BootFile, OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), @@ -182,6 +186,14 @@ func (g GrampusTrainTaskTemplate) CallRestartAPI(ctx *context.CreationContext) * return response.SYSTEM_ERROR } form := ctx.Request + centerIds, bizErr := GetAvailableCenterIds(ctx.Spec, models.GetAvailableCenterIdOpts{ + UserId: ctx.User.ID, + JobType: g.JobType, + HasInternet: form.HasInternet, + }, form.ComputeSource, form.ImageID, g.ClusterType) + if bizErr != nil { + return bizErr + } req := entity.CreateTrainTaskRequest{ Name: form.JobName, DisplayJobName: form.DisplayJobName, @@ -193,16 +205,12 @@ func (g GrampusTrainTaskTemplate) CallRestartAPI(ctx *context.CreationContext) * ImageUrl: strings.TrimSpace(form.ImageUrl), Datasets: ctx.GetContainerDataArray(entity.ContainerDataset), Code: ctx.GetContainerDataArray(entity.ContainerCode), - CenterID: ctx.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ - UserId: ctx.User.ID, - JobType: g.JobType, - HasInternet: form.HasInternet, - }), - PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), - BootFile: form.BootFile, - OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), - Params: form.ParamArray, - Spec: ctx.Spec, + CenterID: centerIds, + PreTrainModel: ctx.GetContainerDataArray(entity.ContainerPreTrainModel), + BootFile: form.BootFile, + OutPut: ctx.GetContainerDataArray(entity.ContainerOutPutPath), + Params: form.ParamArray, + Spec: ctx.Spec, }, }, } diff --git a/services/ai_task_service/task/opt_handler.go b/services/ai_task_service/task/opt_handler.go index d2d8338bf7..5937bce3b1 100644 --- a/services/ai_task_service/task/opt_handler.go +++ b/services/ai_task_service/task/opt_handler.go @@ -692,7 +692,7 @@ func (g DefaultCreationHandler) NotifyCreation(ctx *context.CreationContext) *re func (DefaultCreationHandler) CheckNotebookCount(ctx *context.CreationContext) *response.BizError { - if setting.NotebookStrategy.ClearEnabled && ctx.Request.JobType == models.JobTypeDebug { + if ctx.Request.JobType == models.JobTypeDebug { count, err := models.GetNotebooksCountByUser(ctx.User.ID) if err != nil { log.Warn("can not get user notebook count", err) diff --git a/services/ai_task_service/task/super_compute_task.go b/services/ai_task_service/task/super_compute_task.go index d32fd8a3fd..06be39028f 100644 --- a/services/ai_task_service/task/super_compute_task.go +++ b/services/ai_task_service/task/super_compute_task.go @@ -29,7 +29,7 @@ func init() { RegisterTask(models.JobTypeSuperCompute, entity.C2Net, t) } -func (g SuperComputeTaskTemplate) GetImages(computeSource models.ComputeSource) ([]entity.ClusterImage, bool, *response.BizError) { +func (g SuperComputeTaskTemplate) GetImages(computeSource models.ComputeSource, centerId ...string) ([]entity.ClusterImage, bool, *response.BizError) { c := g.GetMyCluster() if c == nil { log.Error("Get cluster failed") diff --git a/services/ai_task_service/task/task_base.go b/services/ai_task_service/task/task_base.go index 9f62eba2cf..6bce1e7bc9 100644 --- a/services/ai_task_service/task/task_base.go +++ b/services/ai_task_service/task/task_base.go @@ -60,7 +60,7 @@ type AITaskTemplate interface { GetDebugUrl(cloudbrainId int64, fileName ...string) (string, *response.BizError) GetOperationProfile(cloudbrainId int64) (*entity.OperationProfile, *response.BizError) GetResourceUsage(opts entity.GetResourceUsageOpts) (*entity.ResourceUsage, *response.BizError) - GetImages(computeSource models.ComputeSource) ([]entity.ClusterImage, bool, *response.BizError) + GetImages(computeSource models.ComputeSource, centerId ...string) ([]entity.ClusterImage, bool, *response.BizError) GetSpecs(opts entity.GetSpecOpts) ([]*api.SpecificationShow, *response.BizError) GetConfig(opts entity.AITaskConfigKey) *entity.AITaskBaseConfig GetNodeInfo(cloudbrainId int64) ([]entity.AITaskNodeInfo, *response.BizError) @@ -237,6 +237,10 @@ func (g DefaultAITaskTemplate) Stop(cloudbrainId int64) (*entity.AITaskBriefInfo } if err != nil { + log.Error("StopTask err.cloudbrainId=%d err=%v", cloudbrainId, err) + if models.IsErrCannotStopCreatingGrampusJob(err) { + return nil, response.CAN_NOT_STOP_CREATING_JOB + } log.Error("StopTask err.cloudbrainId=%d err=%v", cloudbrainId, err) return nil, response.NewBizError(err) } @@ -456,7 +460,7 @@ func (g DefaultAITaskTemplate) GetResourceUsage(opts entity.GetResourceUsageOpts return res, nil } -func (g DefaultAITaskTemplate) GetImages(computeSource models.ComputeSource) ([]entity.ClusterImage, bool, *response.BizError) { +func (g DefaultAITaskTemplate) GetImages(computeSource models.ComputeSource, centerId ...string) ([]entity.ClusterImage, bool, *response.BizError) { c := g.GetMyCluster() if c == nil { log.Error("Get cluster failed") @@ -470,12 +474,12 @@ func (g DefaultAITaskTemplate) GetImages(computeSource models.ComputeSource) ([] images, customFlag, err = c.GetNotebookImages(entity.GetImageReq{ ComputeSource: computeSource, JobType: g.JobType, - }) + }, centerId...) } else { images, customFlag, err = c.GetTrainImages(entity.GetImageReq{ ComputeSource: computeSource, JobType: g.JobType, - }) + }, centerId...) } if err != nil { log.Error("GetImages err.computeSource=%s err =%v", computeSource.Name, err) diff --git a/services/ai_task_service/task/task_creation_info.go b/services/ai_task_service/task/task_creation_info.go index 53560b4d85..82d67bf244 100644 --- a/services/ai_task_service/task/task_creation_info.go +++ b/services/ai_task_service/task/task_creation_info.go @@ -55,11 +55,6 @@ func GetAITaskCreationInfo(req entity.GetAITaskCreationInfoReq) (*entity.Creatio //生成任务名称 result.DisplayJobName = t.GetDisplayJobName(req.User.Name) - // 查询镜像列表 - if images, canUseAll, err := t.GetImages(*req.ComputeSource); err == nil { - result.Images = images - result.CanUseAllImages = canUseAll - } specsMap := make(map[string][]*structs.SpecificationShow, 0) //查询有网资源规格 if specs, err := t.GetSpecs(entity.GetSpecOpts{ @@ -85,6 +80,12 @@ func GetAITaskCreationInfo(req entity.GetAITaskCreationInfoReq) (*entity.Creatio specsMap["all"] = specs } result.Specs = specsMap + // 查询镜像列表 + if images, canUseAll, err := t.GetImages(*req.ComputeSource); err == nil { + result.Images = images + result.CanUseAllImages = canUseAll + } + result.Config = entity.AITaskCreationConfig{ DatasetMaxSize: setting.DebugAttachSize * 1000 * 1000 * 1000, } @@ -96,3 +97,24 @@ func GetAITaskCreationInfo(req entity.GetAITaskCreationInfoReq) (*entity.Creatio } return result, nil } + +func GetAvailableImageInfoBySpec(req entity.GetAITaskCreationImageInfoReq) (*entity.ImageRequiredInfo, *response.BizError) { + result := &entity.ImageRequiredInfo{} + t, err := GetAITaskTemplate(req.JobType, req.ClusterType) + + if err != nil { + log.Error("param error") + return nil, err + } + centerIds := req.Spec.GetAvailableCenterIds(models.GetAvailableCenterIdOpts{ + UserId: req.UserID, + JobType: req.JobType, + }) + + if images, canUseAll, err := t.GetImages(*req.ComputeSource, centerIds...); err == nil { + result.Images = images + result.CanUseAllImages = canUseAll + } + return result, nil + +} diff --git a/services/ai_task_service/task/task_extend.go b/services/ai_task_service/task/task_extend.go index f4b1f89a53..ec11d4f464 100644 --- a/services/ai_task_service/task/task_extend.go +++ b/services/ai_task_service/task/task_extend.go @@ -96,6 +96,7 @@ func getCloudBrainDatasetInfo4Local(uuid string, datasetname string, isNeedDown link := "" url := "" isDelete := false + var size int64 attachment, err := models.GetAttachmentByUUID(uuidStr) if err != nil { log.Error("GetAttachmentByUUID failed:%v", err.Error()) @@ -106,6 +107,7 @@ func getCloudBrainDatasetInfo4Local(uuid string, datasetname string, isNeedDown isDelete = true } else { name = attachment.Name + size = attachment.Size dataset, err := models.GetDatasetByID(attachment.DatasetID) if err != nil { log.Error("GetDatasetByID failed:%v", err.Error()) @@ -128,6 +130,7 @@ func getCloudBrainDatasetInfo4Local(uuid string, datasetname string, isNeedDown RepositoryLink: link, IsDelete: isDelete, UUID: uuidStr, + Size: size, }) } log.Info("dataset length=" + fmt.Sprint(len(datasetDownload))) diff --git a/services/ai_task_service/task/task_service.go b/services/ai_task_service/task/task_service.go index a1ef16fe46..f7a41d6b8c 100644 --- a/services/ai_task_service/task/task_service.go +++ b/services/ai_task_service/task/task_service.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/entity" + "code.gitea.io/gitea/manager/client/grampus" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" @@ -100,6 +101,7 @@ func buildAITaskInfo(task *models.Cloudbrain, creator *models.User, config *enti ID: task.ID, JobID: task.JobID, Status: task.Status, + DetailedStatus: task.DetailedStatus, JobType: task.JobType, DisplayJobName: task.DisplayJobName, FormattedDuration: task.TrainJobDuration, @@ -240,6 +242,8 @@ func StopAITaskByJobNameFromRemote(task *models.Cloudbrain, queryFunc QueryListF return nil } +const DEFAULT_DETAILED_STATUS = "-" + func UpdateByQueryResponse(res *entity.QueryTaskResponse, task *models.Cloudbrain) error { if res.JobId == "" { return nil @@ -260,6 +264,10 @@ func UpdateByQueryResponse(res *entity.QueryTaskResponse, task *models.Cloudbrai newStatus := TransAITaskStatus(res.Status) task.Status = newStatus + task.DetailedStatus = res.DetailedStatus + if res.DetailedStatus == "" || res.DetailedStatus == res.Status { + task.DetailedStatus = DEFAULT_DETAILED_STATUS + } if res.StartedAt > 0 { task.StartTime = res.StartedAt } @@ -722,6 +730,49 @@ func SyncAITaskStatus() { } } } +func GetAvailableCenterIds(specification *models.Specification, opts models.GetAvailableCenterIdOpts, computeSource *models.ComputeSource, + imageId string, clusterType entity.ClusterType) ([]string, *response.BizError) { + centerIds := specification.GetAvailableCenterIds(opts) + + if len(centerIds) == 0 || imageId == "" || clusterType != entity.C2Net { + return centerIds, nil + } + + processType := computeSource.FullName + images, err := grampus.GetImages(processType, string(opts.JobType)) + if err != nil { + log.Warn("can not get image info from grampus", err) + return centerIds, nil + } + var imageCenterIds []string + for _, image := range images.Infos { + if image.ID == imageId { + for _, centerInfo := range image.AICenterImage { + imageCenterIds = append(imageCenterIds, centerInfo.AICenterID) + } + break + } + } + if len(imageCenterIds) == 0 { + return centerIds, nil + } + + var intersectionCenterIds []string + for _, centerId := range centerIds { + for _, imageCenterId := range imageCenterIds { + if centerId == imageCenterId { + intersectionCenterIds = append(intersectionCenterIds, centerId) + break + } + } + } + if len(intersectionCenterIds) == 0 { + return intersectionCenterIds, response.NO_CENTER_MATCH + } + + return intersectionCenterIds, nil + +} func HandleNoJobIdAITasks() { defer func() { diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go index 9e79689c5f..6fe1284e86 100644 --- a/services/cloudbrain/resource/resource_specification.go +++ b/services/cloudbrain/resource/resource_specification.go @@ -290,6 +290,7 @@ func GetAndCheckSpec(userId int64, specId int64, opts models.FindSpecsOptions) ( return nil, nil } opts.SpecId = specId + opts.HasInternet = models.QueryAllSpecs r, err := FindAvailableSpecs(userId, opts) log.Info("FindAvailableSpecs result=%+v", r) if err != nil { diff --git a/services/cloudbrain/statistic.go b/services/cloudbrain/statistic.go new file mode 100644 index 0000000000..328d32a5b9 --- /dev/null +++ b/services/cloudbrain/statistic.go @@ -0,0 +1,159 @@ +package cloudbrain + +import ( + "code.gitea.io/gitea/modules/util" + + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +var pageSize = 500 + +func CardStatistic() { + sevenDayStatistic() + thirtyDayStatistic() + daysAllStatistic() + +} + +func daysAllStatistic() { + + daysStatistic(0, models.TypeAllDays) +} + +func sevenDayStatistic() { + + daysStatistic(7, models.TypeSevenDays) + +} + +func thirtyDayStatistic() { + + daysStatistic(30, models.TypeThirtyDays) + +} + +func daysStatistic(days int, statisticType int) { + xpuCardStatisticMap, err := getStatisticInfoMap(days, statisticType) + if err != nil { + return + } + + updateOrInsertStatisticRecord(xpuCardStatisticMap) + +} + +func updateOrInsertStatisticRecord(xpuCardStatisticMap map[string]*models.XPUInfoStatistic) { + for _, v := range xpuCardStatisticMap { + v.UsedCardHour = v.UsedDuration / 3600 + err := models.UpdateOrInsertXPUInfoStatistic(v) + if err != nil { + log.Warn("update or insert xpu statistic err", err) + } + } +} + +func getStatisticInfoMap(days int, statisticType int, beginTime ...int64) (map[string]*models.XPUInfoStatistic, error) { + endTimeUnix := time.Now().Unix() + beginTimeUnix := time.Now().AddDate(0, 0, -days).Unix() + if days == 0 { + beginTimeUnix = 1 + } + if len(beginTime) > 0 { + beginTimeUnix = beginTime[0] + } + + _, count, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: 1, + PageSize: 1, + }, + NeedRepoInfo: false, + BeginTimeUnix: beginTimeUnix, + EndTimeUnix: endTimeUnix, + Type: -1, + AccCardsNum: -1, + }) + if err != nil { + log.Error("Get job failed:", err) + return nil, err + } + + xpuInfos, err := models.GetXPUInfos() + if err != nil { + log.Error("can not get XPU base info", err) + return nil, err + } + + var xpuCardStatisticMap = make(map[string]*models.XPUInfoStatistic, len(xpuInfos)) + var xpuCardStatisticUserMap = make(map[string]map[int64]struct{}, len(xpuInfos)) + for _, xpuInfo := range xpuInfos { + xpuCardStatisticMap[xpuInfo.CardType] = &models.XPUInfoStatistic{ + InfoID: xpuInfo.ID, + Type: statisticType, + UpdatedUnix: endTimeUnix, + } + xpuCardStatisticUserMap[xpuInfo.CardType] = make(map[int64]struct{}, 0) + } + + totalPage := util.GetTotalPage(count, pageSize) + + for i := 0; i < totalPage; i++ { + + ciTasks, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: i + 1, + PageSize: pageSize, + }, + NeedRepoInfo: false, + BeginTimeUnix: beginTimeUnix, + EndTimeUnix: endTimeUnix, + Type: -1, + AccCardsNum: -1, + }) + if err != nil { + log.Error("Get job failed:", err) + return nil, err + } + err = models.LoadSpecs4CloudbrainInfo(ciTasks) + if err != nil { + log.Error("can not load spec info", err) + return nil, err + } + + for _, task := range ciTasks { + if task.Spec != nil { + if v, ok := xpuCardStatisticMap[task.Spec.AccCardType]; ok { + v.TaskCount += 1 + if calculateCardDuration(task.Cloudbrain) > 0 { + v.UsedDuration += calculateCardDuration(task.Cloudbrain) + } + if _, found := xpuCardStatisticUserMap[task.Spec.AccCardType][task.UserID]; !found { + + v.UserCount += 1 + xpuCardStatisticUserMap[task.Spec.AccCardType][task.UserID] = struct{}{} + } + } + + } + + } + + } + return xpuCardStatisticMap, nil +} + +func calculateCardDuration(task models.Cloudbrain) int64 { + + cardNum := task.Spec.AccCardsNum + + var workServerNumber int64 + if task.WorkServerNumber >= 1 { + workServerNumber = int64(task.WorkServerNumber) + } else { + workServerNumber = 1 + } + return workServerNumber * int64(cardNum) * task.Duration +} diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index ec858379ac..4199b92fc2 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -46,7 +46,7 @@
{{$.i18n.Tr "repo.modelarts.cluster"}}
-
+
{{$.i18n.Tr "repo.modelarts.status"}}
@@ -62,11 +62,11 @@ {{$.i18n.Tr "repo.modelarts.computing_resources"}}
-
+
{{$.i18n.Tr "repo.modelarts.ai_center"}}
-
+
{{$.i18n.Tr "repo.modelarts.card_type"}}
@@ -153,14 +153,16 @@
+ style="width: 8% !important;"> - {{.Status}} + data-cloudbrainid="{{.Cloudbrain.ID}}" data-datamigrate='{{$.i18n.Tr "repo.migratingData"}}' data-centerpend='{{$.i18n.Tr "repo.centerPending"}}'> + + + {{.Status}} + {{if eq .Status "WAITING"}}{{end}} +
@@ -184,11 +186,11 @@ style="font-size: 12px;" title="{{.ComputeResource}}">{{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}}
-
+
{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
-
+
+{{template "base/footer" .}} diff --git a/templates/repo/datasets/dirs/dir_preview.tmpl b/templates/repo/datasets/dirs/dir_preview.tmpl index 04553efb68..50e335bfee 100644 --- a/templates/repo/datasets/dirs/dir_preview.tmpl +++ b/templates/repo/datasets/dirs/dir_preview.tmpl @@ -1,212 +1,97 @@ {{if .Dirs}} - - -