@@ -2,6 +2,7 @@ package repo
import (
"errors"
"sync"
"time"
"code.gitea.io/gitea/models"
@@ -12,8 +13,23 @@ import (
"code.gitea.io/gitea/services/mailer"
"gitea.com/macaron/macaron"
"github.com/Jeffail/tunny"
)
var statisticMutex sync.Mutex
var statisticWg sync.WaitGroup
type StatisticInput struct {
Date string
Repo *models.Repository
T time.Time
ErrorProjects []string
ReposRadar []*models.RepoStatistic
IsInitMinMaxRadar bool
MinRepoRadar models.RepoStatistic
MaxRepoRadar models.RepoStatistic
}
func StatisticAuto() {
go RepoStatisticAuto()
go TimingCountData()
@@ -52,136 +68,196 @@ func RepoStatisticDaily(date string) {
isInitMinMaxRadar := false
var error_projects = make([]string, 0)
for _, repo := range repos {
projectName := getDistinctProjectName(repo)
log.Info("start statistic: %s", projectName)
var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth, numCommits int64
repoGitStat, err := models.GetRepoKPIStats(repo)
if err != nil {
log.Error("GetRepoKPIStats failed: %s", projectName)
} else {
numDevMonths = repoGitStat.DevelopAge
numKeyContributor = repoGitStat.KeyContributors
numWikiViews = repoGitStat.WikiPages
numContributor = repoGitStat.Contributors
numCommitsGrowth = repoGitStat.CommitsAdded
numCommitLinesGrowth = repoGitStat.CommitLinesModified
numContributorsGrowth = repoGitStat.ContributorsAdded
numCommits = repoGitStat.Commits
pool := tunny.NewFunc(4, statisticOneRepo)
defer pool.Close()
statisticWg.Add(len(repos))
for _, repo := range repos {
input := &StatisticInput{
Date: date,
Repo: repo,
T: t,
ErrorProjects: error_projects,
ReposRadar: reposRadar,
IsInitMinMaxRadar: isInitMinMaxRadar,
MinRepoRadar: minRepoRadar,
MaxRepoRadar: maxRepoRadar,
}
pool.Process(input)
}
statisticWg.Wait()
var issueFixedRate float32
if repo.NumIssues != 0 {
issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues)
if len(error_projects) > 0 {
mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
}
//radar map
log.Info("begin statistic radar")
for _, radarInit := range reposRadar {
if radarInit.IsMirror && setting.RadarMap.IgnoreMirrorRepo {
radarInit.Impact = 0
radarInit.Completeness = 0
radarInit.Liveness = 0
radarInit.ProjectHealth = 0
radarInit.TeamHealth = 0
radarInit.Growth = 0
radarInit.RadarTotal = 0
} else {
issueFixedRate = float32(setting.RadarMap.ProjectHealth0IssueCloseRatio)
radarInit.Impact = normalization.Normalization(radarInit.Impact, minRepoRadar.Impact, maxRepoRadar.Impact)
radarInit.Completeness = normalization.Normalization(radarInit.Completeness, minRepoRadar.Completeness, maxRepoRadar.Completeness)
radarInit.Liveness = normalization.Normalization(radarInit.Liveness, minRepoRadar.Liveness, maxRepoRadar.Liveness)
radarInit.ProjectHealth = normalization.Normalization(radarInit.ProjectHealth, minRepoRadar.ProjectHealth, maxRepoRadar.ProjectHealth)
radarInit.TeamHealth = normalization.Normalization(radarInit.TeamHealth, minRepoRadar.TeamHealth, maxRepoRadar.TeamHealth)
radarInit.Growth = normalization.Normalization(radarInit.Growth, minRepoRadar.Growth, maxRepoRadar.Growth)
radarInit.RadarTotal = normalization.GetRadarValue(radarInit.Impact, radarInit.Completeness, radarInit.Liveness, radarInit.ProjectHealth, radarInit.TeamHealth, radarInit.Growth)
}
models.UpdateRepoStat(radarInit)
}
var numVersions int64
numVersions, err = models.GetReleaseCountByRepoID(repo.ID, models.FindReleasesOptions{})
if err != nil {
log.Error("GetReleaseCountByRepoID failed(%s): %v", projectName, err)
}
log.Info("finish statistic: radar")
var datasetSize int64
datasetSize, err = getDatasetSize(repo)
if err != nil {
log.Error("getDatasetSize failed(%s): %v", projectName, err)
}
}
var numComments int64
numComments, err = models.GetCommentCountByRepoID(repo.ID)
if err != nil {
log.Error("GetCommentCountByRepoID failed(%s): %v", projectName, err)
}
func statisticOneRepo(input interface{}) interface{} {
defer statisticWg.Done()
beginTime, endTime := getStatTime(date)
var numVisits int
numVisits, err = repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime)
if err != nil {
log.Error("AppointProjectView failed(%s): %v", projectName, err)
}
statisticInput := input.(*StatisticInput)
repo := statisticInput.Repo
projectName := getDistinctProjectName(repo)
log.Info("start statistic: %s", projectName)
var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth, numCommits int64
repoGitStat, err := models.GetRepoKPIStats(repo)
if err != nil {
log.Error("GetRepoKPIStats failed: %s", projectName)
} else {
numDevMonths = repoGitStat.DevelopAge
numKeyContributor = repoGitStat.KeyContributors
numWikiViews = repoGitStat.WikiPages
numContributor = repoGitStat.Contributors
numCommitsGrowth = repoGitStat.CommitsAdded
numCommitLinesGrowth = repoGitStat.CommitLinesModified
numContributorsGrowth = repoGitStat.ContributorsAdded
numCommits = repoGitStat.Commits
repoStat := models.RepoStatistic{
RepoID: repo.ID,
Date: date,
Name: repo.Name,
Alias: repo.Alias,
IsPrivate: repo.IsPrivate,
IsMirror: repo.IsMirror,
IsFork: repo.IsFork,
RepoCreatedUnix: repo.CreatedUnix,
OwnerName: repo.OwnerName,
NumWatches: int64(repo.NumWatches),
NumStars: int64(repo.NumStars),
NumForks: int64(repo.NumForks),
NumDownloads: repo.CloneCnt,
NumComments: numComments,
NumVisits: int64(numVisits),
NumClosedIssues: int64(repo.NumClosedIssues),
NumVersions: numVersions,
NumDevMonths: numDevMonths,
RepoSize: repo.Size,
DatasetSize: datasetSize,
NumModels: repo.ModelCnt,
NumWikiViews: numWikiViews,
NumCommits: numCommits,
NumIssues: int64(repo.NumIssues),
NumPulls: int64(repo.NumPulls),
IssueFixedRate: issueFixedRate,
NumContributor: numContributor,
NumKeyContributor: numKeyContributor,
NumCommitsGrowth: numCommitsGrowth,
NumCommitLinesGrowth: numCommitLinesGrowth,
NumContributorsGrowth: numContributorsGrowth,
NumCloudbrain: repo.AiTaskCnt,
NumDatasetFile: repo.DatasetCnt,
NumModelConvert: models.QueryModelConvertCountByRepoID(repo.ID),
}
}
dayBeforeDate := t.AddDate(0, 0, -1).Format("2006-01-02")
repoStatisticsBefore, err := models.GetRepoStatisticByDate(dayBeforeDate, repo.ID)
var issueFixedRate float32
if repo.NumIssues != 0 {
issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues)
} else {
issueFixedRate = float32(setting.RadarMap.ProjectHealth0IssueCloseRatio)
}
if err != nil {
log.Error("get data of day before the date failed ", err)
} else {
if len(repoStatisticsBefore) > 0 {
repoStatisticBefore := repoStatisticsBefore[0]
repoStat.NumWatchesAdded = repoStat.NumWatches - repoStatisticBefore.NumWatches
repoStat.NumStarsAdded = repoStat.NumStars - repoStatisticBefore.NumStars
repoStat.NumForksAdded = repoStat.NumForks - repoStatisticBefore.NumForks
repoStat.NumDownloadsAdded = repoStat.NumDownloads - repoStatisticBefore.NumDownloads
repoStat.NumCommentsAdded = repoStat.NumComments - repoStatisticBefore.NumComments
repoStat.NumClosedIssuesAdded = repoStat.NumClosedIssues - repoStatisticBefore.NumClosedIssues
repoStat.NumCommitsAdded = repoStat.NumCommits - repoStatisticBefore.NumCommits
repoStat.NumIssuesAdded = repoStat.NumIssues - repoStatisticBefore.NumIssues
repoStat.NumPullsAdded = repoStat.NumPulls - repoStatisticBefore.NumPulls
repoStat.NumContributorAdded = repoStat.NumContributor - repoStatisticBefore.NumContributor
repoStat.NumModelsAdded = repoStat.NumModels - repoStatisticBefore.NumModels
repoStat.NumCloudbrainAdded = repoStat.NumCloudbrain - repoStatisticBefore.NumCloudbrain
repoStat.NumModelConvertAdded = repoStat.NumModelConvert - repoStatisticBefore.NumModelConvert
repoStat.NumDatasetFileAdded = repoStat.NumDatasetFile - repoStatisticBefore.NumDatasetFile
}
}
day4MonthsAgo := t.AddDate(0, -4, 0)
repoStatisticFourMonthsAgo, err := models.GetOneRepoStatisticBeforeTime(day4MonthsAgo)
if err != nil {
log.Error("Get data of 4 moth ago failed.", err)
} else {
repoStat.NumCommentsGrowth = repoStat.NumComments - repoStatisticFourMonthsAgo.NumComments
repoStat.NumIssuesGrowth = repoStat.NumIssues - repoStatisticFourMonthsAgo.NumIssues
}
var numVersions int64
numVersions, err = models.GetReleaseCountByRepoID(repo.ID, models.FindReleasesOptions{})
if err != nil {
log.Error("GetReleaseCountByRepoID failed(%s): %v", projectName, err)
}
var datasetSize int64
datasetSize, err = getDatasetSize(repo)
if err != nil {
log.Error("getDatasetSize failed(%s): %v", projectName, err)
}
models.SyncStatDataToRepo(repo)
var numComments int64
numComments, err = models.GetCommentCountByRepoID(repo.ID)
if err != nil {
log.Error("GetCommentCountByRepoID failed(%s): %v", projectName, err)
}
if _, err = models.InsertRepoStat(&repoStat); err != nil {
log.Error("InsertRepoStat failed(%s): %v", projectName, err)
log.Error("failed statistic: %s", projectName)
error_projects = append(error_projects, projectName)
beginTime, endTime := getStatTime(statisticInput.Date)
var numVisits int
numVisits, err = repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime)
if err != nil {
log.Error("AppointProjectView failed(%s): %v", projectName, err)
}
continue
repoStat := models.RepoStatistic{
RepoID: repo.ID,
Date: statisticInput.Date,
Name: repo.Name,
Alias: repo.Alias,
IsPrivate: repo.IsPrivate,
IsMirror: repo.IsMirror,
IsFork: repo.IsFork,
RepoCreatedUnix: repo.CreatedUnix,
OwnerName: repo.OwnerName,
NumWatches: int64(repo.NumWatches),
NumStars: int64(repo.NumStars),
NumForks: int64(repo.NumForks),
NumDownloads: repo.CloneCnt,
NumComments: numComments,
NumVisits: int64(numVisits),
NumClosedIssues: int64(repo.NumClosedIssues),
NumVersions: numVersions,
NumDevMonths: numDevMonths,
RepoSize: repo.Size,
DatasetSize: datasetSize,
NumModels: repo.ModelCnt,
NumWikiViews: numWikiViews,
NumCommits: numCommits,
NumIssues: int64(repo.NumIssues),
NumPulls: int64(repo.NumPulls),
IssueFixedRate: issueFixedRate,
NumContributor: numContributor,
NumKeyContributor: numKeyContributor,
NumCommitsGrowth: numCommitsGrowth,
NumCommitLinesGrowth: numCommitLinesGrowth,
NumContributorsGrowth: numContributorsGrowth,
NumCloudbrain: repo.AiTaskCnt,
NumDatasetFile: repo.DatasetCnt,
NumModelConvert: models.QueryModelConvertCountByRepoID(repo.ID),
}
dayBeforeDate := statisticInput.T.AddDate(0, 0, -1).Format("2006-01-02")
repoStatisticsBefore, err := models.GetRepoStatisticByDate(dayBeforeDate, repo.ID)
if err != nil {
log.Error("get data of day before the date failed ", err)
} else {
if len(repoStatisticsBefore) > 0 {
repoStatisticBefore := repoStatisticsBefore[0]
repoStat.NumWatchesAdded = repoStat.NumWatches - repoStatisticBefore.NumWatches
repoStat.NumStarsAdded = repoStat.NumStars - repoStatisticBefore.NumStars
repoStat.NumForksAdded = repoStat.NumForks - repoStatisticBefore.NumForks
repoStat.NumDownloadsAdded = repoStat.NumDownloads - repoStatisticBefore.NumDownloads
repoStat.NumCommentsAdded = repoStat.NumComments - repoStatisticBefore.NumComments
repoStat.NumClosedIssuesAdded = repoStat.NumClosedIssues - repoStatisticBefore.NumClosedIssues
repoStat.NumCommitsAdded = repoStat.NumCommits - repoStatisticBefore.NumCommits
repoStat.NumIssuesAdded = repoStat.NumIssues - repoStatisticBefore.NumIssues
repoStat.NumPullsAdded = repoStat.NumPulls - repoStatisticBefore.NumPulls
repoStat.NumContributorAdded = repoStat.NumContributor - repoStatisticBefore.NumContributor
repoStat.NumModelsAdded = repoStat.NumModels - repoStatisticBefore.NumModels
repoStat.NumCloudbrainAdded = repoStat.NumCloudbrain - repoStatisticBefore.NumCloudbrain
repoStat.NumModelConvertAdded = repoStat.NumModelConvert - repoStatisticBefore.NumModelConvert
repoStat.NumDatasetFileAdded = repoStat.NumDatasetFile - repoStatisticBefore.NumDatasetFile
}
}
day4MonthsAgo := statisticInput.T.AddDate(0, -4, 0)
repoStatisticFourMonthsAgo, err := models.GetOneRepoStatisticBeforeTime(day4MonthsAgo)
if err != nil {
log.Error("Get data of 4 moth ago failed.", err)
} else {
repoStat.NumCommentsGrowth = repoStat.NumComments - repoStatisticFourMonthsAgo.NumComments
repoStat.NumIssuesGrowth = repoStat.NumIssues - repoStatisticFourMonthsAgo.NumIssues
}
models.SyncStatDataToRepo(repo)
if _, err = models.InsertRepoStat(&repoStat); err != nil {
log.Error("InsertRepoStat failed(%s): %v", projectName, err)
log.Error("failed statistic: %s", projectName)
statisticMutex.Lock()
{
statisticInput.ErrorProjects = append(statisticInput.ErrorProjects, projectName)
}
statisticMutex.Unlock()
return nil
}
statisticMutex.Lock()
{
tempRepoStat := models.RepoStatistic{
RepoID: repoStat.RepoID,
@@ -195,102 +271,74 @@ func RepoStatisticDaily(date string) {
Growth: normalization.GetRepoGrowthInitValue(repoStat.NumCommitLinesGrowth, repoStat.NumIssuesGrowth, repoStat.NumCommitsGrowth, repoStat.NumContributorsGrowth, repoStat.NumCommentsGrowth),
}
reposRadar = append(r eposRadar, &tempRepoStat)
statisticInput.ReposRadar = append(statisticInput.R eposRadar, &tempRepoStat)
if !isInitMinMaxRadar {
if !stat isticInput.I sInitMinMaxRadar {
if !setting.RadarMap.IgnoreMirrorRepo || (setting.RadarMap.IgnoreMirrorRepo && !tempRepoStat.IsMirror) {
m inRepoRadar = tempRepoStat
m axRepoRadar = tempRepoStat
isInitMinMaxRadar = true
statisticInput.M inRepoRadar = tempRepoStat
statisticInput.M axRepoRadar = tempRepoStat
stat isticInput.I sInitMinMaxRadar = true
}
} else {
if !setting.RadarMap.IgnoreMirrorRepo || (setting.RadarMap.IgnoreMirrorRepo && !tempRepoStat.IsMirror) {
if tempRepoStat.Impact < m inRepoRadar.Impact {
m inRepoRadar.Impact = tempRepoStat.Impact
if tempRepoStat.Impact < statisticInput.M inRepoRadar.Impact {
statisticInput.M inRepoRadar.Impact = tempRepoStat.Impact
}
if tempRepoStat.Impact > m axRepoRadar.Impact {
m axRepoRadar.Impact = tempRepoStat.Impact
if tempRepoStat.Impact > statisticInput.M axRepoRadar.Impact {
statisticInput.M axRepoRadar.Impact = tempRepoStat.Impact
}
if tempRepoStat.Completeness < m inRepoRadar.Completeness {
m inRepoRadar.Completeness = tempRepoStat.Completeness
if tempRepoStat.Completeness < statisticInput.M inRepoRadar.Completeness {
statisticInput.M inRepoRadar.Completeness = tempRepoStat.Completeness
}
if tempRepoStat.Completeness > m axRepoRadar.Completeness {
m axRepoRadar.Completeness = tempRepoStat.Completeness
if tempRepoStat.Completeness > statisticInput.M axRepoRadar.Completeness {
statisticInput.M axRepoRadar.Completeness = tempRepoStat.Completeness
}
if tempRepoStat.Liveness < m inRepoRadar.Liveness {
m inRepoRadar.Liveness = tempRepoStat.Liveness
if tempRepoStat.Liveness < statisticInput.M inRepoRadar.Liveness {
statisticInput.M inRepoRadar.Liveness = tempRepoStat.Liveness
}
if tempRepoStat.Liveness > m axRepoRadar.Liveness {
m axRepoRadar.Liveness = tempRepoStat.Liveness
if tempRepoStat.Liveness > statisticInput.M axRepoRadar.Liveness {
statisticInput.M axRepoRadar.Liveness = tempRepoStat.Liveness
}
if tempRepoStat.ProjectHealth < m inRepoRadar.ProjectHealth {
m inRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
if tempRepoStat.ProjectHealth < statisticInput.M inRepoRadar.ProjectHealth {
statisticInput.M inRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
}
if tempRepoStat.ProjectHealth > m axRepoRadar.ProjectHealth {
m axRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
if tempRepoStat.ProjectHealth > statisticInput.M axRepoRadar.ProjectHealth {
statisticInput.M axRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth
}
if tempRepoStat.TeamHealth < m inRepoRadar.TeamHealth {
m inRepoRadar.TeamHealth = tempRepoStat.TeamHealth
if tempRepoStat.TeamHealth < statisticInput.M inRepoRadar.TeamHealth {
statisticInput.M inRepoRadar.TeamHealth = tempRepoStat.TeamHealth
}
if tempRepoStat.TeamHealth > m axRepoRadar.TeamHealth {
m axRepoRadar.TeamHealth = tempRepoStat.TeamHealth
if tempRepoStat.TeamHealth > statisticInput.M axRepoRadar.TeamHealth {
statisticInput.M axRepoRadar.TeamHealth = tempRepoStat.TeamHealth
}
if tempRepoStat.Growth < m inRepoRadar.Growth {
m inRepoRadar.Growth = tempRepoStat.Growth
if tempRepoStat.Growth < statisticInput.M inRepoRadar.Growth {
statisticInput.M inRepoRadar.Growth = tempRepoStat.Growth
}
if tempRepoStat.Growth > m axRepoRadar.Growth {
m axRepoRadar.Growth = tempRepoStat.Growth
if tempRepoStat.Growth > statisticInput.M axRepoRadar.Growth {
statisticInput.M axRepoRadar.Growth = tempRepoStat.Growth
}
}
}
log.Info("finish statistic: %s", getDistinctProjectName(repo))
}
statisticMutex.Unlock()
if len(error_projects) > 0 {
mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
}
//radar map
log.Info("begin statistic radar")
for _, radarInit := range reposRadar {
if radarInit.IsMirror && setting.RadarMap.IgnoreMirrorRepo {
radarInit.Impact = 0
radarInit.Completeness = 0
radarInit.Liveness = 0
radarInit.ProjectHealth = 0
radarInit.TeamHealth = 0
radarInit.Growth = 0
radarInit.RadarTotal = 0
} else {
radarInit.Impact = normalization.Normalization(radarInit.Impact, minRepoRadar.Impact, maxRepoRadar.Impact)
radarInit.Completeness = normalization.Normalization(radarInit.Completeness, minRepoRadar.Completeness, maxRepoRadar.Completeness)
radarInit.Liveness = normalization.Normalization(radarInit.Liveness, minRepoRadar.Liveness, maxRepoRadar.Liveness)
radarInit.ProjectHealth = normalization.Normalization(radarInit.ProjectHealth, minRepoRadar.ProjectHealth, maxRepoRadar.ProjectHealth)
radarInit.TeamHealth = normalization.Normalization(radarInit.TeamHealth, minRepoRadar.TeamHealth, maxRepoRadar.TeamHealth)
radarInit.Growth = normalization.Normalization(radarInit.Growth, minRepoRadar.Growth, maxRepoRadar.Growth)
radarInit.RadarTotal = normalization.GetRadarValue(radarInit.Impact, radarInit.Completeness, radarInit.Liveness, radarInit.ProjectHealth, radarInit.TeamHealth, radarInit.Growth)
}
models.UpdateRepoStat(radarInit)
}
log.Info("finish statistic: radar")
log.Info("finish statistic: %s", getDistinctProjectName(repo))
return nil
}
func getDistinctProjectName(repo *models.Repository) string {
