From ee1d64ddd1456764de692fcdeb42db1398fcf97b Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 10 Nov 2019 21:33:47 +0000 Subject: [PATCH] Stop using git count-objects and use raw directory size for repository (#8848) * Migrate from git count-objects to a raw directory size * As per @guillep2k ignore unusual files --- models/migrations/v28.go | 4 ++-- models/repo.go | 7 ++++--- modules/git/repo.go | 4 ++-- modules/util/path.go | 19 ++++++++++++++++++- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/models/migrations/v28.go b/models/migrations/v28.go index 587e944ce6..a849fea3c2 100644 --- a/models/migrations/v28.go +++ b/models/migrations/v28.go @@ -60,9 +60,9 @@ func addRepoSize(x *xorm.Engine) (err error) { } repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(user.Name), strings.ToLower(repo.Name)) + ".git" - countObject, err := git.GetRepoSize(repoPath) + countObject, err := git.CountObjects(repoPath) if err != nil { - log.Warn("GetRepoSize: %v", err) + log.Warn("CountObjects: %v", err) continue } diff --git a/models/repo.go b/models/repo.go index 90918025fb..d79a8fdf61 100644 --- a/models/repo.go +++ b/models/repo.go @@ -36,6 +36,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" "github.com/mcuadros/go-version" "github.com/unknwon/com" @@ -708,17 +709,17 @@ func (repo *Repository) IsOwnedBy(userID int64) bool { } func (repo *Repository) updateSize(e Engine) error { - repoInfoSize, err := git.GetRepoSize(repo.repoPath(e)) + size, err := util.GetDirectorySize(repo.repoPath(e)) if err != nil { return fmt.Errorf("UpdateSize: %v", err) } - repo.Size = repoInfoSize.Size + repoInfoSize.SizePack + repo.Size = size _, err = e.ID(repo.ID).Cols("size").Update(repo) return err } -// UpdateSize updates the repository size, calculating it using git.GetRepoSize +// UpdateSize updates the repository size, calculating it using util.GetDirectorySize func (repo *Repository) UpdateSize() error { return repo.updateSize(x) } diff --git a/modules/git/repo.go b/modules/git/repo.go index 4c6690b913..80f6109772 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -304,8 +304,8 @@ const ( statSizeGarbage = "size-garbage: " ) -// GetRepoSize returns disk consumption for repo in path -func GetRepoSize(repoPath string) (*CountObject, error) { +// CountObjects returns the results of git count-objects on the repoPath +func CountObjects(repoPath string) (*CountObject, error) { cmd := NewCommand("count-objects", "-v") stdout, err := cmd.RunInDir(repoPath) if err != nil { diff --git a/modules/util/path.go b/modules/util/path.go index f79334209c..2b198eb6dc 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -4,7 +4,10 @@ package util -import "path/filepath" +import ( + "os" + "path/filepath" +) // EnsureAbsolutePath ensure that a path is absolute, making it // relative to absoluteBase if necessary @@ -14,3 +17,17 @@ func EnsureAbsolutePath(path string, absoluteBase string) string { } return filepath.Join(absoluteBase, path) } + +const notRegularFileMode os.FileMode = os.ModeDir | os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular + +// GetDirectorySize returns the dumb disk consumption for a given path +func GetDirectorySize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if info != nil && (info.Mode()¬RegularFileMode) == 0 { + size += info.Size() + } + return err + }) + return size, err +}