From 7315019e91a4018dc7f4faff480371e323f73a4b Mon Sep 17 00:00:00 2001 From: "alessandro.pinna" Date: Mon, 22 Nov 2021 14:11:26 +0100 Subject: [PATCH] enhancement: automatic cleanup of old repos/branches --- internal/services/config/config.go | 7 + internal/services/gitserver/main.go | 6 + internal/services/gitserver/repo-cleaner.go | 196 ++++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 internal/services/gitserver/repo-cleaner.go diff --git a/internal/services/config/config.go b/internal/services/config/config.go index 8308ca142..ad86bfd4a 100644 --- a/internal/services/config/config.go +++ b/internal/services/config/config.go @@ -162,6 +162,13 @@ type Gitserver struct { Web Web `yaml:"web"` Etcd Etcd `yaml:"etcd"` ObjectStorage ObjectStorage `yaml:"objectStorage"` + + RepoCleaner RepoCleaner `yaml:"repoCleaner"` +} + +type RepoCleaner struct { + ScanTime string `yaml:"scanTime"` + DeleteBranchAge string `yaml:"deleteBranchAge"` } type Web struct { diff --git a/internal/services/gitserver/main.go b/internal/services/gitserver/main.go index 03d6ce895..c27222f84 100644 --- a/internal/services/gitserver/main.go +++ b/internal/services/gitserver/main.go @@ -128,6 +128,7 @@ func Matcher(matchRegexp *regexp.Regexp) mux.MatcherFunc { type Gitserver struct { c *config.Gitserver + r *RepoCleaner } func NewGitserver(ctx context.Context, l *zap.Logger, c *config.Gitserver) (*Gitserver, error) { @@ -139,8 +140,11 @@ func NewGitserver(ctx context.Context, l *zap.Logger, c *config.Gitserver) (*Git } log = logger.Sugar() + repoCleaner := NewRepoCleaner(c, log) + return &Gitserver{ c: c, + r: repoCleaner, }, nil } @@ -179,6 +183,8 @@ func (s *Gitserver) Run(ctx context.Context) error { } }() + s.r.Run(ctx) + select { case <-ctx.Done(): log.Infof("gitserver exiting") diff --git a/internal/services/gitserver/repo-cleaner.go b/internal/services/gitserver/repo-cleaner.go new file mode 100644 index 000000000..fa68ec3d6 --- /dev/null +++ b/internal/services/gitserver/repo-cleaner.go @@ -0,0 +1,196 @@ +package gitserver + +import ( + "context" + "io/ioutil" + "os/exec" + "path/filepath" + "strings" + "time" + + "agola.io/agola/internal/services/config" + "agola.io/agola/internal/util" + "go.uber.org/zap" +) + +type RepoCleaner struct { + dataDir string + logger *zap.SugaredLogger + scanTime time.Duration + deleteBranchTime time.Duration +} + +const ( + DEFAULT_REPO_DELETE_BRANCH string = "720h" + DEFAULT_SCAN_TIME string = "24h" +) + +func NewRepoCleaner(c *config.Gitserver, l *zap.SugaredLogger) *RepoCleaner { + stDuration, err := time.ParseDuration(c.RepoCleaner.ScanTime) + if err != nil { + l.Error("repo-cleaner error:", err) + stDuration, _ = time.ParseDuration(DEFAULT_SCAN_TIME) + } + + btDuration, err := time.ParseDuration(c.RepoCleaner.DeleteBranchAge) + if err != nil { + l.Error("repo-cleaner error:", err) + btDuration, _ = time.ParseDuration(DEFAULT_REPO_DELETE_BRANCH) + } + + return &RepoCleaner{ + dataDir: c.DataDir, + logger: l, + scanTime: stDuration, + deleteBranchTime: btDuration, + } +} + +func (s *RepoCleaner) Run(ctx context.Context) { + go func() { + for { + select { + case <-ctx.Done(): + s.logger.Infof("repoCleaner exiting") + + return + + case <-time.After(s.scanTime): + s.scanRepos(ctx) + } + } + }() +} + +func (s *RepoCleaner) scanRepos(ctx context.Context) error { + s.logger.Info("repoCleaner scanRepos start") + + usersDir, err := ioutil.ReadDir(s.dataDir) + if err != nil { + return err + } + + for _, u := range usersDir { + if u.IsDir() { + reposDir, _ := ioutil.ReadDir(s.dataDir + string(filepath.Separator) + u.Name()) + for _, r := range reposDir { + if r.IsDir() { + s.scanRepo(ctx, s.dataDir+string(filepath.Separator)+u.Name()+string(filepath.Separator)+r.Name()) + } + } + } + } + + s.logger.Info("repoCleaner scanRepos end") + + return nil +} + +func (s *RepoCleaner) scanRepo(ctx context.Context, repoDir string) { + git := &util.Git{GitDir: repoDir} + + branches, _ := s.getBranches(git, ctx) + deletedCount := 0 + for _, b := range *branches { + date, _ := s.getLastCommitTime(ctx, git, "refs/heads/"+b) + + if time.Since(*date) >= s.deleteBranchTime { + err := s.deleteBranch(ctx, git, b) + if err != nil { + s.logger.Error("repoCleaner error:", err) + } else { + deletedCount++ + } + } + } + + tags, _ := s.getTags(git, ctx) + for _, tag := range *tags { + date, _ := s.getLastCommitTime(ctx, git, "refs/tags/"+tag) + + if time.Since(*date) >= s.deleteBranchTime { + err := s.deleteTag(ctx, git, tag) + if err != nil { + s.logger.Error("repoCleaner error:", err) + } else { + deletedCount++ + } + } + } + + if len(*branches)+len(*tags) == deletedCount { + s.logger.Info("delete repo:", repoDir) + err := s.deleteRepo(ctx, repoDir) + if err != nil { + log.Error("repoCleaner error:", err) + } + } +} + +func (s *RepoCleaner) getBranches(git *util.Git, ctx context.Context) (*[]string, error) { + output, err := git.OutputLines(ctx, nil, "branch") + if err != nil { + return nil, err + } + + branches := make([]string, 0) + for _, l := range output { + splitSpace := strings.Split(l, " ") + for _, s := range splitSpace { + if len(s) != 0 && !strings.Contains(s, "*") { + branches = append(branches, s) + } + } + } + + return &branches, nil +} + +func (s *RepoCleaner) getTags(git *util.Git, ctx context.Context) (*[]string, error) { + output, err := git.Output(ctx, nil, "tag") + if err != nil { + return nil, err + } + + splitNewLine := strings.Split(string(output), "\n") + branches := make([]string, 0) + for _, l := range splitNewLine { + splitSpace := strings.Split(l, " ") + for _, s := range splitSpace { + if len(s) != 0 { + branches = append(branches, s) + } + } + } + + return &branches, nil +} + +func (s *RepoCleaner) getLastCommitTime(ctx context.Context, git *util.Git, ref string) (*time.Time, error) { + output, err := git.Output(ctx, nil, "log", "-1", "--format=%cd", ref) + if err != nil { + return nil, err + } + + date, err := time.Parse("Mon Jan 2 15:04:05 2006 -0700", strings.TrimSuffix(string(output), "\n")) + if err != nil { + return nil, err + } + + return &date, nil +} + +func (s *RepoCleaner) deleteBranch(ctx context.Context, git *util.Git, branch string) error { + _, err := git.Output(ctx, nil, "branch", "-D", branch) + return err +} + +func (s *RepoCleaner) deleteTag(ctx context.Context, git *util.Git, tag string) error { + _, err := git.Output(ctx, nil, "tag", "-d", tag) + return err +} + +func (s *RepoCleaner) deleteRepo(ctx context.Context, repoDir string) error { + cmd := exec.CommandContext(ctx, "rm", repoDir, "-R") + return cmd.Run() +}