enhancement: automatic cleanup of old repos/branches

internal/services/config/config.go

@@ -162,6 +162,13 @@ type Gitserver struct {
 	Web           Web           `yaml:"web"`
 	Etcd          Etcd          `yaml:"etcd"`
 	ObjectStorage ObjectStorage `yaml:"objectStorage"`
+
+	RepoCleaner RepoCleaner `yaml:"repoCleaner"`
+}
+
+type RepoCleaner struct {
+	ScanTime        string `yaml:"scanTime"`
+	DeleteBranchAge string `yaml:"deleteBranchAge"`
 }
 
 type Web struct {

internal/services/gitserver/main.go

@@ -128,6 +128,7 @@ func Matcher(matchRegexp *regexp.Regexp) mux.MatcherFunc {
 
 type Gitserver struct {
 	c *config.Gitserver
+	r *RepoCleaner
 }
 
 func NewGitserver(ctx context.Context, l *zap.Logger, c *config.Gitserver) (*Gitserver, error) {
@@ -139,8 +140,11 @@ func NewGitserver(ctx context.Context, l *zap.Logger, c *config.Gitserver) (*Git
 	}
 	log = logger.Sugar()
 
+	repoCleaner := NewRepoCleaner(c, log)
+
 	return &Gitserver{
 		c: c,
+		r: repoCleaner,
 	}, nil
 }
 
@@ -179,6 +183,8 @@ func (s *Gitserver) Run(ctx context.Context) error {
 		}
 	}()
 
+	s.r.Run(ctx)
+
 	select {
 	case <-ctx.Done():
 		log.Infof("gitserver exiting")

internal/services/gitserver/repo-cleaner.go

@@ -0,0 +1,196 @@
+package gitserver
+
+import (
+	"context"
+	"io/ioutil"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"agola.io/agola/internal/services/config"
+	"agola.io/agola/internal/util"
+	"go.uber.org/zap"
+)
+
+type RepoCleaner struct {
+	dataDir          string
+	logger           *zap.SugaredLogger
+	scanTime         time.Duration
+	deleteBranchTime time.Duration
+}
+
+const (
+	DEFAULT_REPO_DELETE_BRANCH string = "720h"
+	DEFAULT_SCAN_TIME          string = "24h"
+)
+
+func NewRepoCleaner(c *config.Gitserver, l *zap.SugaredLogger) *RepoCleaner {
+	stDuration, err := time.ParseDuration(c.RepoCleaner.ScanTime)
+	if err != nil {
+		l.Error("repo-cleaner error:", err)
+		stDuration, _ = time.ParseDuration(DEFAULT_SCAN_TIME)
+	}
+
+	btDuration, err := time.ParseDuration(c.RepoCleaner.DeleteBranchAge)
+	if err != nil {
+		l.Error("repo-cleaner error:", err)
+		btDuration, _ = time.ParseDuration(DEFAULT_REPO_DELETE_BRANCH)
+	}
+
+	return &RepoCleaner{
+		dataDir:          c.DataDir,
+		logger:           l,
+		scanTime:         stDuration,
+		deleteBranchTime: btDuration,
+	}
+}
+
+func (s *RepoCleaner) Run(ctx context.Context) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				s.logger.Infof("repoCleaner exiting")
+
+				return
+
+			case <-time.After(s.scanTime):
+				s.scanRepos(ctx)
+			}
+		}
+	}()
+}
+
+func (s *RepoCleaner) scanRepos(ctx context.Context) error {
+	s.logger.Info("repoCleaner scanRepos start")
+
+	usersDir, err := ioutil.ReadDir(s.dataDir)
+	if err != nil {
+		return err
+	}
+
+	for _, u := range usersDir {
+		if u.IsDir() {
+			reposDir, _ := ioutil.ReadDir(s.dataDir + string(filepath.Separator) + u.Name())
+			for _, r := range reposDir {
+				if r.IsDir() {
+					s.scanRepo(ctx, s.dataDir+string(filepath.Separator)+u.Name()+string(filepath.Separator)+r.Name())
+				}
+			}
+		}
+	}
+
+	s.logger.Info("repoCleaner scanRepos end")
+
+	return nil
+}
+
+func (s *RepoCleaner) scanRepo(ctx context.Context, repoDir string) {
+	git := &util.Git{GitDir: repoDir}
+
+	branches, _ := s.getBranches(git, ctx)
+	deletedCount := 0
+	for _, b := range *branches {
+		date, _ := s.getLastCommitTime(ctx, git, "refs/heads/"+b)
+
+		if time.Since(*date) >= s.deleteBranchTime {
+			err := s.deleteBranch(ctx, git, b)
+			if err != nil {
+				s.logger.Error("repoCleaner error:", err)
+			} else {
+				deletedCount++
+			}
+		}
+	}
+
+	tags, _ := s.getTags(git, ctx)
+	for _, tag := range *tags {
+		date, _ := s.getLastCommitTime(ctx, git, "refs/tags/"+tag)
+
+		if time.Since(*date) >= s.deleteBranchTime {
+			err := s.deleteTag(ctx, git, tag)
+			if err != nil {
+				s.logger.Error("repoCleaner error:", err)
+			} else {
+				deletedCount++
+			}
+		}
+	}
+
+	if len(*branches)+len(*tags) == deletedCount {
+		s.logger.Info("delete repo:", repoDir)
+		err := s.deleteRepo(ctx, repoDir)
+		if err != nil {
+			log.Error("repoCleaner error:", err)
+		}
+	}
+}
+
+func (s *RepoCleaner) getBranches(git *util.Git, ctx context.Context) (*[]string, error) {
+	output, err := git.OutputLines(ctx, nil, "branch")
+	if err != nil {
+		return nil, err
+	}
+
+	branches := make([]string, 0)
+	for _, l := range output {
+		splitSpace := strings.Split(l, " ")
+		for _, s := range splitSpace {
+			if len(s) != 0 && !strings.Contains(s, "*") {
+				branches = append(branches, s)
+			}
+		}
+	}
+
+	return &branches, nil
+}
+
+func (s *RepoCleaner) getTags(git *util.Git, ctx context.Context) (*[]string, error) {
+	output, err := git.Output(ctx, nil, "tag")
+	if err != nil {
+		return nil, err
+	}
+
+	splitNewLine := strings.Split(string(output), "\n")
+	branches := make([]string, 0)
+	for _, l := range splitNewLine {
+		splitSpace := strings.Split(l, " ")
+		for _, s := range splitSpace {
+			if len(s) != 0 {
+				branches = append(branches, s)
+			}
+		}
+	}
+
+	return &branches, nil
+}
+
+func (s *RepoCleaner) getLastCommitTime(ctx context.Context, git *util.Git, ref string) (*time.Time, error) {
+	output, err := git.Output(ctx, nil, "log", "-1", "--format=%cd", ref)
+	if err != nil {
+		return nil, err
+	}
+
+	date, err := time.Parse("Mon Jan 2 15:04:05 2006 -0700", strings.TrimSuffix(string(output), "\n"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &date, nil
+}
+
+func (s *RepoCleaner) deleteBranch(ctx context.Context, git *util.Git, branch string) error {
+	_, err := git.Output(ctx, nil, "branch", "-D", branch)
+	return err
+}
+
+func (s *RepoCleaner) deleteTag(ctx context.Context, git *util.Git, tag string) error {
+	_, err := git.Output(ctx, nil, "tag", "-d", tag)
+	return err
+}
+
+func (s *RepoCleaner) deleteRepo(ctx context.Context, repoDir string) error {
+	cmd := exec.CommandContext(ctx, "rm", repoDir, "-R")
+	return cmd.Run()
+}