package forge import ( "context" "crypto/aes" "crypto/cipher" "crypto/rand" "crypto/sha256" "encoding/hex" "errors" "fmt" "html/template" "io" "log" "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "sync" "time" ) const mirrorSchema = ` create table if not exists mirrors ( id integer primary key, repo integer not null references repos(id) on delete cascade, kind text not null, url text not null, login text not null default '', secret blob, interval integer not null default 0, next integer not null default 0, synced integer not null default 0, error text not null default '', state text not null default '', created integer not null ); create unique index if not exists mirrors_pull on mirrors(repo) where kind = 'pull'; create index if not exists mirrors_repo on mirrors(repo); ` const ( maxPushMirrors = 5 pushRetry = 10 * time.Minute pushResync = 24 * time.Hour defaultInterval = 8 * time.Hour mirrorTick = 30 * time.Second mirrorTimeout = 15 * time.Minute ) var mirrorIntervals = []time.Duration{time.Hour, 4 * time.Hour, 8 * time.Hour, 24 * time.Hour} type Mirror struct { ID int64 Repo int64 Kind string URL string Login string secret []byte Interval time.Duration Next time.Time Synced time.Time Error string State string } func (m Mirror) HasToken() bool { return len(m.secret) > 0 } func (m Mirror) Source() string { u, err := url.Parse(m.URL) if err != nil { return "" } return u.Redacted() } func (m Mirror) Hours() int { return int(m.Interval / time.Hour) } func (m Mirror) Never() bool { return m.Synced.Unix() == 0 } const mirrorColumns = `id, repo, kind, url, login, secret, interval, next, synced, error, state` func scanMirror(scan func(...any) error) (Mirror, error) { var m Mirror var interval, next, synced int64 err := scan(&m.ID, &m.Repo, &m.Kind, &m.URL, &m.Login, &m.secret, &interval, &next, &synced, &m.Error, &m.State) m.Interval = time.Duration(interval) * time.Second m.Next, m.Synced = time.Unix(next, 0), time.Unix(synced, 0) return m, err } func (s *Store) Mirrors(repo int64) []Mirror { rows, err := s.db.Query(`select `+mirrorColumns+` from mirrors where repo = ? order by kind, id`, repo) if err != nil { return nil } defer rows.Close() var out []Mirror for rows.Next() { if m, err := scanMirror(rows.Scan); err == nil { out = append(out, m) } } return out } func (s *Store) allMirrors() []Mirror { rows, err := s.db.Query(`select ` + mirrorColumns + ` from mirrors`) if err != nil { return nil } defer rows.Close() var out []Mirror for rows.Next() { if m, err := scanMirror(rows.Scan); err == nil { out = append(out, m) } } return out } func (s *Store) PullMirror(repo int64) (Mirror, bool) { m, err := scanMirror(s.db.QueryRow(`select `+mirrorColumns+` from mirrors where repo = ? and kind = 'pull'`, repo).Scan) return m, err == nil } func (s *Store) IsPullMirror(repo int64) bool { _, ok := s.PullMirror(repo) return ok } func (s *Store) AddMirror(repo int64, kind, rawURL, login string, secret []byte, interval time.Duration) error { if kind == "push" { var n int s.db.QueryRow(`select count(*) from mirrors where repo = ? and kind = 'push'`, repo).Scan(&n) if n >= maxPushMirrors { return fmt.Errorf("A repository can have at most %d push mirrors", maxPushMirrors) } } next := time.Now().Add(interval).Unix() if kind == "push" { next = 0 } _, err := s.db.Exec(`insert into mirrors(repo, kind, url, login, secret, interval, next, created) values (?, ?, ?, ?, ?, ?, ?, ?)`, repo, kind, rawURL, login, secret, int64(interval/time.Second), next, time.Now().Unix()) if err != nil && strings.Contains(err.Error(), "UNIQUE") { return errors.New("This repository already mirrors another one") } return err } func (s *Store) DeleteMirror(repo, id int64) { s.db.Exec(`delete from mirrors where id = ? and repo = ?`, id, repo) } func (s *Store) SyncSoon(repo, id int64) { s.db.Exec(`update mirrors set next = 0 where id = ? and repo = ?`, id, repo) } func (s *Store) SetInterval(repo int64, interval time.Duration) { s.db.Exec(`update mirrors set interval = ?, next = min(next, ?) where repo = ? and kind = 'pull'`, int64(interval/time.Second), time.Now().Add(interval).Unix(), repo) } func (s *Store) MirrorDone(id int64, next time.Time, state string, err error) { msg := "" if err != nil { msg = err.Error() if len(msg) > 300 { msg = msg[:300] } } if err == nil { s.db.Exec(`update mirrors set next = ?, synced = ?, error = '', state = ? where id = ?`, next.Unix(), time.Now().Unix(), state, id) return } s.db.Exec(`update mirrors set next = ?, error = ? where id = ?`, next.Unix(), msg, id) } func mirrorKey(root string) ([]byte, error) { keyPath := filepath.Join(root, "mirror.key") if b, err := os.ReadFile(keyPath); err == nil && len(b) == 32 { return b, nil } key := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, key); err != nil { return nil, err } f, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { if b, rerr := os.ReadFile(keyPath); rerr == nil && len(b) == 32 { return b, nil } return nil, err } defer f.Close() if _, err := f.Write(key); err != nil { return nil, err } return key, nil } func seal(root, plain string) ([]byte, error) { if plain == "" { return nil, nil } key, err := mirrorKey(root) if err != nil { return nil, err } block, err := aes.NewCipher(key) if err != nil { return nil, err } gcm, err := cipher.NewGCM(block) if err != nil { return nil, err } nonce := make([]byte, gcm.NonceSize()) if _, err := io.ReadFull(rand.Reader, nonce); err != nil { return nil, err } return gcm.Seal(nonce, nonce, []byte(plain), nil), nil } func unseal(root string, box []byte) (string, error) { if len(box) == 0 { return "", nil } key, err := mirrorKey(root) if err != nil { return "", err } block, err := aes.NewCipher(key) if err != nil { return "", err } gcm, err := cipher.NewGCM(block) if err != nil { return "", err } if len(box) < gcm.NonceSize() { return "", errors.New("stored credential is corrupt") } plain, err := gcm.Open(nil, box[:gcm.NonceSize()], box[gcm.NonceSize():], nil) if err != nil { return "", errors.New("stored credential cannot be decrypted") } return string(plain), nil } func (g Git) refPrint() string { out, err := g.run(nil, "for-each-ref", "--format=%(objectname) %(refname)", "refs/heads", "refs/tags") if err != nil { return "" } sum := sha256.Sum256(out) return hex.EncodeToString(sum[:]) } var mirrorRuns = struct { sync.Mutex m map[int64]bool }{m: map[int64]bool{}} var mirrorSlots = make(chan struct{}, 2) func (s *Server) mirrorLoop() { for { s.mirrorTick() time.Sleep(mirrorTick) } } func (s *Server) mirrorTick() { now := time.Now() for _, m := range s.store.allMirrors() { due := !m.Next.After(now) if m.Kind == "push" && !due && m.Error == "" { repo, err := s.store.RepoByID(m.Repo) due = err == nil && (Git{Dir: s.repoDir(repo)}).refPrint() != m.State } if !due { continue } mirrorRuns.Lock() busy := mirrorRuns.m[m.ID] mirrorRuns.m[m.ID] = true mirrorRuns.Unlock() if busy { continue } go func(m Mirror) { mirrorSlots <- struct{}{} defer func() { <-mirrorSlots mirrorRuns.Lock() delete(mirrorRuns.m, m.ID) mirrorRuns.Unlock() }() s.runMirror(m) }(m) } } func (s *Server) runMirror(m Mirror) { repo, err := s.store.RepoByID(m.Repo) if err != nil { return } g := Git{Dir: s.repoDir(repo)} if m.Kind == "pull" { err = s.pullMirror(m, repo) interval := m.Interval if interval < time.Hour { interval = defaultInterval } s.store.MirrorDone(m.ID, time.Now().Add(interval), "", err) } else { state := g.refPrint() err = s.pushMirror(m, repo) next := time.Now().Add(pushResync) if err != nil { next = time.Now().Add(pushRetry) } s.store.MirrorDone(m.ID, next, state, err) } if err != nil { log.Printf("mirror %s %s: %v", m.Kind, repo.Path(), err) } } func (s *Server) pullMirror(m Mirror, repo Repo) error { if over, used, quota := overQuota(s.root, repo.Owner.Name); over { return errors.New(quotaMessage(used, quota)) } src, pin, err := importSource(m.URL) if err != nil { return err } token, err := unseal(s.root, m.secret) if err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), mirrorTimeout) defer cancel() dir := s.repoDir(repo) head, err := remoteAuth(ctx, dir, pin, m.Login, token, "ls-remote", "--symref", src.String(), "HEAD") if err != nil { return err } if _, err := remoteAuth(ctx, dir, pin, m.Login, token, "fetch", "--quiet", "--prune", "--no-tags", src.String(), "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"); err != nil { return err } for line := range strings.Lines(string(head)) { if ref, ok := strings.CutPrefix(line, "ref: refs/heads/"); ok { branch, _, _ := strings.Cut(ref, "\t") if validBranch(branch) { Git{Dir: dir}.run(nil, "symbolic-ref", "HEAD", "refs/heads/"+branch) } } } settleRefs(dir) if over, used, quota := overQuota(s.root, repo.Owner.Name); over { return errors.New(quotaMessage(used, quota)) } return nil } func (s *Server) pushMirror(m Mirror, repo Repo) error { dst, pin, err := importSource(m.URL) if err != nil { return err } token, err := unseal(s.root, m.secret) if err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), mirrorTimeout) defer cancel() _, err = remoteAuth(ctx, s.repoDir(repo), pin, m.Login, token, "push", "--force", "--prune", "--porcelain", dst.String(), "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*") return err } func parseInterval(raw string) time.Duration { n, err := strconv.Atoi(raw) if err != nil { return defaultInterval } for _, d := range mirrorIntervals { if time.Duration(n)*time.Hour == d { return d } } return defaultInterval } func (s *Server) mirrors(w http.ResponseWriter, r *http.Request, repo Repo) { rp, _, _ := s.repoPage(r, repo, "") if !rp.IsOwner { s.fail(w, r, 403, "Not your repository") return } back := "/" + repo.Path() + "/settings#mirrors" fail := func(msg string) { http.Redirect(w, r, "/"+repo.Path()+"/settings?error="+template.URLQueryEscaper(msg)+"#mirrors", http.StatusSeeOther) } id, _ := strconv.ParseInt(r.FormValue("id"), 10, 64) switch r.FormValue("action") { case "add": if s.uploads.blocked("mirror:" + rp.User.Name) { fail("Too many mirror changes, try again later") return } dst, _, err := importSource(r.FormValue("url")) if err != nil { fail(err.Error()) return } login := strings.TrimSpace(r.FormValue("login")) if login == "" { login = "oauth2" } if !validLogin(login) { fail("That username is not valid") return } box, err := seal(s.root, strings.TrimSpace(r.FormValue("token"))) if err != nil { fail("Could not store the token") return } if err := s.store.AddMirror(repo.ID, "push", dst.String(), login, box, 0); err != nil { fail(err.Error()) return } s.uploads.fail("mirror:" + rp.User.Name) case "delete": s.store.DeleteMirror(repo.ID, id) case "sync": if s.uploads.blocked("mirror:" + rp.User.Name) { fail("Too many mirror changes, try again later") return } s.uploads.fail("mirror:" + rp.User.Name) s.store.SyncSoon(repo.ID, id) case "interval": s.store.SetInterval(repo.ID, parseInterval(r.FormValue("hours"))) default: fail("Unknown action") return } http.Redirect(w, r, back, http.StatusSeeOther) } func validLogin(login string) bool { if len(login) > 100 { return false } for _, c := range login { if c <= 0x20 || c == 0x7f || c == ':' || c == '@' || c == '/' { return false } } return true }