package forge import ( "bufio" "bytes" "context" "encoding/base64" "errors" "fmt" "html/template" "io" "log" "net" "net/http" "net/url" "os" "os/exec" "path" "slices" "strconv" "strings" "sync" "time" ) type importState struct { Source string Done bool } var imports = struct { sync.Mutex m map[int64]*importState }{m: map[int64]*importState{}} var importSlots = make(chan struct{}, 2) var failedImports = struct { sync.Mutex m map[string]string }{m: map[string]string{}} func failedImport(path string) string { failedImports.Lock() defer failedImports.Unlock() return failedImports.m[path] } func importStatus(repo int64) (importState, bool) { imports.Lock() defer imports.Unlock() st, ok := imports.m[repo] if !ok { return importState{}, false } return *st, true } func setImport(repo int64, f func(*importState)) { imports.Lock() defer imports.Unlock() if st, ok := imports.m[repo]; ok { f(st) } } func publicAddr(ip net.IP) bool { if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() || ip.IsInterfaceLocalMulticast() { return false } _, cgnat, _ := net.ParseCIDR("100.64.0.0/10") return !cgnat.Contains(ip) } func importSource(raw string) (*url.URL, string, error) { u, err := url.Parse(strings.TrimSpace(raw)) if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { return nil, "", errors.New("Use an https clone URL such as https://github.com/owner/repo") } port := u.Port() if port == "" { port = "443" } ips, err := net.LookupIP(u.Hostname()) if err != nil || len(ips) == 0 { return nil, "", errors.New("Could not resolve " + u.Hostname()) } for _, ip := range ips { if !publicAddr(ip) { return nil, "", errors.New("That host is not reachable from this server") } } ip := ips[0].String() if strings.Contains(ip, ":") { ip = "[" + ip + "]" } return u, u.Hostname() + ":" + port + ":" + ip, nil } func importName(u *url.URL) string { name := strings.ToLower(strings.TrimSuffix(path.Base(strings.TrimSuffix(u.Path, "/")), ".git")) if validName(name) && !reservedRepo(name) { return name } return "" } func remoteGit(ctx context.Context, dir, pin, token string, args ...string) ([]byte, error) { cfg := [][2]string{{"http.curloptResolve", pin}, {"http.followRedirects", "false"}, {"protocol.allow", "never"}, {"protocol.https.allow", "always"}, {"credential.helper", ""}} if token != "" { cfg = append(cfg, [2]string{"http.extraHeader", "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte("oauth2:"+token))}) } env := []string{"GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", "HOME=/nonexistent", "LC_ALL=C.UTF-8", "PATH=" + os.Getenv("PATH"), "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/false", "GIT_ALLOW_PROTOCOL=https", "GIT_CONFIG_COUNT=" + strconv.Itoa(len(cfg))} for i, kv := range cfg { env = append(env, fmt.Sprintf("GIT_CONFIG_KEY_%d=%s", i, kv[0]), fmt.Sprintf("GIT_CONFIG_VALUE_%d=%s", i, kv[1])) } cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) cmd.Env = env var out, errb bytes.Buffer cmd.Stdout, cmd.Stderr = &out, &errb if err := cmd.Run(); err != nil { msg := strings.TrimSpace(errb.String()) if token != "" { msg = strings.ReplaceAll(msg, token, "…") } if i := strings.LastIndex(msg, "\n"); i >= 0 { msg = msg[i+1:] } if msg == "" { msg = err.Error() } return nil, errors.New(msg) } return out.Bytes(), nil } func (s *Server) importForm(w http.ResponseWriter, r *http.Request) { u := s.require(w, r) if u == nil { return } s.render(w, r, "import", page{Title: "Import repository", User: u, Data: map[string]any{"Orgs": s.store.Orgs(u.ID), "Owner": r.URL.Query().Get("owner")}}) } func (s *Server) importRepo(w http.ResponseWriter, r *http.Request) { u := s.require(w, r) if u == nil { return } form := map[string]any{"Orgs": s.store.Orgs(u.ID), "Owner": r.FormValue("owner"), "Source": r.FormValue("source"), "Name": r.FormValue("name"), "Description": r.FormValue("description")} fail := func(msg string) { w.WriteHeader(400) s.render(w, r, "import", page{Title: "Import repository", User: u, Error: msg, Data: form}) } owner := *u if o := r.FormValue("owner"); o != "" && o != u.Name { org, err := s.store.UserByName(o) if err != nil || !s.store.IsOrg(org.ID) || !s.store.IsMember(org.ID, u.ID) { fail("You are not a member of that organization") return } owner = org } src, pin, err := importSource(r.FormValue("source")) if err != nil { fail(err.Error()) return } name := strings.TrimSpace(r.FormValue("name")) if name == "" { name = importName(src) } if s.uploads.blocked("import:" + u.Name) { fail("Too many imports, try again later") return } if over, used, quota := overQuota(s.root, owner.Name); over { fail(quotaMessage(used, quota)) return } repo, err := s.store.CreateRepo(owner, name, strings.TrimSpace(r.FormValue("description")), visibility(r.FormValue("visibility")), "") if err != nil { fail(err.Error()) return } dir := s.repoDir(repo) if err := os.MkdirAll(path.Dir(dir), 0o755); err == nil { err = initRepo(dir) } if err != nil { os.RemoveAll(dir) s.store.DeleteRepo(repo.ID) fail("Could not create the repository") return } installHook(s.root, dir) s.uploads.fail("import:" + u.Name) imports.Lock() imports.m[repo.ID] = &importState{Source: src.Redacted()} imports.Unlock() go s.runImport(repo, src.String(), pin, strings.TrimSpace(r.FormValue("token"))) http.Redirect(w, r, "/"+repo.Path()+"/import", http.StatusSeeOther) } func (s *Server) runImport(repo Repo, src, pin, token string) { importSlots <- struct{}{} defer func() { <-importSlots }() ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() dir := s.repoDir(repo) err := func() error { head, err := remoteGit(ctx, dir, pin, token, "ls-remote", "--symref", src, "HEAD") if err != nil { return err } if _, err := remoteGit(ctx, dir, pin, token, "fetch", "--quiet", "--no-tags", src, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"); err != nil { return err } for line := range strings.Lines(string(head)) { if ref, ok := strings.CutPrefix(line, "ref: refs/heads/"); ok { branch, _, _ := strings.Cut(ref, "\t") Git{Dir: dir}.run(nil, "symbolic-ref", "HEAD", "refs/heads/"+branch) } } if over, used, quota := overQuota(s.root, repo.Owner.Name); over { return errors.New(quotaMessage(used, quota)) } return nil }() if err != nil { log.Printf("import %s: %v", repo.Path(), err) os.RemoveAll(dir) s.store.DeleteRepo(repo.ID) imports.Lock() delete(imports.m, repo.ID) imports.Unlock() failedImports.Lock() failedImports.m[repo.Path()] = err.Error() failedImports.Unlock() time.AfterFunc(30*time.Minute, func() { failedImports.Lock() delete(failedImports.m, repo.Path()) failedImports.Unlock() }) return } setImport(repo.ID, func(st *importState) { st.Done = true }) } func (s *Server) importProgress(w http.ResponseWriter, r *http.Request, repo Repo) { rp, _, _ := s.repoPage(r, repo, "") st, ok := importStatus(repo.ID) if st.Done { imports.Lock() delete(imports.m, repo.ID) imports.Unlock() } if !ok || st.Done { target := "/" + repo.Path() if rp.IsOwner && len(s.claimable(repo, rp.User)) > 0 { target += "/claim" } http.Redirect(w, r, target, http.StatusSeeOther) return } w.Header().Set("Refresh", "3") s.render(w, r, "message", page{Title: "Importing", Data: map[string]any{"Heading": "Importing " + repo.Path(), "Text": "Fetching branches and tags from " + st.Source + ". This page refreshes on its own."}}) } type identity struct { Name, Email string Commits int } func (i identity) Key() string { return i.Name + " <" + i.Email + ">" } func (s *Server) claimable(repo Repo, u *User) []identity { if u == nil || s.store.OpenPullCount(repo.ID) > 0 { return nil } g := Git{Dir: s.repoDir(repo)} out, err := g.run(nil, "log", "--all", "--format=%an%x00%ae%x00%cn%x00%ce") if err != nil { return nil } names := map[string]bool{strings.ToLower(u.Name): true} if d := strings.TrimSpace(s.store.Profile(u.ID).Display); d != "" { names[strings.ToLower(d)] = true } email := strings.ToLower(s.store.Account(u.ID).Email) mine := func(name, mail string) bool { mail = strings.ToLower(mail) local, domain, _ := strings.Cut(mail, "@") if _, after, ok := strings.Cut(local, "+"); ok && strings.HasPrefix(domain, "users.noreply.") { local = after } return names[strings.ToLower(strings.TrimSpace(name))] || (email != "" && mail == email) || (strings.HasPrefix(domain, "users.noreply.") && local == strings.ToLower(u.Name)) } seen := map[string]*identity{} var order []string for line := range strings.Lines(string(out)) { f := strings.Split(strings.TrimRight(line, "\n"), "\x00") if len(f) != 4 { continue } for _, p := range [][2]string{{f[0], f[1]}, {f[2], f[3]}} { if !mine(p[0], p[1]) { continue } k := p[0] + " <" + p[1] + ">" if seen[k] == nil { seen[k] = &identity{Name: p[0], Email: p[1]} order = append(order, k) } if p == [2]string{f[0], f[1]} { seen[k].Commits++ } } } out2 := make([]identity, 0, len(order)) for _, k := range order { out2 = append(out2, *seen[k]) } return out2 } func (s *Server) claimIdentity(u User) (string, string) { name := strings.TrimSpace(s.store.Profile(u.ID).Display) if name == "" { name = u.Name } email := s.store.VerifiedEmail(u.ID) if email == "" { email = u.Name + "@users.noreply." + s.host } return name, email } func (s *Server) claim(w http.ResponseWriter, r *http.Request, repo Repo) { rp, _, _ := s.repoPage(r, repo, "") if !rp.IsOwner { s.fail(w, r, 403, "Not your repository") return } ids := s.claimable(repo, rp.User) name, email := s.claimIdentity(*rp.User) if r.Method == "POST" { r.ParseForm() picked := map[string]bool{} for _, id := range ids { if slices.Contains(r.Form["id"], id.Key()) { picked[id.Key()] = true } } if len(picked) > 0 { if err := rewriteAuthors(s.repoDir(repo), picked, name+" <"+email+">"); err != nil { log.Printf("claim %s: %v", repo.Path(), err) http.Redirect(w, r, "/"+repo.Path()+"/claim?error="+template.URLQueryEscaper("Could not rewrite the history"), http.StatusSeeOther) return } } http.Redirect(w, r, "/"+repo.Path(), http.StatusSeeOther) return } rp.Tab = "settings" s.render(w, r, "claim", page{Title: "Claim commits", Error: r.URL.Query().Get("error"), Data: map[string]any{"Page": rp, "Identities": ids, "Name": name, "Email": email}}) } func rewriteAuthors(dir string, picked map[string]bool, to string) error { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() env := []string{"GIT_CONFIG_NOSYSTEM=1", "HOME=/nonexistent", "LC_ALL=C.UTF-8", "PATH=" + os.Getenv("PATH")} export := exec.CommandContext(ctx, "git", "-C", dir, "fast-export", "--all", "--no-data", "--signed-tags=strip", "--signed-commits=strip", "--reencode=yes") imp := exec.CommandContext(ctx, "git", "-C", dir, "fast-import", "--quiet", "--force") export.Env, imp.Env = env, env var errb bytes.Buffer export.Stderr, imp.Stderr = &errb, &errb out, err := export.StdoutPipe() if err != nil { return err } in, err := imp.StdinPipe() if err != nil { return err } if err := export.Start(); err != nil { return err } if err := imp.Start(); err != nil { export.Process.Kill() return err } ferr := filterIdents(bufio.NewReader(out), in, picked, to) in.Close() eerr, ierr := export.Wait(), imp.Wait() for _, e := range []error{ferr, eerr, ierr} { if e != nil { return fmt.Errorf("%v: %s", e, strings.TrimSpace(errb.String())) } } return nil } func filterIdents(rd *bufio.Reader, w io.Writer, picked map[string]bool, to string) error { bw := bufio.NewWriter(w) defer bw.Flush() for { line, err := rd.ReadString('\n') if err == io.EOF && line == "" { return nil } if err != nil && err != io.EOF { return err } if n, ok := strings.CutPrefix(line, "data "); ok { size, perr := strconv.Atoi(strings.TrimSpace(n)) if perr != nil { return perr } bw.WriteString(line) if _, err := io.CopyN(bw, rd, int64(size)); err != nil { return err } continue } for _, kind := range []string{"author ", "committer "} { rest, ok := strings.CutPrefix(line, kind) if !ok { continue } end := strings.LastIndex(rest, "> ") if end >= 0 && picked[rest[:end+1]] { line = kind + to + rest[end+1:] } } bw.WriteString(line) } }