| @@ -0,0 +1,438 @@ |
| 1 | +package forge |
| 2 | + |
| 3 | +import ( |
| 4 | + "bufio" |
| 5 | + "bytes" |
| 6 | + "context" |
| 7 | + "encoding/base64" |
| 8 | + "errors" |
| 9 | + "fmt" |
| 10 | + "html/template" |
| 11 | + "io" |
| 12 | + "log" |
| 13 | + "net" |
| 14 | + "net/http" |
| 15 | + "net/url" |
| 16 | + "os" |
| 17 | + "os/exec" |
| 18 | + "path" |
| 19 | + "slices" |
| 20 | + "strconv" |
| 21 | + "strings" |
| 22 | + "sync" |
| 23 | + "time" |
| 24 | +) |
| 25 | + |
| 26 | +type importState struct { |
| 27 | + Source string |
| 28 | + Done bool |
| 29 | +} |
| 30 | + |
| 31 | +var imports = struct { |
| 32 | + sync.Mutex |
| 33 | + m map[int64]*importState |
| 34 | +}{m: map[int64]*importState{}} |
| 35 | + |
| 36 | +var importSlots = make(chan struct{}, 2) |
| 37 | + |
| 38 | +var failedImports = struct { |
| 39 | + sync.Mutex |
| 40 | + m map[string]string |
| 41 | +}{m: map[string]string{}} |
| 42 | + |
| 43 | +func failedImport(path string) string { |
| 44 | + failedImports.Lock() |
| 45 | + defer failedImports.Unlock() |
| 46 | + return failedImports.m[path] |
| 47 | +} |
| 48 | + |
| 49 | +func importStatus(repo int64) (importState, bool) { |
| 50 | + imports.Lock() |
| 51 | + defer imports.Unlock() |
| 52 | + st, ok := imports.m[repo] |
| 53 | + if !ok { |
| 54 | + return importState{}, false |
| 55 | + } |
| 56 | + return *st, true |
| 57 | +} |
| 58 | + |
| 59 | +func setImport(repo int64, f func(*importState)) { |
| 60 | + imports.Lock() |
| 61 | + defer imports.Unlock() |
| 62 | + if st, ok := imports.m[repo]; ok { |
| 63 | + f(st) |
| 64 | + } |
| 65 | +} |
| 66 | + |
| 67 | +func publicAddr(ip net.IP) bool { |
| 68 | + if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() || ip.IsInterfaceLocalMulticast() { |
| 69 | + return false |
| 70 | + } |
| 71 | + _, cgnat, _ := net.ParseCIDR("100.64.0.0/10") |
| 72 | + return !cgnat.Contains(ip) |
| 73 | +} |
| 74 | + |
| 75 | +func importSource(raw string) (*url.URL, string, error) { |
| 76 | + u, err := url.Parse(strings.TrimSpace(raw)) |
| 77 | + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { |
| 78 | + return nil, "", errors.New("Use an https clone URL such as https://github.com/owner/repo") |
| 79 | + } |
| 80 | + port := u.Port() |
| 81 | + if port == "" { |
| 82 | + port = "443" |
| 83 | + } |
| 84 | + ips, err := net.LookupIP(u.Hostname()) |
| 85 | + if err != nil || len(ips) == 0 { |
| 86 | + return nil, "", errors.New("Could not resolve " + u.Hostname()) |
| 87 | + } |
| 88 | + for _, ip := range ips { |
| 89 | + if !publicAddr(ip) { |
| 90 | + return nil, "", errors.New("That host is not reachable from this server") |
| 91 | + } |
| 92 | + } |
| 93 | + ip := ips[0].String() |
| 94 | + if strings.Contains(ip, ":") { |
| 95 | + ip = "[" + ip + "]" |
| 96 | + } |
| 97 | + return u, u.Hostname() + ":" + port + ":" + ip, nil |
| 98 | +} |
| 99 | + |
| 100 | +func importName(u *url.URL) string { |
| 101 | + name := strings.ToLower(strings.TrimSuffix(path.Base(strings.TrimSuffix(u.Path, "/")), ".git")) |
| 102 | + if validName(name) && !reservedRepo(name) { |
| 103 | + return name |
| 104 | + } |
| 105 | + return "" |
| 106 | +} |
| 107 | + |
| 108 | +func remoteGit(ctx context.Context, dir, pin, token string, args ...string) ([]byte, error) { |
| 109 | + cfg := [][2]string{{"http.curloptResolve", pin}, {"http.followRedirects", "false"}, {"protocol.allow", "never"}, {"protocol.https.allow", "always"}, {"credential.helper", ""}} |
| 110 | + if token != "" { |
| 111 | + cfg = append(cfg, [2]string{"http.extraHeader", "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte("oauth2:"+token))}) |
| 112 | + } |
| 113 | + env := []string{"GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", "HOME=/nonexistent", "LC_ALL=C.UTF-8", "PATH=" + os.Getenv("PATH"), "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/false", "GIT_ALLOW_PROTOCOL=https", "GIT_CONFIG_COUNT=" + strconv.Itoa(len(cfg))} |
| 114 | + for i, kv := range cfg { |
| 115 | + env = append(env, fmt.Sprintf("GIT_CONFIG_KEY_%d=%s", i, kv[0]), fmt.Sprintf("GIT_CONFIG_VALUE_%d=%s", i, kv[1])) |
| 116 | + } |
| 117 | + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) |
| 118 | + cmd.Env = env |
| 119 | + var out, errb bytes.Buffer |
| 120 | + cmd.Stdout, cmd.Stderr = &out, &errb |
| 121 | + if err := cmd.Run(); err != nil { |
| 122 | + msg := strings.TrimSpace(errb.String()) |
| 123 | + if token != "" { |
| 124 | + msg = strings.ReplaceAll(msg, token, "…") |
| 125 | + } |
| 126 | + if i := strings.LastIndex(msg, "\n"); i >= 0 { |
| 127 | + msg = msg[i+1:] |
| 128 | + } |
| 129 | + if msg == "" { |
| 130 | + msg = err.Error() |
| 131 | + } |
| 132 | + return nil, errors.New(msg) |
| 133 | + } |
| 134 | + return out.Bytes(), nil |
| 135 | +} |
| 136 | + |
| 137 | +func (s *Server) importForm(w http.ResponseWriter, r *http.Request) { |
| 138 | + u := s.require(w, r) |
| 139 | + if u == nil { |
| 140 | + return |
| 141 | + } |
| 142 | + s.render(w, r, "import", page{Title: "Import repository", User: u, Data: map[string]any{"Orgs": s.store.Orgs(u.ID), "Owner": r.URL.Query().Get("owner")}}) |
| 143 | +} |
| 144 | + |
| 145 | +func (s *Server) importRepo(w http.ResponseWriter, r *http.Request) { |
| 146 | + u := s.require(w, r) |
| 147 | + if u == nil { |
| 148 | + return |
| 149 | + } |
| 150 | + form := map[string]any{"Orgs": s.store.Orgs(u.ID), "Owner": r.FormValue("owner"), "Source": r.FormValue("source"), "Name": r.FormValue("name"), "Description": r.FormValue("description")} |
| 151 | + fail := func(msg string) { |
| 152 | + w.WriteHeader(400) |
| 153 | + s.render(w, r, "import", page{Title: "Import repository", User: u, Error: msg, Data: form}) |
| 154 | + } |
| 155 | + owner := *u |
| 156 | + if o := r.FormValue("owner"); o != "" && o != u.Name { |
| 157 | + org, err := s.store.UserByName(o) |
| 158 | + if err != nil || !s.store.IsOrg(org.ID) || !s.store.IsMember(org.ID, u.ID) { |
| 159 | + fail("You are not a member of that organization") |
| 160 | + return |
| 161 | + } |
| 162 | + owner = org |
| 163 | + } |
| 164 | + src, pin, err := importSource(r.FormValue("source")) |
| 165 | + if err != nil { |
| 166 | + fail(err.Error()) |
| 167 | + return |
| 168 | + } |
| 169 | + name := strings.TrimSpace(r.FormValue("name")) |
| 170 | + if name == "" { |
| 171 | + name = importName(src) |
| 172 | + } |
| 173 | + if s.uploads.blocked("import:" + u.Name) { |
| 174 | + fail("Too many imports, try again later") |
| 175 | + return |
| 176 | + } |
| 177 | + if over, used, quota := overQuota(s.root, owner.Name); over { |
| 178 | + fail(quotaMessage(used, quota)) |
| 179 | + return |
| 180 | + } |
| 181 | + repo, err := s.store.CreateRepo(owner, name, strings.TrimSpace(r.FormValue("description")), visibility(r.FormValue("visibility")), "") |
| 182 | + if err != nil { |
| 183 | + fail(err.Error()) |
| 184 | + return |
| 185 | + } |
| 186 | + dir := s.repoDir(repo) |
| 187 | + if err := os.MkdirAll(path.Dir(dir), 0o755); err == nil { |
| 188 | + err = initRepo(dir) |
| 189 | + } |
| 190 | + if err != nil { |
| 191 | + os.RemoveAll(dir) |
| 192 | + s.store.DeleteRepo(repo.ID) |
| 193 | + fail("Could not create the repository") |
| 194 | + return |
| 195 | + } |
| 196 | + installHook(s.root, dir) |
| 197 | + s.uploads.fail("import:" + u.Name) |
| 198 | + imports.Lock() |
| 199 | + imports.m[repo.ID] = &importState{Source: src.Redacted()} |
| 200 | + imports.Unlock() |
| 201 | + go s.runImport(repo, src.String(), pin, strings.TrimSpace(r.FormValue("token"))) |
| 202 | + http.Redirect(w, r, "/"+repo.Path()+"/import", http.StatusSeeOther) |
| 203 | +} |
| 204 | + |
| 205 | +func (s *Server) runImport(repo Repo, src, pin, token string) { |
| 206 | + importSlots <- struct{}{} |
| 207 | + defer func() { <-importSlots }() |
| 208 | + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) |
| 209 | + defer cancel() |
| 210 | + dir := s.repoDir(repo) |
| 211 | + err := func() error { |
| 212 | + head, err := remoteGit(ctx, dir, pin, token, "ls-remote", "--symref", src, "HEAD") |
| 213 | + if err != nil { |
| 214 | + return err |
| 215 | + } |
| 216 | + if _, err := remoteGit(ctx, dir, pin, token, "fetch", "--quiet", "--no-tags", src, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"); err != nil { |
| 217 | + return err |
| 218 | + } |
| 219 | + for line := range strings.Lines(string(head)) { |
| 220 | + if ref, ok := strings.CutPrefix(line, "ref: refs/heads/"); ok { |
| 221 | + branch, _, _ := strings.Cut(ref, "\t") |
| 222 | + Git{Dir: dir}.run(nil, "symbolic-ref", "HEAD", "refs/heads/"+branch) |
| 223 | + } |
| 224 | + } |
| 225 | + if over, used, quota := overQuota(s.root, repo.Owner.Name); over { |
| 226 | + return errors.New(quotaMessage(used, quota)) |
| 227 | + } |
| 228 | + return nil |
| 229 | + }() |
| 230 | + if err != nil { |
| 231 | + log.Printf("import %s: %v", repo.Path(), err) |
| 232 | + os.RemoveAll(dir) |
| 233 | + s.store.DeleteRepo(repo.ID) |
| 234 | + imports.Lock() |
| 235 | + delete(imports.m, repo.ID) |
| 236 | + imports.Unlock() |
| 237 | + failedImports.Lock() |
| 238 | + failedImports.m[repo.Path()] = err.Error() |
| 239 | + failedImports.Unlock() |
| 240 | + time.AfterFunc(30*time.Minute, func() { |
| 241 | + failedImports.Lock() |
| 242 | + delete(failedImports.m, repo.Path()) |
| 243 | + failedImports.Unlock() |
| 244 | + }) |
| 245 | + return |
| 246 | + } |
| 247 | + setImport(repo.ID, func(st *importState) { st.Done = true }) |
| 248 | +} |
| 249 | + |
| 250 | +func (s *Server) importProgress(w http.ResponseWriter, r *http.Request, repo Repo) { |
| 251 | + rp, _, _ := s.repoPage(r, repo, "") |
| 252 | + st, ok := importStatus(repo.ID) |
| 253 | + if st.Done { |
| 254 | + imports.Lock() |
| 255 | + delete(imports.m, repo.ID) |
| 256 | + imports.Unlock() |
| 257 | + } |
| 258 | + if !ok || st.Done { |
| 259 | + target := "/" + repo.Path() |
| 260 | + if rp.IsOwner && len(s.claimable(repo, rp.User)) > 0 { |
| 261 | + target += "/claim" |
| 262 | + } |
| 263 | + http.Redirect(w, r, target, http.StatusSeeOther) |
| 264 | + return |
| 265 | + } |
| 266 | + w.Header().Set("Refresh", "3") |
| 267 | + s.render(w, r, "message", page{Title: "Importing", Data: map[string]any{"Heading": "Importing " + repo.Path(), "Text": "Fetching branches and tags from " + st.Source + ". This page refreshes on its own."}}) |
| 268 | +} |
| 269 | + |
| 270 | +type identity struct { |
| 271 | + Name, Email string |
| 272 | + Commits int |
| 273 | +} |
| 274 | + |
| 275 | +func (i identity) Key() string { return i.Name + " <" + i.Email + ">" } |
| 276 | + |
| 277 | +func (s *Server) claimable(repo Repo, u *User) []identity { |
| 278 | + if u == nil || s.store.OpenPullCount(repo.ID) > 0 { |
| 279 | + return nil |
| 280 | + } |
| 281 | + g := Git{Dir: s.repoDir(repo)} |
| 282 | + out, err := g.run(nil, "log", "--all", "--format=%an%x00%ae%x00%cn%x00%ce") |
| 283 | + if err != nil { |
| 284 | + return nil |
| 285 | + } |
| 286 | + names := map[string]bool{strings.ToLower(u.Name): true} |
| 287 | + if d := strings.TrimSpace(s.store.Profile(u.ID).Display); d != "" { |
| 288 | + names[strings.ToLower(d)] = true |
| 289 | + } |
| 290 | + email := strings.ToLower(s.store.Account(u.ID).Email) |
| 291 | + mine := func(name, mail string) bool { |
| 292 | + mail = strings.ToLower(mail) |
| 293 | + local, domain, _ := strings.Cut(mail, "@") |
| 294 | + if _, after, ok := strings.Cut(local, "+"); ok && strings.HasPrefix(domain, "users.noreply.") { |
| 295 | + local = after |
| 296 | + } |
| 297 | + return names[strings.ToLower(strings.TrimSpace(name))] || (email != "" && mail == email) || (strings.HasPrefix(domain, "users.noreply.") && local == strings.ToLower(u.Name)) |
| 298 | + } |
| 299 | + seen := map[string]*identity{} |
| 300 | + var order []string |
| 301 | + for line := range strings.Lines(string(out)) { |
| 302 | + f := strings.Split(strings.TrimRight(line, "\n"), "\x00") |
| 303 | + if len(f) != 4 { |
| 304 | + continue |
| 305 | + } |
| 306 | + for _, p := range [][2]string{{f[0], f[1]}, {f[2], f[3]}} { |
| 307 | + if !mine(p[0], p[1]) { |
| 308 | + continue |
| 309 | + } |
| 310 | + k := p[0] + " <" + p[1] + ">" |
| 311 | + if seen[k] == nil { |
| 312 | + seen[k] = &identity{Name: p[0], Email: p[1]} |
| 313 | + order = append(order, k) |
| 314 | + } |
| 315 | + if p == [2]string{f[0], f[1]} { |
| 316 | + seen[k].Commits++ |
| 317 | + } |
| 318 | + } |
| 319 | + } |
| 320 | + out2 := make([]identity, 0, len(order)) |
| 321 | + for _, k := range order { |
| 322 | + out2 = append(out2, *seen[k]) |
| 323 | + } |
| 324 | + return out2 |
| 325 | +} |
| 326 | + |
| 327 | +func (s *Server) claimIdentity(u User) (string, string) { |
| 328 | + name := strings.TrimSpace(s.store.Profile(u.ID).Display) |
| 329 | + if name == "" { |
| 330 | + name = u.Name |
| 331 | + } |
| 332 | + email := s.store.VerifiedEmail(u.ID) |
| 333 | + if email == "" { |
| 334 | + email = u.Name + "@users.noreply." + s.host |
| 335 | + } |
| 336 | + return name, email |
| 337 | +} |
| 338 | + |
| 339 | +func (s *Server) claim(w http.ResponseWriter, r *http.Request, repo Repo) { |
| 340 | + rp, _, _ := s.repoPage(r, repo, "") |
| 341 | + if !rp.IsOwner { |
| 342 | + s.fail(w, r, 403, "Not your repository") |
| 343 | + return |
| 344 | + } |
| 345 | + ids := s.claimable(repo, rp.User) |
| 346 | + name, email := s.claimIdentity(*rp.User) |
| 347 | + if r.Method == "POST" { |
| 348 | + r.ParseForm() |
| 349 | + picked := map[string]bool{} |
| 350 | + for _, id := range ids { |
| 351 | + if slices.Contains(r.Form["id"], id.Key()) { |
| 352 | + picked[id.Key()] = true |
| 353 | + } |
| 354 | + } |
| 355 | + if len(picked) > 0 { |
| 356 | + if err := rewriteAuthors(s.repoDir(repo), picked, name+" <"+email+">"); err != nil { |
| 357 | + log.Printf("claim %s: %v", repo.Path(), err) |
| 358 | + http.Redirect(w, r, "/"+repo.Path()+"/claim?error="+template.URLQueryEscaper("Could not rewrite the history"), http.StatusSeeOther) |
| 359 | + return |
| 360 | + } |
| 361 | + } |
| 362 | + http.Redirect(w, r, "/"+repo.Path(), http.StatusSeeOther) |
| 363 | + return |
| 364 | + } |
| 365 | + rp.Tab = "settings" |
| 366 | + s.render(w, r, "claim", page{Title: "Claim commits", Error: r.URL.Query().Get("error"), Data: map[string]any{"Page": rp, "Identities": ids, "Name": name, "Email": email}}) |
| 367 | +} |
| 368 | + |
| 369 | +func rewriteAuthors(dir string, picked map[string]bool, to string) error { |
| 370 | + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) |
| 371 | + defer cancel() |
| 372 | + env := []string{"GIT_CONFIG_NOSYSTEM=1", "HOME=/nonexistent", "LC_ALL=C.UTF-8", "PATH=" + os.Getenv("PATH")} |
| 373 | + export := exec.CommandContext(ctx, "git", "-C", dir, "fast-export", "--all", "--no-data", "--signed-tags=strip", "--signed-commits=strip", "--reencode=yes") |
| 374 | + imp := exec.CommandContext(ctx, "git", "-C", dir, "fast-import", "--quiet", "--force") |
| 375 | + export.Env, imp.Env = env, env |
| 376 | + var errb bytes.Buffer |
| 377 | + export.Stderr, imp.Stderr = &errb, &errb |
| 378 | + out, err := export.StdoutPipe() |
| 379 | + if err != nil { |
| 380 | + return err |
| 381 | + } |
| 382 | + in, err := imp.StdinPipe() |
| 383 | + if err != nil { |
| 384 | + return err |
| 385 | + } |
| 386 | + if err := export.Start(); err != nil { |
| 387 | + return err |
| 388 | + } |
| 389 | + if err := imp.Start(); err != nil { |
| 390 | + export.Process.Kill() |
| 391 | + return err |
| 392 | + } |
| 393 | + ferr := filterIdents(bufio.NewReader(out), in, picked, to) |
| 394 | + in.Close() |
| 395 | + eerr, ierr := export.Wait(), imp.Wait() |
| 396 | + for _, e := range []error{ferr, eerr, ierr} { |
| 397 | + if e != nil { |
| 398 | + return fmt.Errorf("%v: %s", e, strings.TrimSpace(errb.String())) |
| 399 | + } |
| 400 | + } |
| 401 | + return nil |
| 402 | +} |
| 403 | + |
| 404 | +func filterIdents(rd *bufio.Reader, w io.Writer, picked map[string]bool, to string) error { |
| 405 | + bw := bufio.NewWriter(w) |
| 406 | + defer bw.Flush() |
| 407 | + for { |
| 408 | + line, err := rd.ReadString('\n') |
| 409 | + if err == io.EOF && line == "" { |
| 410 | + return nil |
| 411 | + } |
| 412 | + if err != nil && err != io.EOF { |
| 413 | + return err |
| 414 | + } |
| 415 | + if n, ok := strings.CutPrefix(line, "data "); ok { |
| 416 | + size, perr := strconv.Atoi(strings.TrimSpace(n)) |
| 417 | + if perr != nil { |
| 418 | + return perr |
| 419 | + } |
| 420 | + bw.WriteString(line) |
| 421 | + if _, err := io.CopyN(bw, rd, int64(size)); err != nil { |
| 422 | + return err |
| 423 | + } |
| 424 | + continue |
| 425 | + } |
| 426 | + for _, kind := range []string{"author ", "committer "} { |
| 427 | + rest, ok := strings.CutPrefix(line, kind) |
| 428 | + if !ok { |
| 429 | + continue |
| 430 | + } |
| 431 | + end := strings.LastIndex(rest, "> ") |
| 432 | + if end >= 0 && picked[rest[:end+1]] { |
| 433 | + line = kind + to + rest[end+1:] |
| 434 | + } |
| 435 | + } |
| 436 | + bw.WriteString(line) |
| 437 | + } |
| 438 | +} |
~james SHA256:KlzJ0P0gvJcnOYpaxD5yCUw52ImHttTKRjVAEnnYnKg