package forge import ( "bytes" "net/http" "path" "slices" "strconv" "strings" "sync" "time" "github.com/go-enry/go-enry/v2" ) type language struct { Name string Colour string Bytes int64 Percent float64 } var languages = map[string][2]string{ ".go": {"Go", "#00add8"}, ".ml": {"OCaml", "#ef7a08"}, ".mli": {"OCaml", "#ef7a08"}, ".mly": {"OCaml", "#ef7a08"}, ".mll": {"OCaml", "#ef7a08"}, ".thy": {"Isabelle", "#fefe00"}, ".ML": {"Standard ML", "#dc566d"}, ".sml": {"Standard ML", "#dc566d"}, ".v": {"Rocq", "#d0b68c"}, ".lean": {"Lean", "#4a4a8c"}, ".agda": {"Agda", "#315665"}, ".hs": {"Haskell", "#5e5086"}, ".idr": {"Idris", "#b30000"}, ".c": {"C", "#555555"}, ".h": {"C", "#555555"}, ".cc": {"C++", "#f34b7d"}, ".cpp": {"C++", "#f34b7d"}, ".hpp": {"C++", "#f34b7d"}, ".rs": {"Rust", "#dea584"}, ".zig": {"Zig", "#ec915c"}, ".S": {"Assembly", "#6e4c13"}, ".s": {"Assembly", "#6e4c13"}, ".asm": {"Assembly", "#6e4c13"}, ".HC": {"HolyC", "#ffefaf"}, ".smt2": {"SMT", "#7b8d9a"}, ".py": {"Python", "#3572a5"}, ".rb": {"Ruby", "#701516"}, ".js": {"JavaScript", "#f1e05a"}, ".mjs": {"JavaScript", "#f1e05a"}, ".ts": {"TypeScript", "#3178c6"}, ".tsx": {"TypeScript", "#3178c6"}, ".java": {"Java", "#b07219"}, ".kt": {"Kotlin", "#a97bff"}, ".swift": {"Swift", "#f05138"}, ".scala": {"Scala", "#c22d40"}, ".ex": {"Elixir", "#6e4a7e"}, ".exs": {"Elixir", "#6e4a7e"}, ".erl": {"Erlang", "#b83998"}, ".gleam": {"Gleam", "#ffaff3"}, ".lua": {"Lua", "#000080"}, ".nix": {"Nix", "#7e7eff"}, ".sh": {"Shell", "#89e051"}, ".tex": {"TeX", "#3d6117"}, ".html": {"HTML", "#e34c26"}, ".css": {"CSS", "#663399"}, ".scm": {"Scheme", "#1e4aec"}, ".rkt": {"Racket", "#3c5caa"}, ".lisp": {"Common Lisp", "#3fb68b"}, ".clj": {"Clojure", "#db5855"}, ".fs": {"F#", "#b845fc"}, ".nim": {"Nim", "#ffc200"}, ".tmpl": {"Go Template", "#5a9fb5"}, ".gotmpl": {"Go Template", "#5a9fb5"}, ".gohtml": {"Go Template", "#5a9fb5"}, ".mm": {"Objective-C++", "#6866fb"}, ".fst": {"F*", "#572e30"}, ".fsti": {"F*", "#572e30"}, ".dfy": {"Dafny", "#ffec25"}, ".tla": {"TLA+", "#4b0079"}, ".als": {"Alloy", "#64c800"}, ".mlw": {"WhyML", "#9e3b3b"}, ".bpl": {"Boogie", "#c80fa0"}, ".vpr": {"Viper", "#7d5c99"}, ".pvs": {"PVS", "#7a5a3b"}, ".pml": {"Promela", "#de0000"}, ".smv": {"NuSMV", "#5c6bc0"}, ".sy": {"SyGuS", "#7b8d9a"}, ".sl": {"SyGuS", "#7b8d9a"}, ".smt": {"SMT", "#7b8d9a"}, ".z3": {"SMT", "#7b8d9a"}, ".maude": {"Maude", "#8a4f9e"}, ".k": {"K", "#c22d40"}, ".bel": {"Beluga", "#4f6d8f"}, ".thm": {"Abella", "#2f5d50"}, ".ott": {"Ott", "#7a6f45"}, ".lem": {"Lem", "#b8860b"}, ".elf": {"Twelf", "#a33b3b"}, ".miz": {"Mizar", "#6a5acd"}, ".mlg": {"Rocq", "#d0b68c"}, ".lagda": {"Agda", "#315665"}, ".lidr": {"Idris", "#b30000"}, ".lhs": {"Haskell", "#5e5086"}, ".hsc": {"Haskell", "#5e5086"}, ".dl": {"Datalog", "#cc6633"}, ".pro": {"Prolog", "#74283c"}, ".prolog": {"Prolog", "#74283c"}, ".php": {"PHP", "#4f5d95"}, ".cs": {"C#", "#178600"}, ".dart": {"Dart", "#00b4ab"}, ".r": {"R", "#198ce7"}, ".R": {"R", "#198ce7"}, ".jl": {"Julia", "#a270ba"}, ".pl": {"Perl", "#0298c3"}, ".pm": {"Perl", "#0298c3"}, ".raku": {"Raku", "#0000fb"}, ".rakumod": {"Raku", "#0000fb"}, ".m": {"Objective-C", "#438eff"}, ".jsx": {"JavaScript", "#f1e05a"}, ".cjs": {"JavaScript", "#f1e05a"}, ".mts": {"TypeScript", "#3178c6"}, ".cts": {"TypeScript", "#3178c6"}, ".vue": {"Vue", "#41b883"}, ".svelte": {"Svelte", "#ff3e00"}, ".sql": {"SQL", "#e38c00"}, ".ps1": {"PowerShell", "#012456"}, ".psm1": {"PowerShell", "#012456"}, ".bash": {"Shell", "#89e051"}, ".zsh": {"Shell", "#89e051"}, ".fish": {"Shell", "#89e051"}, ".cr": {"Crystal", "#000100"}, ".d": {"D", "#ba595e"}, ".odin": {"Odin", "#60afff"}, ".ha": {"Hare", "#9d7424"}, ".purs": {"PureScript", "#1d222d"}, ".elm": {"Elm", "#60b5cc"}, ".re": {"Reason", "#ff5847"}, ".rei": {"Reason", "#ff5847"}, ".res": {"ReScript", "#ed5051"}, ".resi": {"ReScript", "#ed5051"}, ".el": {"Emacs Lisp", "#c065db"}, ".vim": {"Vim Script", "#199f4b"}, ".fnl": {"Fennel", "#fff3d7"}, ".f90": {"Fortran", "#4d41b1"}, ".f95": {"Fortran", "#4d41b1"}, ".f03": {"Fortran", "#4d41b1"}, ".f08": {"Fortran", "#4d41b1"}, ".f": {"Fortran", "#4d41b1"}, ".cob": {"COBOL", "#005ca5"}, ".cbl": {"COBOL", "#005ca5"}, ".ada": {"Ada", "#02f88c"}, ".adb": {"Ada", "#02f88c"}, ".ads": {"Ada", "#02f88c"}, ".pas": {"Pascal", "#e3f171"}, ".pp": {"Pascal", "#e3f171"}, ".vhd": {"VHDL", "#adb2cb"}, ".vhdl": {"VHDL", "#adb2cb"}, ".sv": {"SystemVerilog", "#dae1c2"}, ".svh": {"SystemVerilog", "#dae1c2"}, ".cu": {"CUDA", "#3a4e3a"}, ".cuh": {"CUDA", "#3a4e3a"}, ".glsl": {"GLSL", "#5686a5"}, ".frag": {"GLSL", "#5686a5"}, ".vert": {"GLSL", "#5686a5"}, ".wgsl": {"WGSL", "#1a5e9a"}, ".hlsl": {"HLSL", "#aace60"}, ".tf": {"HCL", "#844fba"}, ".hcl": {"HCL", "#844fba"}, ".cmake": {"CMake", "#da3434"}, ".mk": {"Makefile", "#427819"}, ".scss": {"SCSS", "#c6538c"}, ".sass": {"SCSS", "#c6538c"}, ".less": {"Less", "#1d365d"}, ".coffee": {"CoffeeScript", "#244776"}, ".groovy": {"Groovy", "#4298b8"}, ".gradle": {"Groovy", "#4298b8"}, ".cljs": {"Clojure", "#db5855"}, ".cljc": {"Clojure", "#db5855"}, ".hrl": {"Erlang", "#b83998"}, ".pony": {"Pony", "#8a6d3b"}, ".chpl": {"Chapel", "#8dc63f"}, ".apl": {"APL", "#5a8164"}, ".bqn": {"BQN", "#2b7067"}, ".ijs": {"J", "#9eedff"}, ".sol": {"Solidity", "#aa6746"}, ".move": {"Move", "#4a137a"}, ".cairo": {"Cairo", "#ff4a48"}, ".wat": {"WebAssembly", "#04133b"}, ".wast": {"WebAssembly", "#04133b"}, ".ll": {"LLVM", "#185619"}, ".mlir": {"MLIR", "#5ec8db"}, ".td": {"TableGen", "#5f7a8c"}, ".hx": {"Haxe", "#df7900"}, ".kk": {"Koka", "#215166"}, ".roc": {"Roc", "#7c38f5"}, ".pyi": {"Python", "#3572a5"}, ".pyw": {"Python", "#3572a5"}, ".ipynb": {"Jupyter Notebook", "#da5b0b"}, ".ss": {"Scheme", "#1e4aec"}, ".sld": {"Scheme", "#1e4aec"}, ".lsp": {"Common Lisp", "#3fb68b"}, ".asd": {"Common Lisp", "#3fb68b"}, ".awk": {"Awk", "#c30e9b"}, ".tcl": {"Tcl", "#e4cc98"}, ".mojo": {"Mojo", "#ff4c1f"}, ".typ": {"Typst", "#239dad"}, ".sty": {"TeX", "#3d6117"}, ".cls": {"TeX", "#3d6117"}, ".ltx": {"TeX", "#3d6117"}, ".dhall": {"Dhall", "#dfafff"}, ".ncl": {"Nickel", "#e0c3fc"}, ".jsonnet": {"Jsonnet", "#0064bd"}, ".libsonnet": {"Jsonnet", "#0064bd"}, ".bzl": {"Starlark", "#76d275"}, ".star": {"Starlark", "#76d275"}, ".nu": {"Nushell", "#4e9906"}, ".kts": {"Kotlin", "#a97bff"}, ".heex": {"Elixir", "#6e4a7e"}, ".vb": {"Visual Basic", "#945db7"}, ".nasm": {"Assembly", "#6e4c13"}, ".zon": {"Zig", "#ec915c"}, ".eliom": {"OCaml", "#ef7a08"}, ".sig": {"Standard ML", "#dc566d"}, ".fun": {"Standard ML", "#dc566d"}, ".fut": {"Futhark", "#5f021f"}, ".u": {"Unison", "#2f4f4f"}, } type langEntry struct { head string stamp time.Time langs []language } var langCache = struct { sync.Mutex m map[int64]langEntry }{m: map[int64]langEntry{}} func (s *Server) repoLanguages(repo Repo) []language { g := Git{Dir: s.repoDir(repo)} stamp := repoUpdated(g.Dir) langCache.Lock() e, ok := langCache.m[repo.ID] langCache.Unlock() if ok && e.stamp.Equal(stamp) { return e.langs } head, err := g.Resolve("HEAD") if err != nil { return nil } if ok && e.head == head { e.stamp = stamp langCache.Lock() langCache.m[repo.ID] = e langCache.Unlock() return e.langs } langs := g.languages(head) langCache.Lock() langCache.m[repo.ID] = langEntry{head, stamp, langs} langCache.Unlock() return langs } type repoHit struct { Repo Language *language score int } type userHit struct { User Profile Profile Repos int Org bool score int } type langCount struct { Name string Colour string Count int Query string } func langQualifier(name string) string { if strings.ContainsRune(name, ' ') { return `language:"` + name + `"` } return "language:" + name } func matchScore(q string, fields ...string) int { if q == "" { return 1 } best := 0 for i, f := range fields { f = strings.ToLower(f) weight := len(fields) - i switch { case f == q: best = max(best, 100*weight) case strings.HasPrefix(f, q): best = max(best, 50*weight) case strings.Contains(f, q): best = max(best, 10*weight) } } return best } func (s *Server) find(w http.ResponseWriter, r *http.Request) { query := strings.TrimSpace(r.URL.Query().Get("q")) kind := r.URL.Query().Get("type") text, quals := parseQuery(query) lang := r.URL.Query().Get("lang") if l, ok := quals["language"]; ok { lang = l } lang = canonicalLanguage(lang) owner, tag := quals["user"], strings.ToLower(quals["tag"]) sort := r.URL.Query().Get("sort") q := strings.ToLower(text) if kind != "users" { kind = "repos" } all, _ := s.store.PublicRepos("", "") var repos []repoHit counts := map[string]*langCount{} for _, repo := range all { langs := s.repoLanguages(repo) var names []string for _, l := range langs { names = append(names, l.Name) } score := matchScore(q, repo.Name, repo.Owner.Name, strings.Join(repo.Tags, " "), strings.Join(names, " "), repo.Description) if score == 0 || (owner != "" && !strings.EqualFold(owner, repo.Owner.Name)) || (tag != "" && !slices.Contains(repo.Tags, tag)) { continue } hit := repoHit{Repo: repo, score: score} if len(langs) > 0 { hit.Language = &langs[0] c, ok := counts[langs[0].Name] if !ok { c = &langCount{Name: langs[0].Name, Colour: langs[0].Colour} counts[langs[0].Name] = c } c.Count++ } if lang != "" && (hit.Language == nil || hit.Language.Name != lang) { continue } hit.Updated = repoUpdated(s.repoDir(repo)) repos = append(repos, hit) } switch sort { case "updated": slices.SortStableFunc(repos, func(a, b repoHit) int { return b.Updated.Compare(a.Updated) }) case "stars": slices.SortStableFunc(repos, func(a, b repoHit) int { return b.Stars - a.Stars }) default: sort = "" slices.SortStableFunc(repos, func(a, b repoHit) int { if a.score != b.score { return b.score - a.score } return b.Updated.Compare(a.Updated) }) } var facets []langCount base := strings.TrimSpace(stripLanguage(query)) for _, c := range counts { c.Query = strings.TrimSpace(base + " " + langQualifier(c.Name)) facets = append(facets, *c) } box := base if lang != "" { box = strings.TrimSpace(base + " " + langQualifier(lang)) } slices.SortFunc(facets, func(a, b langCount) int { if a.Count != b.Count { return b.Count - a.Count } return strings.Compare(a.Name, b.Name) }) var users []userHit for _, hit := range s.people() { p := hit.Profile score := matchScore(q, hit.Name, p.Display, p.Location, p.Workplace, p.Bio) if score == 0 || (owner != "" && !strings.EqualFold(owner, hit.Name)) { continue } hit.score = score u := hit.User if lang != "" { found := false for _, repo := range repos { if repo.Owner.ID == u.ID { found = true break } } if !found { continue } } users = append(users, hit) } slices.SortStableFunc(users, func(a, b userHit) int { if a.score != b.score { return b.score - a.score } return b.Repos - a.Repos }) if len(repos) > 100 { repos = repos[:100] } if len(users) > 100 { users = users[:100] } title := "Search" if query != "" { title = query + " ยท search" } s.render(w, r, "search", page{Title: title, User: s.current(r), Data: map[string]any{ "Query": box, "Base": base, "Type": kind, "Lang": lang, "Sort": sort, "Repos": repos, "Users": users, "Languages": facets, "Facets": true, }}) } var qualifiers = map[string]string{"language": "language", "lang": "language", "user": "user", "owner": "user", "tag": "tag", "topic": "tag"} func queryTokens(query string) []string { var tokens []string var cur strings.Builder quoted := false for _, r := range query { switch { case r == '"': quoted = !quoted case r == ' ' && !quoted: if cur.Len() > 0 { tokens = append(tokens, cur.String()) cur.Reset() } default: cur.WriteRune(r) } } if cur.Len() > 0 { tokens = append(tokens, cur.String()) } return tokens } func parseQuery(query string) (string, map[string]string) { quals := map[string]string{} var words []string for _, tok := range queryTokens(query) { key, value, ok := strings.Cut(tok, ":") if name, known := qualifiers[strings.ToLower(key)]; ok && known && value != "" { quals[name] = strings.TrimPrefix(value, "~") continue } words = append(words, tok) } return strings.Join(words, " "), quals } func stripLanguage(query string) string { var keep []string for _, tok := range queryTokens(query) { key, _, ok := strings.Cut(tok, ":") if ok && qualifiers[strings.ToLower(key)] == "language" { continue } if strings.ContainsRune(tok, ' ') { k, v, _ := strings.Cut(tok, ":") tok = k + `:"` + v + `"` } keep = append(keep, tok) } return strings.Join(keep, " ") } func canonicalLanguage(name string) string { if name == "" { return "" } for _, l := range languages { if strings.EqualFold(l[0], name) { return l[0] } } return name } var peopleCache = struct { sync.Mutex at time.Time hits []userHit }{} func (s *Server) people() []userHit { peopleCache.Lock() defer peopleCache.Unlock() if time.Since(peopleCache.at) < 30*time.Second { return peopleCache.hits } counts := map[int64]int{} repos, _ := s.store.PublicRepos("", "") for _, r := range repos { counts[r.Owner.ID]++ } var hits []userHit for _, u := range s.store.SearchableUsers() { hits = append(hits, userHit{User: u, Profile: s.store.Profile(u.ID), Org: s.store.IsOrg(u.ID), Repos: counts[u.ID]}) } peopleCache.hits, peopleCache.at = hits, time.Now() return hits } type blob struct { name, id string size int64 lang string guesses []string } func (g Git) languages(head string) []language { out, err := g.run(nil, "ls-tree", "-r", "-l", "-z", head) if err != nil { return nil } var blobs []blob attributes := false for _, rec := range strings.Split(string(out), "\x00") { meta, name, ok := strings.Cut(rec, "\t") f := strings.Fields(meta) if !ok || len(f) != 4 || f[1] != "blob" { continue } size, _ := strconv.ParseInt(f[3], 10, 64) if size > 0 { blobs = append(blobs, blob{name: name, id: f[2], size: size}) } attributes = attributes || path.Base(name) == ".gitattributes" } var attrs map[string]map[string]string if attributes { attrs = g.linguist(head, blobs) } var queued []*blob var budget int64 = 8 << 20 kept := blobs[:0] for _, b := range blobs { a := attrs[b.name] if skip(a["linguist-vendored"], enry.IsVendor(b.name)) || skip(a["linguist-generated"], enry.IsGenerated(b.name, nil)) || skip(a["linguist-documentation"], enry.IsDocumentation(b.name)) || a["linguist-detectable"] != "set" && (enry.IsDotFile(b.name) || enry.IsConfiguration(b.name)) { continue } if v := a["linguist-language"]; v != "" && v != "set" && v != "unset" { b.lang, _ = enry.GetLanguageByAlias(v) } if b.lang == "" { b.guesses = enry.GetLanguagesByFilename(b.name, nil, nil) if len(b.guesses) == 0 { b.guesses = enry.GetLanguagesByExtension(b.name, nil, nil) } } kept = append(kept, b) } for i := range kept { b := &kept[i] if len(b.guesses) > 1 && b.size <= 256<<10 && budget >= b.size { budget -= b.size queued = append(queued, b) } } if len(queued) > 0 { var ids strings.Builder for _, b := range queued { ids.WriteString(b.id + "\n") } out, _ := g.run([]byte(ids.String()), "cat-file", "--batch") for _, b := range queued { header, rest, ok := bytes.Cut(out, []byte("\n")) f := strings.Fields(string(header)) if !ok || len(f) != 3 { break } n, _ := strconv.Atoi(f[2]) if n > len(rest) { break } content := rest[:min(n, 16<<10)] out = rest[min(n+1, len(rest)):] if l := enry.GetLanguage(path.Base(b.name), content); l != "" { b.lang = l } } } votes := map[string]map[string]int{} for _, b := range kept { if len(b.guesses) > 1 && b.lang != "" { ext := strings.ToLower(path.Ext(b.name)) if votes[ext] == nil { votes[ext] = map[string]int{} } votes[ext][b.lang]++ } } for i := range kept { b := &kept[i] if len(b.guesses) <= 1 { continue } best, n := "", 0 for lang, c := range votes[strings.ToLower(path.Ext(b.name))] { if c > n || c == n && lang < best { best, n = lang, c } } if best != "" { b.lang = best } } sizes := map[string]int64{} colours := map[string]string{} var total int64 for _, b := range kept { lang, colour, ok := classify(b, attrs[b.name]["linguist-detectable"]) if !ok { continue } sizes[lang] += b.size colours[lang] = colour total += b.size } var langs []language for name, n := range sizes { langs = append(langs, language{Name: name, Colour: colours[name], Bytes: n, Percent: float64(n) * 100 / float64(total)}) } slices.SortFunc(langs, func(a, b language) int { if a.Bytes != b.Bytes { return int(b.Bytes - a.Bytes) } return strings.Compare(a.Name, b.Name) }) return langs } func skip(attr string, guess bool) bool { if attr == "set" || attr == "true" { return true } if attr == "unset" || attr == "false" { return false } return guess } func classify(b blob, detectable string) (string, string, bool) { lang := b.lang if lang == "" && len(b.guesses) > 0 { lang = b.guesses[0] } if lang == "" { l, ok := languages[strings.ToLower(path.Ext(b.name))] return l[0], l[1], ok && detectable != "unset" && detectable != "false" } if group := enry.GetLanguageGroup(lang); group != "" { lang = group } switch detectable { case "set", "true": case "unset", "false": return "", "", false default: if enry.GetLanguageType(lang) != enry.Programming { return "", "", false } } return lang, enry.GetColor(lang), true } func (g Git) linguist(head string, blobs []blob) map[string]map[string]string { var names strings.Builder for _, b := range blobs { names.WriteString(b.name + "\x00") } out, err := g.run([]byte(names.String()), "check-attr", "--source="+head, "-z", "--stdin", "linguist-vendored", "linguist-generated", "linguist-documentation", "linguist-detectable", "linguist-language") if err != nil { return nil } attrs := map[string]map[string]string{} f := strings.Split(string(out), "\x00") for i := 0; i+2 < len(f); i += 3 { if f[i+2] == "unspecified" { continue } if attrs[f[i]] == nil { attrs[f[i]] = map[string]string{} } attrs[f[i]][f[i+1]] = f[i+2] } return attrs } func topLanguages(langs []language, limit int) []language { if len(langs) <= limit+1 { return langs } top := slices.Clone(langs[:limit]) other := language{Name: "Other", Colour: "#cccccc"} for _, l := range langs[limit:] { other.Bytes += l.Bytes other.Percent += l.Percent } return append(top, other) }