package main import ( "net/http" "path" "slices" "strconv" "strings" "sync" "time" ) type language struct { Name string Colour string Bytes int64 Percent float64 } var languages = map[string][2]string{ ".go": {"Go", "#00add8"}, ".ml": {"OCaml", "#ef7a08"}, ".mli": {"OCaml", "#ef7a08"}, ".mly": {"OCaml", "#ef7a08"}, ".mll": {"OCaml", "#ef7a08"}, ".thy": {"Isabelle", "#fefe00"}, ".ML": {"Standard ML", "#dc566d"}, ".sml": {"Standard ML", "#dc566d"}, ".v": {"Rocq", "#d0b68c"}, ".lean": {"Lean", "#4a4a8c"}, ".agda": {"Agda", "#315665"}, ".hs": {"Haskell", "#5e5086"}, ".idr": {"Idris", "#b30000"}, ".c": {"C", "#555555"}, ".h": {"C", "#555555"}, ".cc": {"C++", "#f34b7d"}, ".cpp": {"C++", "#f34b7d"}, ".hpp": {"C++", "#f34b7d"}, ".rs": {"Rust", "#dea584"}, ".zig": {"Zig", "#ec915c"}, ".S": {"Assembly", "#6e4c13"}, ".s": {"Assembly", "#6e4c13"}, ".asm": {"Assembly", "#6e4c13"}, ".HC": {"HolyC", "#ffefaf"}, ".smt2": {"SMT", "#7b8d9a"}, ".py": {"Python", "#3572a5"}, ".rb": {"Ruby", "#701516"}, ".js": {"JavaScript", "#f1e05a"}, ".mjs": {"JavaScript", "#f1e05a"}, ".ts": {"TypeScript", "#3178c6"}, ".tsx": {"TypeScript", "#3178c6"}, ".java": {"Java", "#b07219"}, ".kt": {"Kotlin", "#a97bff"}, ".swift": {"Swift", "#f05138"}, ".scala": {"Scala", "#c22d40"}, ".ex": {"Elixir", "#6e4a7e"}, ".exs": {"Elixir", "#6e4a7e"}, ".erl": {"Erlang", "#b83998"}, ".gleam": {"Gleam", "#ffaff3"}, ".lua": {"Lua", "#000080"}, ".nix": {"Nix", "#7e7eff"}, ".sh": {"Shell", "#89e051"}, ".tex": {"TeX", "#3d6117"}, ".html": {"HTML", "#e34c26"}, ".css": {"CSS", "#663399"}, ".scm": {"Scheme", "#1e4aec"}, ".rkt": {"Racket", "#3c5caa"}, ".lisp": {"Common Lisp", "#3fb68b"}, ".clj": {"Clojure", "#db5855"}, ".fs": {"F#", "#b845fc"}, ".nim": {"Nim", "#ffc200"}, ".mm": {"Objective-C++", "#6866fb"}, ".fst": {"F*", "#572e30"}, ".fsti": {"F*", "#572e30"}, ".dfy": {"Dafny", "#ffec25"}, ".tla": {"TLA+", "#4b0079"}, ".als": {"Alloy", "#64c800"}, ".mlw": {"WhyML", "#9e3b3b"}, ".bpl": {"Boogie", "#c80fa0"}, ".vpr": {"Viper", "#7d5c99"}, ".pvs": {"PVS", "#7a5a3b"}, ".pml": {"Promela", "#de0000"}, ".smv": {"NuSMV", "#5c6bc0"}, ".sy": {"SyGuS", "#7b8d9a"}, ".sl": {"SyGuS", "#7b8d9a"}, ".smt": {"SMT", "#7b8d9a"}, ".z3": {"SMT", "#7b8d9a"}, ".maude": {"Maude", "#8a4f9e"}, ".k": {"K", "#c22d40"}, ".bel": {"Beluga", "#4f6d8f"}, ".thm": {"Abella", "#2f5d50"}, ".ott": {"Ott", "#7a6f45"}, ".lem": {"Lem", "#b8860b"}, ".elf": {"Twelf", "#a33b3b"}, ".miz": {"Mizar", "#6a5acd"}, ".mlg": {"Rocq", "#d0b68c"}, ".lagda": {"Agda", "#315665"}, ".lidr": {"Idris", "#b30000"}, ".lhs": {"Haskell", "#5e5086"}, ".hsc": {"Haskell", "#5e5086"}, ".dl": {"Datalog", "#cc6633"}, ".pro": {"Prolog", "#74283c"}, ".prolog": {"Prolog", "#74283c"}, ".php": {"PHP", "#4f5d95"}, ".cs": {"C#", "#178600"}, ".dart": {"Dart", "#00b4ab"}, ".r": {"R", "#198ce7"}, ".R": {"R", "#198ce7"}, ".jl": {"Julia", "#a270ba"}, ".pl": {"Perl", "#0298c3"}, ".pm": {"Perl", "#0298c3"}, ".raku": {"Raku", "#0000fb"}, ".rakumod": {"Raku", "#0000fb"}, ".m": {"Objective-C", "#438eff"}, ".jsx": {"JavaScript", "#f1e05a"}, ".cjs": {"JavaScript", "#f1e05a"}, ".mts": {"TypeScript", "#3178c6"}, ".cts": {"TypeScript", "#3178c6"}, ".vue": {"Vue", "#41b883"}, ".svelte": {"Svelte", "#ff3e00"}, ".sql": {"SQL", "#e38c00"}, ".ps1": {"PowerShell", "#012456"}, ".psm1": {"PowerShell", "#012456"}, ".bash": {"Shell", "#89e051"}, ".zsh": {"Shell", "#89e051"}, ".fish": {"Shell", "#89e051"}, ".cr": {"Crystal", "#000100"}, ".d": {"D", "#ba595e"}, ".odin": {"Odin", "#60afff"}, ".ha": {"Hare", "#9d7424"}, ".purs": {"PureScript", "#1d222d"}, ".elm": {"Elm", "#60b5cc"}, ".re": {"Reason", "#ff5847"}, ".rei": {"Reason", "#ff5847"}, ".res": {"ReScript", "#ed5051"}, ".resi": {"ReScript", "#ed5051"}, ".el": {"Emacs Lisp", "#c065db"}, ".vim": {"Vim Script", "#199f4b"}, ".fnl": {"Fennel", "#fff3d7"}, ".f90": {"Fortran", "#4d41b1"}, ".f95": {"Fortran", "#4d41b1"}, ".f03": {"Fortran", "#4d41b1"}, ".f08": {"Fortran", "#4d41b1"}, ".f": {"Fortran", "#4d41b1"}, ".cob": {"COBOL", "#005ca5"}, ".cbl": {"COBOL", "#005ca5"}, ".ada": {"Ada", "#02f88c"}, ".adb": {"Ada", "#02f88c"}, ".ads": {"Ada", "#02f88c"}, ".pas": {"Pascal", "#e3f171"}, ".pp": {"Pascal", "#e3f171"}, ".vhd": {"VHDL", "#adb2cb"}, ".vhdl": {"VHDL", "#adb2cb"}, ".sv": {"SystemVerilog", "#dae1c2"}, ".svh": {"SystemVerilog", "#dae1c2"}, ".cu": {"CUDA", "#3a4e3a"}, ".cuh": {"CUDA", "#3a4e3a"}, ".glsl": {"GLSL", "#5686a5"}, ".frag": {"GLSL", "#5686a5"}, ".vert": {"GLSL", "#5686a5"}, ".wgsl": {"WGSL", "#1a5e9a"}, ".hlsl": {"HLSL", "#aace60"}, ".tf": {"HCL", "#844fba"}, ".hcl": {"HCL", "#844fba"}, ".cmake": {"CMake", "#da3434"}, ".mk": {"Makefile", "#427819"}, ".scss": {"SCSS", "#c6538c"}, ".sass": {"SCSS", "#c6538c"}, ".less": {"Less", "#1d365d"}, ".coffee": {"CoffeeScript", "#244776"}, ".groovy": {"Groovy", "#4298b8"}, ".gradle": {"Groovy", "#4298b8"}, ".cljs": {"Clojure", "#db5855"}, ".cljc": {"Clojure", "#db5855"}, ".hrl": {"Erlang", "#b83998"}, ".pony": {"Pony", "#8a6d3b"}, ".chpl": {"Chapel", "#8dc63f"}, ".apl": {"APL", "#5a8164"}, ".bqn": {"BQN", "#2b7067"}, ".ijs": {"J", "#9eedff"}, ".sol": {"Solidity", "#aa6746"}, ".move": {"Move", "#4a137a"}, ".cairo": {"Cairo", "#ff4a48"}, ".wat": {"WebAssembly", "#04133b"}, ".wast": {"WebAssembly", "#04133b"}, ".ll": {"LLVM", "#185619"}, ".mlir": {"MLIR", "#5ec8db"}, ".td": {"TableGen", "#5f7a8c"}, ".hx": {"Haxe", "#df7900"}, ".kk": {"Koka", "#215166"}, ".roc": {"Roc", "#7c38f5"}, ".pyi": {"Python", "#3572a5"}, ".pyw": {"Python", "#3572a5"}, ".ipynb": {"Jupyter Notebook", "#da5b0b"}, ".ss": {"Scheme", "#1e4aec"}, ".sld": {"Scheme", "#1e4aec"}, ".lsp": {"Common Lisp", "#3fb68b"}, ".asd": {"Common Lisp", "#3fb68b"}, ".awk": {"Awk", "#c30e9b"}, ".tcl": {"Tcl", "#e4cc98"}, ".mojo": {"Mojo", "#ff4c1f"}, ".typ": {"Typst", "#239dad"}, ".sty": {"TeX", "#3d6117"}, ".cls": {"TeX", "#3d6117"}, ".ltx": {"TeX", "#3d6117"}, ".dhall": {"Dhall", "#dfafff"}, ".ncl": {"Nickel", "#e0c3fc"}, ".jsonnet": {"Jsonnet", "#0064bd"}, ".libsonnet": {"Jsonnet", "#0064bd"}, ".bzl": {"Starlark", "#76d275"}, ".star": {"Starlark", "#76d275"}, ".nu": {"Nushell", "#4e9906"}, ".kts": {"Kotlin", "#a97bff"}, ".heex": {"Elixir", "#6e4a7e"}, ".vb": {"Visual Basic", "#945db7"}, ".nasm": {"Assembly", "#6e4c13"}, ".zon": {"Zig", "#ec915c"}, ".eliom": {"OCaml", "#ef7a08"}, ".sig": {"Standard ML", "#dc566d"}, ".fun": {"Standard ML", "#dc566d"}, ".fut": {"Futhark", "#5f021f"}, ".u": {"Unison", "#2f4f4f"}, } var vendorDirs = []string{"vendor/", "node_modules/", "third_party/", "thirdparty/", "external/", "deps/", "dist/", "katex/"} func vendored(name string) bool { base := path.Base(name) if strings.Contains(base, ".min.") || strings.HasSuffix(base, ".pb.go") || strings.HasSuffix(base, "_generated.go") { return true } for _, d := range vendorDirs { if strings.HasPrefix(name, d) || strings.Contains(name, "/"+d) { return true } } return false } type langEntry struct { head string stamp time.Time langs []language } var langCache = struct { sync.Mutex m map[int64]langEntry }{m: map[int64]langEntry{}} func (s *Server) repoLanguages(repo Repo) []language { g := Git{Dir: s.repoDir(repo)} stamp := repoUpdated(g.Dir) langCache.Lock() e, ok := langCache.m[repo.ID] langCache.Unlock() if ok && e.stamp.Equal(stamp) { return e.langs } head, err := g.Resolve("HEAD") if err != nil { return nil } if ok && e.head == head { e.stamp = stamp langCache.Lock() langCache.m[repo.ID] = e langCache.Unlock() return e.langs } out, err := g.run(nil, "ls-tree", "-r", "-l", "-z", head) if err != nil { return nil } bytes := map[string]int64{} var total int64 for _, rec := range strings.Split(string(out), "\x00") { meta, name, ok := strings.Cut(rec, "\t") f := strings.Fields(meta) if !ok || len(f) != 4 || f[1] != "blob" || vendored(name) { continue } l, known := languages[path.Ext(name)] if !known { l, known = languages[strings.ToLower(path.Ext(name))] } if !known { continue } size, _ := strconv.ParseInt(f[3], 10, 64) bytes[l[0]] += size total += size } var langs []language for _, l := range languages { if n, ok := bytes[l[0]]; ok { langs = append(langs, language{Name: l[0], Colour: l[1], Bytes: n, Percent: float64(n) * 100 / float64(total)}) delete(bytes, l[0]) } } slices.SortFunc(langs, func(a, b language) int { if a.Bytes != b.Bytes { return int(b.Bytes - a.Bytes) } return strings.Compare(a.Name, b.Name) }) langCache.Lock() langCache.m[repo.ID] = langEntry{head, stamp, langs} langCache.Unlock() return langs } type repoHit struct { Repo Language *language score int } type userHit struct { User Profile Profile Repos int Org bool score int } type langCount struct { Name string Colour string Count int Query string } func langQualifier(name string) string { if strings.ContainsRune(name, ' ') { return `language:"` + name + `"` } return "language:" + name } func matchScore(q string, fields ...string) int { if q == "" { return 1 } best := 0 for i, f := range fields { f = strings.ToLower(f) weight := len(fields) - i switch { case f == q: best = max(best, 100*weight) case strings.HasPrefix(f, q): best = max(best, 50*weight) case strings.Contains(f, q): best = max(best, 10*weight) } } return best } func (s *Server) find(w http.ResponseWriter, r *http.Request) { query := strings.TrimSpace(r.URL.Query().Get("q")) kind := r.URL.Query().Get("type") text, quals := parseQuery(query) lang := r.URL.Query().Get("lang") if l, ok := quals["language"]; ok { lang = l } lang = canonicalLanguage(lang) owner, tag := quals["user"], strings.ToLower(quals["tag"]) sort := r.URL.Query().Get("sort") q := strings.ToLower(text) if kind != "users" { kind = "repos" } all, _ := s.store.PublicRepos("", "") var repos []repoHit counts := map[string]*langCount{} for _, repo := range all { langs := s.repoLanguages(repo) var names []string for _, l := range langs { names = append(names, l.Name) } score := matchScore(q, repo.Name, repo.Owner.Name, strings.Join(repo.Tags, " "), strings.Join(names, " "), repo.Description) if score == 0 || (owner != "" && !strings.EqualFold(owner, repo.Owner.Name)) || (tag != "" && !slices.Contains(repo.Tags, tag)) { continue } hit := repoHit{Repo: repo, score: score} if len(langs) > 0 { hit.Language = &langs[0] c, ok := counts[langs[0].Name] if !ok { c = &langCount{Name: langs[0].Name, Colour: langs[0].Colour} counts[langs[0].Name] = c } c.Count++ } if lang != "" && (hit.Language == nil || hit.Language.Name != lang) { continue } hit.Updated = repoUpdated(s.repoDir(repo)) repos = append(repos, hit) } switch sort { case "updated": slices.SortStableFunc(repos, func(a, b repoHit) int { return b.Updated.Compare(a.Updated) }) case "stars": slices.SortStableFunc(repos, func(a, b repoHit) int { return b.Stars - a.Stars }) default: sort = "" slices.SortStableFunc(repos, func(a, b repoHit) int { if a.score != b.score { return b.score - a.score } return b.Updated.Compare(a.Updated) }) } var facets []langCount base := strings.TrimSpace(stripLanguage(query)) for _, c := range counts { c.Query = strings.TrimSpace(base + " " + langQualifier(c.Name)) facets = append(facets, *c) } box := base if lang != "" { box = strings.TrimSpace(base + " " + langQualifier(lang)) } slices.SortFunc(facets, func(a, b langCount) int { if a.Count != b.Count { return b.Count - a.Count } return strings.Compare(a.Name, b.Name) }) var users []userHit for _, hit := range s.people() { p := hit.Profile score := matchScore(q, hit.Name, p.Display, p.Location, p.Workplace, p.Bio) if score == 0 || (owner != "" && !strings.EqualFold(owner, hit.Name)) { continue } hit.score = score u := hit.User if lang != "" { found := false for _, repo := range repos { if repo.Owner.ID == u.ID { found = true break } } if !found { continue } } users = append(users, hit) } slices.SortStableFunc(users, func(a, b userHit) int { if a.score != b.score { return b.score - a.score } return b.Repos - a.Repos }) if len(repos) > 100 { repos = repos[:100] } if len(users) > 100 { users = users[:100] } title := "Search" if query != "" { title = query + " ยท search" } s.render(w, r, "search", page{Title: title, User: s.current(r), Data: map[string]any{ "Query": box, "Base": base, "Type": kind, "Lang": lang, "Sort": sort, "Repos": repos, "Users": users, "Languages": facets, "Facets": true, }}) } var qualifiers = map[string]string{"language": "language", "lang": "language", "user": "user", "owner": "user", "tag": "tag", "topic": "tag"} func queryTokens(query string) []string { var tokens []string var cur strings.Builder quoted := false for _, r := range query { switch { case r == '"': quoted = !quoted case r == ' ' && !quoted: if cur.Len() > 0 { tokens = append(tokens, cur.String()) cur.Reset() } default: cur.WriteRune(r) } } if cur.Len() > 0 { tokens = append(tokens, cur.String()) } return tokens } func parseQuery(query string) (string, map[string]string) { quals := map[string]string{} var words []string for _, tok := range queryTokens(query) { key, value, ok := strings.Cut(tok, ":") if name, known := qualifiers[strings.ToLower(key)]; ok && known && value != "" { quals[name] = strings.TrimPrefix(value, "~") continue } words = append(words, tok) } return strings.Join(words, " "), quals } func stripLanguage(query string) string { var keep []string for _, tok := range queryTokens(query) { key, _, ok := strings.Cut(tok, ":") if ok && qualifiers[strings.ToLower(key)] == "language" { continue } if strings.ContainsRune(tok, ' ') { k, v, _ := strings.Cut(tok, ":") tok = k + `:"` + v + `"` } keep = append(keep, tok) } return strings.Join(keep, " ") } func canonicalLanguage(name string) string { if name == "" { return "" } for _, l := range languages { if strings.EqualFold(l[0], name) { return l[0] } } return name } var peopleCache = struct { sync.Mutex at time.Time hits []userHit }{} func (s *Server) people() []userHit { peopleCache.Lock() defer peopleCache.Unlock() if time.Since(peopleCache.at) < 30*time.Second { return peopleCache.hits } counts := map[int64]int{} repos, _ := s.store.PublicRepos("", "") for _, r := range repos { counts[r.Owner.ID]++ } var hits []userHit for _, u := range s.store.SearchableUsers() { hits = append(hits, userHit{User: u, Profile: s.store.Profile(u.ID), Org: s.store.IsOrg(u.ID), Repos: counts[u.ID]}) } peopleCache.hits, peopleCache.at = hits, time.Now() return hits }