commit d04a1dedc45b2aabb2ec9bae57604939a196b9be from: Murilo Ijanc date: Tue Apr 7 00:10:02 2026 UTC Initial import of wp, static site generator for OpenBSD ports commit - /dev/null commit + d04a1dedc45b2aabb2ec9bae57604939a196b9be blob - /dev/null blob + 5ae953572dda98e3229e9c58710e2d760e76ee88 (mode 644) --- /dev/null +++ .gitignore @@ -0,0 +1,4 @@ +wp +doc/ +site/ +ports-dump.txt blob - /dev/null blob + df99c69198f5813df5fc3eaa007a2af0e60a7bbd (mode 644) --- /dev/null +++ .rustfmt.toml @@ -0,0 +1 @@ +max_width = 80 blob - /dev/null blob + 2bab1c68d626762f3cb3a1c9c29d26c9e493626e (mode 644) --- /dev/null +++ LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2024-2026 Murilo Ijanc' + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. blob - /dev/null blob + b2cffd8e3bd22a9389aa0b7af69c88328e6a4907 (mode 644) --- /dev/null +++ Makefile @@ -0,0 +1,53 @@ +PREFIX = /usr/local +MANDIR = ${PREFIX}/man +BIN = wp +SRC = wp.rs +MAN = wp.1 + +all: ${BIN} + +${BIN}: ${SRC} + rustc -O -o ${BIN} ${SRC} + strip ${BIN} + +dev: ${SRC} + rustc -o ${BIN} ${SRC} + +ci: + rustfmt --check ${SRC} + clippy-driver ${SRC} + mandoc -T lint -W warning ${MAN} + +doc: + mandoc -T ascii ${MAN} + +rustdoc: + rustdoc ${SRC} --edition 2021 --document-private-items \ + -o doc + +SITE = site +PORT = 8080 +PORTSDIR = /usr/ports + +cgi: dev + ./${BIN} -p ${PORTSDIR} -o ${SITE} + mkdir -p ${SITE}/cgi-bin + cp ${BIN} ${SITE}/cgi-bin/search.cgi + cp ${SITE}/INDEX.txt ${SITE}/cgi-bin/ + sed -i 's||

Search

\n|' ${SITE}/index.html + @echo "http://localhost:${PORT}" + cd ${SITE} && python3 -m http.server --cgi ${PORT} + +install: ${BIN} + install -m 755 ${BIN} ${PREFIX}/bin/${BIN} + install -m 644 ${MAN} ${MANDIR}/man1/${MAN} + +uninstall: + rm -f ${PREFIX}/bin/${BIN} + rm -f ${MANDIR}/man1/${MAN} + +clean: + rm -f ${BIN} + rm -rf doc + +.PHONY: all dev ci doc rustdoc cgi install uninstall clean blob - /dev/null blob + a8ebfe98bfe3795581d748c9128fdeb0e9810fe1 (mode 644) --- /dev/null +++ README.md @@ -0,0 +1,78 @@ +wp - where OpenBSD ports +======================== +wp generates a static HTML site for browsing OpenBSD ports. +No JavaScript, works in lynx. Includes built-in CGI search. + +Requirements +------------ +- rustc + +Building +-------- +Edit Makefile to match your local setup (wp is installed +into the /usr/local/bin namespace by default). + +Afterwards enter the following command to build and install +wp (if necessary as root): + + make + make install + +Running +------- + wp -p /usr/ports -o /var/www/htdocs/ports + +Options: + -p portsdir path to ports tree (default: /usr/ports) + -o outdir output directory (default: ./site) + -d dump use dump-vars output instead of parsing Makefiles + +Search +------ +wp has built-in CGI support. When the QUERY_STRING environment +variable is set, wp acts as a CGI program: it reads INDEX.txt +from the current directory and returns search results as HTML. + +The search matches port names, descriptions, and maintainers. + +CGI setup with httpd(8) +----------------------- +Generate the site: + + wp -p /usr/ports -o /var/www/htdocs/ports + +Copy the binary and INDEX.txt into the chroot: + + cp /usr/local/bin/wp /var/www/cgi-bin/search.cgi + cp /var/www/htdocs/ports/INDEX.txt /var/www/cgi-bin/INDEX.txt + +OpenBSD does not support static binaries. Copy the required +shared libraries into the chroot: + + mkdir -p /var/www/usr/lib /var/www/usr/libexec + cp /usr/lib/libc.so.* /var/www/usr/lib/ + cp /usr/lib/libm.so.* /var/www/usr/lib/ + cp /usr/lib/libpthread.so.* /var/www/usr/lib/ + cp /usr/lib/libc++abi.so.* /var/www/usr/lib/ + cp /usr/libexec/ld.so /var/www/usr/libexec/ + +Configure httpd.conf(5): + + server "ports.example.com" { + listen on * port 80 + root "/htdocs/ports" + location "/search.cgi" { + fastcgi socket "/run/slowcgi.sock" + root "/" + } + } + +Enable and start slowcgi(8): + + rcctl enable slowcgi + rcctl start slowcgi + rcctl reload httpd + +License +------- +ISC - see LICENSE. blob - /dev/null blob + 3334a3d1e3c670b8e52e0c00cd96e8b42847e6e0 (mode 644) Binary files /dev/null and assets/favicon.ico differ blob - /dev/null blob + b138bf5ca044cf2046d1f2fe7992e394653502c1 (mode 644) Binary files /dev/null and assets/logo.png differ blob - /dev/null blob + db80dd8ec83f1da43955e0925013d38a9b854218 (mode 644) --- /dev/null +++ wp.1 @@ -0,0 +1,167 @@ +.\" +.\" Copyright (c) 2024-2026 Murilo Ijanc' +.\" +.\" Permission to use, copy, modify, and/or distribute this software for any +.\" purpose with or without fee is hereby granted, provided that the above +.\" copyright notice and this permission notice appear in all copies. +.\" +.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +.\" +.Dd $Mdocdate: April 6 2026 $ +.Dt WP 1 +.Os +.Sh NAME +.Nm wp +.Nd where OpenBSD ports +.Sh SYNOPSIS +.Nm wp +.Op Fl d Ar dump +.Op Fl o Ar outdir +.Op Fl p Ar portsdir +.Sh DESCRIPTION +.Nm +generates a static HTML site for browsing +.Ox +ports. +The generated pages contain no JavaScript and work in text browsers +such as +.Xr lynx 1 . +.Pp +When the +.Ev QUERY_STRING +environment variable is set, +.Nm +operates as a CGI program: it reads +.Pa INDEX.txt +from the current directory, searches for the query term, and writes +HTML results to standard output. +.Pp +The options are as follows: +.Bl -tag -width Ds +.It Fl d Ar dump +Load port data from a +.Xr make 1 +.Cm dump-vars +output file instead of parsing Makefiles directly. +.It Fl o Ar outdir +Write generated files to +.Ar outdir . +The default is +.Pa ./site . +.It Fl p Ar portsdir +Path to the ports tree. +The default is +.Pa /usr/ports . +.El +.Sh OUTPUT +.Nm +generates the following files in +.Ar outdir : +.Bl -tag -width "categories/" +.It Pa index.html +Main page with search form, alphabetical index, and category listing. +.It Pa categories/ +One HTML file per category listing all ports in that category. +.It Pa ports/ +One HTML file per port with description, homepage, maintainer, +dependencies, and sub-packages. +.It Pa search/ +Alphabetical index pages from A to Z. +.It Pa INDEX.txt +Tab-separated index used by CGI search. +Format: path, pkgname, comment, maintainer. +.It Pa favicon.ico , Pa logo.png +Static assets embedded in the binary. +.El +.Sh CGI +To use +.Nm +as a CGI program with +.Xr httpd 8 , +copy the binary and +.Pa INDEX.txt +into the chroot: +.Bd -literal -offset indent +cp /usr/local/bin/wp /var/www/cgi-bin/search.cgi +cp /var/www/htdocs/ports/INDEX.txt /var/www/cgi-bin/ +.Ed +.Pp +Since +.Ox +does not support static binaries, the required shared libraries +must be copied into the chroot: +.Bd -literal -offset indent +mkdir -p /var/www/usr/lib /var/www/usr/libexec +cp /usr/lib/libc.so.* /var/www/usr/lib/ +cp /usr/lib/libm.so.* /var/www/usr/lib/ +cp /usr/lib/libpthread.so.* /var/www/usr/lib/ +cp /usr/lib/libc++abi.so.* /var/www/usr/lib/ +cp /usr/libexec/ld.so /var/www/usr/libexec/ +.Ed +.Pp +Configure +.Xr httpd.conf 5 : +.Bd -literal -offset indent +server "ports.example.com" { + listen on * port 80 + root "/htdocs/ports" + location "/search.cgi" { + fastcgi socket "/run/slowcgi.sock" + root "/" + } +} +.Ed +.Pp +Enable +.Xr slowcgi 8 : +.Bd -literal -offset indent +# rcctl enable slowcgi +# rcctl start slowcgi +# rcctl reload httpd +.Ed +.Sh ENVIRONMENT +.Bl -tag -width "QUERY_STRING" +.It Ev QUERY_STRING +When set, +.Nm +operates in CGI mode. +The query parameter +.Cm q +is used as the search term. +.El +.Sh FILES +.Bl -tag -width "/usr/ports" +.It Pa /usr/ports +Default ports tree location. +.El +.Sh EXIT STATUS +.Ex -std wp +.Sh EXAMPLES +Generate a site from the ports tree: +.Bd -literal -offset indent +$ wp -p /usr/ports -o /var/www/htdocs/ports +.Ed +.Pp +Generate a site from a dump-vars file: +.Bd -literal -offset indent +$ wp -d ports-dump.txt -p /usr/ports -o /var/www/htdocs/ports +.Ed +.Pp +Test CGI search locally: +.Bd -literal -offset indent +$ cd /var/www/htdocs/ports +$ QUERY_STRING="q=curl" /usr/local/bin/wp +.Ed +.Sh SEE ALSO +.Xr httpd.conf 5 , +.Xr ports 7 , +.Xr httpd 8 , +.Xr slowcgi 8 +.Sh AUTHORS +.An Murilo Ijanc' Aq Mt murilo@ijanc.org blob - /dev/null blob + a636ec040dd42996f384f3d9894e6b126d0f2f53 (mode 644) --- /dev/null +++ wp.rs @@ -0,0 +1,1106 @@ +// vim: set tw=79 cc=80 ts=4 sw=4 sts=4 et : +// +// Copyright (c) 2024-2026 Murilo Ijanc' +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +use std::{ + collections::HashMap, + fs, + io::Write, + path::{Path, PathBuf}, +}; + +const VERSION: &str = "0.1.0"; +const NAME: &str = "wp"; +const FAVICON: &[u8] = include_bytes!("assets/favicon.ico"); +const LOGO: &[u8] = include_bytes!("assets/logo.png"); +const CSS: &str = "\ +body{font-family:monospace;max-width:72em;margin:0 auto;padding:1em;\ +color:#222;background:#fff}\ +h1,h2{margin:.5em 0}\ +table{border-collapse:collapse;width:100%}\ +th,td{text-align:left;padding:.2em .5em;border-bottom:1px solid #ccc}\ +th{border-bottom:2px solid #333}\ +a{color:#00e}a:visited{color:#551a8b}\ +pre{overflow-x:auto}\ +footer{color:#666;font-size:.9em;margin-top:1em}\ +img.logo{width:4em;height:4em;vertical-align:middle;margin-right:.5em}\ +@media(prefers-color-scheme:dark){\ +body{color:#ddd;background:#1a1a1a}\ +th{border-bottom-color:#888}\ +th,td{border-bottom-color:#444}\ +a{color:#6bf}a:visited{color:#c9f}\ +footer{color:#888}\ +hr{border-color:#444}}\ +"; + +struct Port { + name: String, + category: String, + comment: String, + descr: String, + homepage: String, + maintainer: String, + categories: Vec, + distname: String, + pkgname: String, + lib_depends: Vec, + run_depends: Vec, + build_depends: Vec, + multi_packages: Vec, + sub_comments: HashMap, +} + +fn apply_modifier(val: &str, modifier: &str) -> String { + if modifier == "R" { + // :R - root (strip extension) + val.rsplit_once('.') + .map(|(r, _)| r.to_string()) + .unwrap_or_else(|| val.to_string()) + } else if modifier == "E" { + // :E - extension + val.rsplit_once('.') + .map(|(_, e)| e.to_string()) + .unwrap_or_default() + } else if modifier == "L" { + // :L - lowercase + val.to_lowercase() + } else if modifier == "U" { + // :U - uppercase + val.to_uppercase() + } else if let Some(rest) = modifier.strip_prefix("S/") { + // :S/old/new/ - substitute first + if let Some((old, tail)) = rest.split_once('/') { + let new = tail.trim_end_matches('/'); + val.replacen(old, new, 1) + } else { + val.to_string() + } + } else if let Some(rest) = modifier.strip_prefix("C/") { + // :C/pat/rep/ - substitute all (simplified, no regex) + if let Some((old, tail)) = rest.split_once('/') { + let new = tail.trim_end_matches('/'); + val.replace(old, new) + } else { + val.to_string() + } + } else { + val.to_string() + } +} + +fn expand_vars(s: &str, vars: &HashMap) -> String { + let mut result = s.to_string(); + for _ in 0..8 { + let prev = result.clone(); + let mut out = String::new(); + let mut chars = result.as_str(); + + while let Some(pos) = chars.find("${") { + out.push_str(&chars[..pos]); + let rest = &chars[pos + 2..]; + let Some(end) = rest.find('}') else { + out.push_str(&chars[pos..]); + chars = ""; + break; + }; + let expr = &rest[..end]; + let (varname, modifiers) = + if let Some((v, m)) = expr.split_once(':') { + (v, Some(m)) + } else { + (expr, None) + }; + + if let Some(val) = vars.get(varname) { + let val = if let Some(mods) = modifiers { + if val.contains("${") { + out.push_str(&format!("${{{}}}", expr)); + chars = &rest[end + 1..]; + continue; + } + let mut v = val.clone(); + for m in mods.split(':') { + v = apply_modifier(&v, m); + } + v + } else { + val.clone() + }; + out.push_str(&val); + } else { + out.push_str(&format!("${{{}}}", expr)); + } + chars = &rest[end + 1..]; + } + out.push_str(chars); + result = out; + + if result == prev { + break; + } + } + result +} + +type ModCache = HashMap>; + +fn load_module( + module: &str, + portsdir: &Path, + cache: &mut ModCache, +) -> Option<()> { + if cache.contains_key(module) { + return Some(()); + } + + let modname = module.split('/').next_back().unwrap_or(module); + let mk_path = portsdir.join(module).join(format!("{}.port.mk", modname)); + let mk_path = if mk_path.exists() { + mk_path + } else { + let alt = portsdir + .join("infrastructure/mk") + .join(format!("{}.port.mk", modname)); + if alt.exists() { + alt + } else { + cache.insert(module.to_string(), Vec::new()); + return None; + } + }; + + let text = fs::read_to_string(&mk_path).ok()?; + let mut modvars = Vec::new(); + + for line in text.lines() { + let line = line.trim(); + if line.starts_with('#') + || line.starts_with('.') + || !line.contains("MOD") + { + continue; + } + if let Some(pos) = line.find("?=") { + let k = line[..pos].trim(); + if k.starts_with("MOD") { + let v = line[pos + 2..].trim(); + modvars.push((k.to_string(), v.to_string())); + } + } else if let Some(pos) = line.find('=') { + if line[..pos].ends_with('+') { + continue; + } + let k = line[..pos].trim(); + if k.starts_with("MOD") { + let v = line[pos + 1..].trim(); + let v = v.trim_end_matches('\\').trim(); + modvars.push((k.to_string(), v.to_string())); + } + } + } + + cache.insert(module.to_string(), modvars); + Some(()) +} + +fn resolve_modules( + vars: &mut HashMap, + portsdir: &Path, + cache: &mut ModCache, +) { + let modules = vars.get("MODULES").cloned().unwrap_or_default(); + + for module in modules.split_whitespace() { + load_module(module, portsdir, cache); + if let Some(modvars) = cache.get(module) { + for (k, v) in modvars { + vars.entry(k.clone()).or_insert_with(|| v.clone()); + } + } + } +} + +fn parse_makefile(path: &Path) -> Option> { + let text = fs::read_to_string(path).ok()?; + let mut vars: HashMap = HashMap::new(); + let mut continued = String::new(); + let mut cont_key = String::new(); + + for line in text.lines() { + let line = line.trim_end(); + + if !continued.is_empty() { + let part = line.trim_start(); + if part.ends_with('\\') { + continued.push(' '); + continued.push_str(part.trim_end_matches('\\').trim()); + } else { + continued.push(' '); + continued.push_str(part); + vars.insert(cont_key.clone(), continued.trim().to_string()); + continued.clear(); + cont_key.clear(); + } + continue; + } + + if line.starts_with('#') + || line.starts_with('.') + || line.starts_with('\t') + { + continue; + } + + let (key, val) = if let Some(pos) = line.find("+=") { + let k = line[..pos].trim(); + let v = line[pos + 2..].trim(); + let prev = vars.get(k).cloned().unwrap_or_default(); + ( + k.to_string(), + format!("{} {}", prev, v.trim_end_matches('\\').trim()), + ) + } else if let Some(pos) = line.find("?=") { + let k = line[..pos].trim(); + if vars.contains_key(k) { + continue; + } + let v = line[pos + 2..].trim(); + (k.to_string(), v.trim_end_matches('\\').trim().to_string()) + } else if let Some(pos) = line.find('=') { + let k = line[..pos].trim(); + let v = line[pos + 1..].trim(); + (k.to_string(), v.trim_end_matches('\\').trim().to_string()) + } else { + continue; + }; + + if line.ends_with('\\') { + continued = val; + cont_key = key; + } else { + vars.insert(key, val); + } + } + + if !cont_key.is_empty() { + vars.insert(cont_key, continued.trim().to_string()); + } + + Some(vars) +} + +fn port_from_makefile( + category: &str, + portname: &str, + portdir: &Path, + portsdir: &Path, + modcache: &mut ModCache, +) -> Option { + let mf = portdir.join("Makefile"); + let mut vars = parse_makefile(&mf)?; + + resolve_modules(&mut vars, portsdir, modcache); + + let get = |k: &str| -> String { + let raw = vars.get(k).cloned().unwrap_or_default(); + expand_vars(&raw, &vars) + }; + + let mut comment = get("COMMENT"); + if comment.is_empty() { + comment = get("COMMENT-main"); + } + if comment.is_empty() { + return None; + } + + let descr_path = portdir.join("pkg").join("DESCR"); + let descr = fs::read_to_string(&descr_path).unwrap_or_default(); + + let categories: Vec = get("CATEGORIES") + .split_whitespace() + .map(String::from) + .collect(); + + let split_deps = |val: &str| -> Vec { + val.split_whitespace() + .map(|s| expand_vars(s, &vars)) + .filter(|s| !s.is_empty() && !s.contains("${")) + .collect() + }; + + let multi_packages: Vec = get("MULTI_PACKAGES") + .split_whitespace() + .map(String::from) + .collect(); + + let mut sub_comments = HashMap::new(); + for sub in &multi_packages { + let key = format!("COMMENT{}", sub); + if let Some(val) = vars.get(&key) { + sub_comments.insert(sub.clone(), expand_vars(val, &vars)); + } + } + + Some(Port { + name: portname.to_string(), + category: category.to_string(), + comment, + descr, + homepage: get("HOMEPAGE"), + maintainer: get("MAINTAINER"), + categories, + distname: get("DISTNAME"), + pkgname: get("PKGNAME"), + lib_depends: split_deps(&get("LIB_DEPENDS")), + run_depends: split_deps(&get("RUN_DEPENDS")), + build_depends: split_deps(&get("BUILD_DEPENDS")), + multi_packages, + sub_comments, + }) +} + +fn port_from_vars( + path: &str, + vars: &HashMap, + portsdir: Option<&Path>, +) -> Option { + let (category, name) = path.split_once('/')?; + let comment = vars.get("COMMENT").filter(|s| !s.is_empty())?; + + let descr = if let Some(pd) = portsdir { + let descr_path = pd.join(path).join("pkg").join("DESCR"); + fs::read_to_string(&descr_path).unwrap_or_default() + } else { + String::new() + }; + + let split = |k: &str| -> Vec { + vars.get(k) + .map(|v| v.split_whitespace().map(String::from).collect()) + .unwrap_or_default() + }; + + Some(Port { + name: name.to_string(), + category: category.to_string(), + comment: comment.clone(), + descr, + homepage: vars.get("HOMEPAGE").cloned().unwrap_or_default(), + maintainer: vars.get("MAINTAINER").cloned().unwrap_or_default(), + categories: split("CATEGORIES"), + distname: String::new(), + pkgname: vars.get("FULLPKGNAME").cloned().unwrap_or_default(), + lib_depends: split("LIB_DEPENDS"), + run_depends: split("RUN_DEPENDS"), + build_depends: split("BUILD_DEPENDS"), + multi_packages: split("MULTI_PACKAGES"), + sub_comments: HashMap::new(), + }) +} + +fn load_dump(path: &Path, portsdir: Option<&Path>) -> Vec { + let text = fs::read_to_string(path).unwrap_or_else(|e| errio(path, &e)); + + let mut by_port: HashMap> = HashMap::new(); + + for line in text.lines() { + // format: category/port.VARIABLE=value + let Some(dot) = line.find('.') else { + continue; + }; + let portpath = &line[..dot]; + let rest = &line[dot + 1..]; + let Some(eq) = rest.find('=') else { + continue; + }; + let var = &rest[..eq]; + let val = &rest[eq + 1..]; + by_port + .entry(portpath.to_string()) + .or_default() + .insert(var.to_string(), val.to_string()); + } + + let mut ports = Vec::new(); + let mut paths: Vec<_> = by_port.keys().cloned().collect(); + paths.sort(); + + for path in paths { + let vars = &by_port[&path]; + if let Some(port) = port_from_vars(&path, vars, portsdir) { + ports.push(port); + } + } + + ports +} + +fn walk_ports(portsdir: &Path) -> Vec { + let mut ports = Vec::new(); + let mut modcache = ModCache::new(); + let skip = [ + "infrastructure", + "tests", + ".git", + ".gitignore", + ".cvsignore", + ]; + + let mut cats: Vec<_> = fs::read_dir(portsdir) + .unwrap_or_else(|e| errio(portsdir, &e)) + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) + .filter(|e| { + let n = e.file_name(); + !skip.contains(&n.to_str().unwrap_or("")) + }) + .collect(); + cats.sort_by_key(|e| e.file_name()); + + for cat in cats { + let catname = cat.file_name().to_string_lossy().to_string(); + let mut entries: Vec<_> = match fs::read_dir(cat.path()) { + Ok(rd) => rd + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) + .collect(), + Err(_) => continue, + }; + entries.sort_by_key(|e| e.file_name()); + + for entry in entries { + let portname = entry.file_name().to_string_lossy().to_string(); + if let Some(port) = port_from_makefile( + &catname, + &portname, + &entry.path(), + portsdir, + &mut modcache, + ) { + ports.push(port); + } + } + } + + ports +} + +fn head(title: &str, depth: usize, extra: &str) -> String { + let rel = "../".repeat(depth); + format!( + "\n\n\ + \ + \ + {}\ + \ + {}\ + \n\n", + html_escape(title), + rel, + extra, + CSS + ) +} + +fn port_meta(p: &Port, depth: usize) -> String { + let rel = "../".repeat(depth); + let keywords: Vec<&str> = p.categories.iter().map(|s| s.as_str()).collect(); + format!( + "\ + \ + \ + \ + \ + \ + ", + html_escape(&p.comment), + html_escape(&p.maintainer), + html_escape(&keywords.join(", ")), + html_escape(&format!("{}/{}", p.category, p.name)), + html_escape(&p.comment), + rel, + ) +} + +fn html_escape(s: &str) -> String { + if !s.contains(&['&', '<', '>', '"'][..]) { + return s.to_string(); + } + let mut out = String::with_capacity(s.len() + 8); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + _ => out.push(c), + } + } + out +} + +fn footer(total: usize) -> String { + let date = { + let now = std::time::SystemTime::now(); + let dur = now + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + let secs = dur.as_secs() as i64; + let days = secs / 86400; + // days since 1970-01-01, convert to y-m-d + let (y, m, d) = days_to_ymd(days); + format!("{:04}-{:02}-{:02}", y, m, d) + }; + format!( + "

\n\n", + NAME, VERSION, total, date + ) +} + +fn days_to_ymd(days: i64) -> (i64, u32, u32) { + // algorithm from http://howardhinnant.github.io/date_algorithms.html + let z = days + 719468; + let era = if z >= 0 { z } else { z - 146096 } / 146097; + let doe = (z - era * 146097) as u32; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let y = (yoe as i64) + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + (y, m, d) +} + +fn dep_link(dep: &str) -> String { + // dependency format: "category/port" or "pkg:category/port" + // or "category/port,subpkg" or "category/port>=version" + let dep = dep.split(':').next_back().unwrap_or(dep); + let path = dep.split(['>', '<', '=']).next().unwrap_or(dep); + let path = path.trim_end_matches(','); + + if let Some((cat, port)) = path.split_once('/') { + let port = port.split(',').next().unwrap_or(port); + format!( + "{}", + html_escape(cat), + html_escape(port), + html_escape(dep) + ) + } else { + html_escape(dep) + } +} + +fn write_file(path: &Path, content: &str) { + let mut f = fs::File::create(path).unwrap_or_else(|e| errio(path, &e)); + if let Err(e) = f.write_all(content.as_bytes()) { + errio(path, &e); + } +} + +fn copy_assets(outdir: &Path) { + let fav = outdir.join("favicon.ico"); + if let Err(e) = fs::write(&fav, FAVICON) { + errio(&fav, &e); + } + let logo = outdir.join("logo.png"); + if let Err(e) = fs::write(&logo, LOGO) { + errio(&logo, &e); + } +} + +fn write_index(ports: &[Port], outdir: &Path, ft: &str) { + let mut cats: HashMap<&str, usize> = HashMap::new(); + for p in ports { + *cats.entry(&p.category).or_default() += 1; + } + let mut sorted: Vec<_> = cats.into_iter().collect(); + sorted.sort_by_key(|&(k, _)| k); + + let mut html = head("OpenBSD Ports", 0, ""); + html.push_str( + "

$\"\"$ OpenBSD Ports

\n", + ); + + html.push_str("\n"); + + html.push_str("

Alphabetical Index

\n"); + for c in b'a'..=b'z' { + let c = c as char; + html.push_str(&format!( + "{} \n", + c, + c.to_ascii_uppercase() + )); + } + html.push_str("

\n"); + + html.push_str("

{}

\n\ +

Index

\n\ + \n\ + \n", + cat + )); + + for p in &cat_ports { + html.push_str(&format!( + "\n", + html_escape(&p.category), + html_escape(&p.name), + html_escape(&p.name), + html_escape(&p.comment) + )); + } + + html.push_str("

Port	Description
\ + {}	{}

\n"); + html.push_str(ft); + html.push_str("\n\n"); + + write_file(&catdir.join(format!("{}.html", cat)), &html); + } +} + +fn write_port_pages(ports: &[Port], outdir: &Path, ft: &str) { + for p in ports { + let meta = port_meta(p, 2); + let mut html = head( + &format!("{}/{} - OpenBSD Ports", &p.category, &p.name), + 2, + &meta, + ); + + let display_name = if !p.pkgname.is_empty() && !p.pkgname.contains("${") + { + &p.pkgname + } else if !p.distname.is_empty() && !p.distname.contains("${") { + &p.distname + } else { + &p.name + }; + html.push_str(&format!("

{}

\n", html_escape(display_name))); + + html.push_str(&format!("

{}

\n", html_escape(&p.comment))); + + html.push_str(&format!( + "

Index | \ + {}

\n", + html_escape(&p.category), + html_escape(&p.category) + )); + + if !p.descr.is_empty() { + html.push_str(&format!( + "

Description

{}

\n", + html_escape(&p.descr) + )); + } + + if !p.homepage.is_empty() { + html.push_str(&format!( + "

Homepage: {}

\n", + html_escape(&p.homepage), + html_escape(&p.homepage) + )); + } + + if !p.maintainer.is_empty() { + html.push_str(&format!( + "

Maintainer: {}

\n", + html_escape(&p.maintainer) + )); + } + + if p.categories.len() > 1 { + html.push_str("

Categories: "); + for (i, cat) in p.categories.iter().enumerate() { + if i > 0 { + html.push_str(", "); + } + html.push_str(&format!( + "\ + {}", + html_escape(cat), + html_escape(cat) + )); + } + html.push_str("

\n"); + } + + if !p.multi_packages.is_empty() { + html.push_str("

Sub-packages

{} - {}

\n"); + } + + let dep_section = |title: &str, deps: &[String]| -> String { + if deps.is_empty() { + return String::new(); + } + let mut s = format!("

{}

{}

\n"); + s + }; + + html.push_str(&dep_section("Library Dependencies", &p.lib_depends)); + html.push_str(&dep_section("Run Dependencies", &p.run_depends)); + html.push_str(&dep_section("Build Dependencies", &p.build_depends)); + + html.push_str(ft); + html.push_str("\n\n"); + + write_file( + &outdir + .join("ports") + .join(&p.category) + .join(format!("{}.html", p.name)), + &html, + ); + } +} + +fn write_alpha(ports: &[Port], outdir: &Path, ft: &str) { + let searchdir = outdir.join("search"); + + for c in b'a'..=b'z' { + let c = c as char; + let mut matching: Vec<&Port> = ports + .iter() + .filter(|p| { + p.name + .chars() + .next() + .map(|ch| ch.to_ascii_lowercase() == c) + .unwrap_or(false) + }) + .collect(); + matching.sort_by(|a, b| a.name.cmp(&b.name)); + + let mut html = head( + &format!("{} - OpenBSD Ports", c.to_ascii_uppercase()), + 1, + "", + ); + html.push_str(&format!( + "

Ports: {}

\n\ +

Index

\n\ +

\n", + c.to_ascii_uppercase() + )); + + for lc in b'a'..=b'z' { + let lc = lc as char; + if lc == c { + html.push_str(&format!( + "{} \n", + lc.to_ascii_uppercase() + )); + } else { + html.push_str(&format!( + "{} \n", + lc, + lc.to_ascii_uppercase() + )); + } + } + html.push_str("

\n"); + + html.push_str( + "\n\ + \n", + ); + for p in &matching { + html.push_str(&format!( + "\n", + html_escape(&p.category), + html_escape(&p.name), + html_escape(&p.name), + html_escape(&p.category), + html_escape(&p.category), + html_escape(&p.comment) + )); + } + + html.push_str("

Port	Category	Description
\ + {}	{}	{}

\n"); + html.push_str(ft); + html.push_str("\n\n"); + + write_file(&searchdir.join(format!("{}.html", c)), &html); + } +} + +fn write_index_txt(ports: &[Port], outdir: &Path) { + let mut out = String::new(); + for p in ports { + // format: category/name\tpkgname\tcomment\tmaintainer + out.push_str(&p.category); + out.push('/'); + out.push_str(&p.name); + out.push('\t'); + out.push_str(&p.pkgname); + out.push('\t'); + out.push_str(&p.comment); + out.push('\t'); + out.push_str(&p.maintainer); + out.push('\n'); + } + write_file(&outdir.join("INDEX.txt"), &out); +} + +fn url_decode(s: &str) -> String { + let mut out = Vec::with_capacity(s.len()); + let mut bytes = s.as_bytes().iter(); + while let Some(&b) = bytes.next() { + if b == b'%' { + let hi = bytes.next().copied().unwrap_or(0); + let lo = bytes.next().copied().unwrap_or(0); + let hex = [hi, lo]; + if let Ok(s) = std::str::from_utf8(&hex) { + if let Ok(n) = u8::from_str_radix(s, 16) { + out.push(n); + continue; + } + } + out.push(b); + } else if b == b'+' { + out.push(b' '); + } else { + out.push(b); + } + } + String::from_utf8_lossy(&out).into_owned() +} + +fn cgi_search(index_path: &Path) { + let qs = std::env::var("QUERY_STRING").unwrap_or_default(); + let query = qs + .split('&') + .find_map(|p| p.strip_prefix("q=").map(url_decode)) + .unwrap_or_default() + .to_lowercase(); + + print!("Content-Type: text/html\r\n\r\n"); + + let mut html = head("Search - OpenBSD Ports", 0, ""); + html.push_str("

Search Results

\n"); + html.push_str( + "\n", + ); + html.push_str("

Index

\n"); + + if query.is_empty() { + html.push_str("

Enter a search term.

\n"); + html.push_str("\n\n"); + print!("{}", html); + return; + } + + let text = fs::read_to_string(index_path).unwrap_or_default(); + + html.push_str( + "\n\ + \n", + ); + + let mut count = 0; + for line in text.lines() { + let lower = line.to_lowercase(); + if !lower.contains(&query) { + continue; + } + let fields: Vec<&str> = line.splitn(4, '\t').collect(); + if fields.len() < 3 { + continue; + } + let path = fields[0]; + let comment = fields[2]; + let maintainer = if fields.len() > 3 { fields[3] } else { "" }; + + let (cat, name) = match path.split_once('/') { + Some(p) => p, + None => continue, + }; + + html.push_str(&format!( + "\n", + html_escape(cat), + html_escape(name), + html_escape(path), + html_escape(comment), + html_escape(maintainer), + )); + count += 1; + } + + html.push_str("

Port	Description	Maintainer
\ + {}	{}	{}

\n"); + html.push_str(&format!( + "

{} results for \"{}\"

\n", + count, + html_escape(&query) + )); + html.push_str("\n\n"); + print!("{}", html); +} + +fn err(msg: &str) -> ! { + eprintln!("{}: {}", NAME, msg); + std::process::exit(1); +} + +fn errio(path: &Path, e: &std::io::Error) -> ! { + err(&format!( + "{}: {}", + path.display(), + e.kind() + .to_string() + .replace("entity not found", "No such file or directory") + .replace("permission denied", "Permission denied") + )) +} + +fn usage() -> ! { + eprintln!("usage: {} [-d dump] [-p portsdir] [-o outdir]", NAME); + std::process::exit(1); +} + +fn main() { + // CGI mode: QUERY_STRING is set by the HTTP server + if std::env::var("QUERY_STRING").is_ok() { + let index = PathBuf::from("INDEX.txt"); + cgi_search(&index); + return; + } + + let mut portsdir = PathBuf::from("/usr/ports"); + let mut outdir = PathBuf::from("site"); + let mut dumpfile: Option = None; + + let args: Vec = std::env::args().collect(); + let mut i = 1; + while i < args.len() { + match args[i].as_str() { + "-d" => { + i += 1; + if i >= args.len() { + usage(); + } + dumpfile = Some(PathBuf::from(&args[i])); + } + "-p" => { + i += 1; + if i >= args.len() { + usage(); + } + portsdir = PathBuf::from(&args[i]); + } + "-o" => { + i += 1; + if i >= args.len() { + usage(); + } + outdir = PathBuf::from(&args[i]); + } + _ => usage(), + } + i += 1; + } + + let ports = if let Some(ref dump) = dumpfile { + eprintln!("loading dump from {}", dump.display()); + let pd = if portsdir.is_dir() { + Some(portsdir.as_path()) + } else { + None + }; + load_dump(dump, pd) + } else { + if !portsdir.is_dir() { + err(&format!("{}: not a directory", portsdir.display())); + } + eprintln!("parsing ports from {}", portsdir.display()); + walk_ports(&portsdir) + }; + eprintln!("{} ports found", ports.len()); + + eprintln!("generating site in {}", outdir.display()); + fs::create_dir_all(outdir.join("categories")).ok(); + fs::create_dir_all(outdir.join("search")).ok(); + for p in &ports { + fs::create_dir_all(outdir.join("ports").join(&p.category)).ok(); + } + copy_assets(&outdir); + let ft = footer(ports.len()); + write_index(&ports, &outdir, &ft); + write_categories(&ports, &outdir, &ft); + write_port_pages(&ports, &outdir, &ft); + write_alpha(&ports, &outdir, &ft); + write_index_txt(&ports, &outdir); + eprintln!("done"); +}