/* $NetBSD: makemandb.c,v 1.60.2.1 2022/06/06 11:34:11 martin Exp $ */ /* * Copyright (c) 2011 Abhinav Upadhyay * Copyright (c) 2011 Kristaps Dzonsons * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __RCSID("$NetBSD: makemandb.c,v 1.60.2.1 2022/06/06 11:34:11 martin Exp $"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include "apropos-utils.h" #include "dist/man.h" #include "dist/mandoc.h" #include "dist/mdoc.h" #include "dist/roff.h" #include "dist/mandoc_parse.h" #define BUFLEN 1024 #define MDOC 0 //If the page is of mdoc(7) type #define MAN 1 //If the page is of man(7) type /* * A data structure for holding section specific data. */ typedef struct secbuff { char *data; size_t buflen; //Total length of buffer allocated initially size_t offset; // Current offset in the buffer. } secbuff; typedef struct makemandb_flags { int optimize; int limit; // limit the indexing to only NAME section int recreate; // Database was created from scratch int verbosity; // 0: quiet, 1: default, 2: verbose } makemandb_flags; typedef struct roff_mandb_rec { /* Fields for mandb table */ char *name; // for storing the name of the man page char *name_desc; // for storing the one line description (.Nd) secbuff desc; // for storing the DESCRIPTION section secbuff lib; // for the LIBRARY section secbuff return_vals; // RETURN VALUES secbuff env; // ENVIRONMENT secbuff files; // FILES secbuff exit_status; // EXIT STATUS secbuff diagnostics; // DIAGNOSTICS secbuff errors; // ERRORS char *section; int xr_found; // To track whether a .Xr was seen when parsing a section /* Fields for mandb_meta table */ char *md5_hash; dev_t device; ino_t inode; time_t mtime; /* Fields for mandb_links table */ char *machine; char *links; //all the links to a page in a space separated form char *file_path; /* Non-db fields */ int page_type; //Indicates the type of page: mdoc or man } mandb_rec; typedef void (*proff_nf)(const struct roff_node *n, mandb_rec *); static void append(secbuff *sbuff, const char *src); static void init_secbuffs(mandb_rec *); static void free_secbuffs(mandb_rec *); static int check_md5(const char *, sqlite3 *, char **, void *, size_t); static void cleanup(mandb_rec *); static void set_section(const struct roff_meta *, mandb_rec *); static void set_machine(const struct roff_meta *, mandb_rec *); static int insert_into_db(sqlite3 *, mandb_rec *); static void begin_parse(const char *, struct mparse *, mandb_rec *, int); static void proff_node(const struct roff_node *, mandb_rec *, struct roff_meta *, const proff_nf *); static void pmdoc_Nm(const struct roff_node *, mandb_rec *); static void pmdoc_Nd(const struct roff_node *, mandb_rec *); static void pmdoc_Sh(const struct roff_node *, mandb_rec *); static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *); static void pmdoc_Xr(const struct roff_node *, mandb_rec *); static void pmdoc_Pp(const struct roff_node *, mandb_rec *); static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int); static void pman_parse_node(const struct roff_node *, secbuff *); static void pman_parse_name(const struct roff_node *, mandb_rec *); static void pman_sh(const struct roff_node *, mandb_rec *); static void pman_block(const struct roff_node *, mandb_rec *); static void traversedir(const char *, const char *, sqlite3 *, struct mparse *); static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *); static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *); static void build_file_cache(sqlite3 *, const char *, const char *, struct stat *); static void update_db(sqlite3 *, struct mparse *, mandb_rec *); __dead static void usage(void); static void optimize(sqlite3 *); static char *parse_escape(const char *); static void replace_hyph(char *); static makemandb_flags mflags = { .verbosity = 1 }; static const proff_nf mdocs[MDOC_MAX - MDOC_Dd] = { NULL, /* Dd */ NULL, /* Dt */ NULL, /* Os */ pmdoc_Sh, /* Sh */ NULL, /* Ss */ pmdoc_Pp, /* Pp */ NULL, /* D1 */ NULL, /* Dl */ NULL, /* Bd */ NULL, /* Ed */ NULL, /* Bl */ NULL, /* El */ NULL, /* It */ NULL, /* Ad */ NULL, /* An */ NULL, /* Ap */ NULL, /* Ar */ NULL, /* Cd */ NULL, /* Cm */ NULL, /* Dv */ NULL, /* Er */ NULL, /* Ev */ NULL, /* Ex */ NULL, /* Fa */ NULL, /* Fd */ NULL, /* Fl */ NULL, /* Fn */ NULL, /* Ft */ NULL, /* Ic */ NULL, /* In */ NULL, /* Li */ pmdoc_Nd, /* Nd */ pmdoc_Nm, /* Nm */ NULL, /* Op */ NULL, /* Ot */ NULL, /* Pa */ NULL, /* Rv */ NULL, /* St */ NULL, /* Va */ NULL, /* Vt */ pmdoc_Xr, /* Xr */ NULL, /* %A */ NULL, /* %B */ NULL, /* %D */ NULL, /* %I */ NULL, /* %J */ NULL, /* %N */ NULL, /* %O */ NULL, /* %P */ NULL, /* %R */ NULL, /* %T */ NULL, /* %V */ NULL, /* Ac */ NULL, /* Ao */ NULL, /* Aq */ NULL, /* At */ NULL, /* Bc */ NULL, /* Bf */ NULL, /* Bo */ NULL, /* Bq */ NULL, /* Bsx */ NULL, /* Bx */ NULL, /* Db */ NULL, /* Dc */ NULL, /* Do */ NULL, /* Dq */ NULL, /* Ec */ NULL, /* Ef */ NULL, /* Em */ NULL, /* Eo */ NULL, /* Fx */ NULL, /* Ms */ NULL, /* No */ NULL, /* Ns */ NULL, /* Nx */ NULL, /* Ox */ NULL, /* Pc */ NULL, /* Pf */ NULL, /* Po */ NULL, /* Pq */ NULL, /* Qc */ NULL, /* Ql */ NULL, /* Qo */ NULL, /* Qq */ NULL, /* Re */ NULL, /* Rs */ NULL, /* Sc */ NULL, /* So */ NULL, /* Sq */ NULL, /* Sm */ NULL, /* Sx */ NULL, /* Sy */ NULL, /* Tn */ NULL, /* Ux */ NULL, /* Xc */ NULL, /* Xo */ NULL, /* Fo */ NULL, /* Fc */ NULL, /* Oo */ NULL, /* Oc */ NULL, /* Bk */ NULL, /* Ek */ NULL, /* Bt */ NULL, /* Hf */ NULL, /* Fr */ NULL, /* Ud */ NULL, /* Lb */ NULL, /* Lp */ NULL, /* Lk */ NULL, /* Mt */ NULL, /* Brq */ NULL, /* Bro */ NULL, /* Brc */ NULL, /* %C */ NULL, /* Es */ NULL, /* En */ NULL, /* Dx */ NULL, /* %Q */ NULL, /* %U */ NULL /* Ta */ }; static const proff_nf mans[MAN_MAX - MAN_TH] = { NULL, //TH pman_sh, //SH NULL, //SS NULL, //TP NULL, //LP NULL, //PP NULL, //P NULL, //IP NULL, //HP NULL, //SM NULL, //SB NULL, //BI NULL, //IB NULL, //BR NULL, //RB NULL, //R pman_block, //B NULL, //I NULL, //IR NULL, //RI NULL, //nf NULL, //fi NULL, //RE NULL, //RS NULL, //DT NULL, //UC NULL, //PD NULL, //AT NULL, //in NULL, //OP NULL, //EX NULL, //EE NULL, //UR NULL, //UE NULL, //MT NULL //ME }; int main(int argc, char *argv[]) { FILE *file; const char *sqlstr, *manconf = NULL; char *line, *command; char *errmsg; int ch; struct mparse *mp; sqlite3 *db; ssize_t len; size_t linesize; struct roff_mandb_rec rec; while ((ch = getopt(argc, argv, "C:floQqv")) != -1) { switch (ch) { case 'C': manconf = optarg; break; case 'f': mflags.recreate = 1; break; case 'l': mflags.limit = 1; break; case 'o': mflags.optimize = 1; break; case 'Q': mflags.verbosity = 0; break; case 'q': mflags.verbosity = 1; break; case 'v': mflags.verbosity = 2; break; default: usage(); } } memset(&rec, 0, sizeof(rec)); init_secbuffs(&rec); mchars_alloc(); mp = mparse_alloc(MPARSE_SO | MPARSE_UTF8 | MPARSE_LATIN1 | MPARSE_VALIDATE, MANDOC_OS_OTHER, NULL); if (manconf) { char *arg; size_t command_len = shquote(manconf, NULL, 0) + 1; arg = emalloc(command_len); shquote(manconf, arg, command_len); easprintf(&command, "man -p -C %s", arg); free(arg); } else { command = estrdup("man -p"); manconf = MANCONF; } if (mflags.recreate) { char *dbp = get_dbpath(manconf); /* No error here, it will fail in init_db in the same call */ if (dbp != NULL) remove(dbp); } if ((db = init_db(MANDB_CREATE, manconf)) == NULL) exit(EXIT_FAILURE); sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg); if (errmsg != NULL) { warnx("%s", errmsg); free(errmsg); close_db(db); exit(EXIT_FAILURE); } sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL, &errmsg); if (errmsg != NULL) { warnx("%s", errmsg); free(errmsg); close_db(db); exit(EXIT_FAILURE); } /* Call man -p to get the list of man page dirs */ if ((file = popen(command, "r")) == NULL) { close_db(db); err(EXIT_FAILURE, "fopen failed"); } free(command); /* Begin the transaction for indexing the pages */ sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg); if (errmsg != NULL) { warnx("%s", errmsg); free(errmsg); close_db(db); exit(EXIT_FAILURE); } sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent," " file PRIMARY KEY);" "CREATE UNIQUE INDEX metadb.index_file_cache_dev" " ON file_cache (device, inode)"; sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); if (errmsg != NULL) { warnx("%s", errmsg); free(errmsg); close_db(db); exit(EXIT_FAILURE); } if (mflags.verbosity) printf("Building temporary file cache\n"); line = NULL; linesize = 0; while ((len = getline(&line, &linesize, file)) != -1) { /* Replace the new line character at the end of string with '\0' */ line[len - 1] = '\0'; char *pdir = estrdup(dirname(line)); /* Traverse the man page directories and parse the pages */ traversedir(pdir, line, db, mp); free(pdir); } free(line); if (pclose(file) == -1) { close_db(db); cleanup(&rec); free_secbuffs(&rec); err(EXIT_FAILURE, "pclose error"); } if (mflags.verbosity) printf("Performing index update\n"); update_db(db, mp, &rec); mparse_free(mp); mchars_free(); free_secbuffs(&rec); /* Commit the transaction */ sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg); if (errmsg != NULL) { warnx("%s", errmsg); free(errmsg); close_db(db); exit(EXIT_FAILURE); } if (mflags.optimize) optimize(db); close_db(db); return 0; } /* * traversedir -- * Traverses the given directory recursively and passes all the man page files * in the way to build_file_cache() */ static void traversedir(const char *parent, const char *file, sqlite3 *db, struct mparse *mp) { struct stat sb; struct dirent *dirp; DIR *dp; char *buf; if (stat(file, &sb) < 0) { if (mflags.verbosity) warn("stat failed: %s", file); return; } /* If it is a directory, traverse it recursively */ if (S_ISDIR(sb.st_mode)) { if ((dp = opendir(file)) == NULL) { if (mflags.verbosity) warn("opendir error: %s", file); return; } while ((dirp = readdir(dp)) != NULL) { /* Avoid . and .. entries in a directory */ if (dirp->d_name[0] != '.') { easprintf(&buf, "%s/%s", file, dirp->d_name); traversedir(parent, buf, db, mp); free(buf); } } closedir(dp); return; } if (!S_ISREG(sb.st_mode)) return; if (sb.st_size == 0) { if (mflags.verbosity) warnx("Empty file: %s", file); return; } build_file_cache(db, parent, file, &sb); } /* build_file_cache -- * This function generates an md5 hash of the file passed as its 2nd parameter * and stores it in a temporary table file_cache along with the full file path. * This is done to support incremental updation of the database. * The temporary table file_cache is dropped thereafter in the function * update_db(), once the database has been updated. */ static void build_file_cache(sqlite3 *db, const char *parent, const char *file, struct stat *sb) { const char *sqlstr; sqlite3_stmt *stmt = NULL; int rc, idx; assert(file != NULL); dev_t device_cache = sb->st_dev; ino_t inode_cache = sb->st_ino; time_t mtime_cache = sb->st_mtime; sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode," " :mtime, :parent, :file)"; rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); return; } idx = sqlite3_bind_parameter_index(stmt, ":device"); rc = sqlite3_bind_int64(stmt, idx, device_cache); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); sqlite3_finalize(stmt); return; } idx = sqlite3_bind_parameter_index(stmt, ":inode"); rc = sqlite3_bind_int64(stmt, idx, inode_cache); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); sqlite3_finalize(stmt); return; } idx = sqlite3_bind_parameter_index(stmt, ":mtime"); rc = sqlite3_bind_int64(stmt, idx, mtime_cache); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); sqlite3_finalize(stmt); return; } idx = sqlite3_bind_parameter_index(stmt, ":parent"); rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); sqlite3_finalize(stmt); return; } idx = sqlite3_bind_parameter_index(stmt, ":file"); rc = sqlite3_bind_text(stmt, idx, file, -1, NULL); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); sqlite3_finalize(stmt); return; } sqlite3_step(stmt); sqlite3_finalize(stmt); } /* read_and_decompress -- * Reads the given file into memory. If it is compressed, decompress * it before returning to the caller. */ static int read_and_decompress(const char *file, void **bufp, size_t *len) { size_t off; ssize_t r; struct archive *a; struct archive_entry *ae; char *buf; if ((a = archive_read_new()) == NULL) errx(EXIT_FAILURE, "memory allocation failed"); *bufp = NULL; if (archive_read_support_filter_all(a) != ARCHIVE_OK || archive_read_support_format_raw(a) != ARCHIVE_OK || archive_read_open_filename(a, file, 65536) != ARCHIVE_OK || archive_read_next_header(a, &ae) != ARCHIVE_OK) goto archive_error; *len = 65536; buf = emalloc(*len); off = 0; for (;;) { r = archive_read_data(a, buf + off, *len - off); if (r == ARCHIVE_OK) { archive_read_free(a); *bufp = buf; *len = off; return 0; } if (r <= 0) { free(buf); break; } off += r; if (off == *len) { *len *= 2; if (*len < off) { if (mflags.verbosity) warnx("File too large: %s", file); free(buf); archive_read_free(a); return -1; } buf = erealloc(buf, *len); } } archive_error: warnx("Error while reading `%s': %s", file, archive_error_string(a)); archive_read_free(a); return -1; } static void update_existing_entry(sqlite3 *db, const char *file, const char *hash, mandb_rec *rec, int *new_count, int *link_count, int *err_count) { int update_count, rc, idx; const char *inner_sqlstr; sqlite3_stmt *inner_stmt; update_count = sqlite3_total_changes(db); inner_sqlstr = "UPDATE mandb_meta SET device = :device," " inode = :inode, mtime = :mtime WHERE" " md5_hash = :md5 AND file = :file AND" " (device <> :device2 OR inode <> " " :inode2 OR mtime <> :mtime2)"; rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); return; } idx = sqlite3_bind_parameter_index(inner_stmt, ":device"); sqlite3_bind_int64(inner_stmt, idx, rec->device); idx = sqlite3_bind_parameter_index(inner_stmt, ":inode"); sqlite3_bind_int64(inner_stmt, idx, rec->inode); idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime"); sqlite3_bind_int64(inner_stmt, idx, rec->mtime); idx = sqlite3_bind_parameter_index(inner_stmt, ":md5"); sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL); idx = sqlite3_bind_parameter_index(inner_stmt, ":file"); sqlite3_bind_text(inner_stmt, idx, file, -1, NULL); idx = sqlite3_bind_parameter_index(inner_stmt, ":device2"); sqlite3_bind_int64(inner_stmt, idx, rec->device); idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2"); sqlite3_bind_int64(inner_stmt, idx, rec->inode); idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2"); sqlite3_bind_int64(inner_stmt, idx, rec->mtime); rc = sqlite3_step(inner_stmt); if (rc == SQLITE_DONE) { /* Check if an update has been performed. */ if (update_count != sqlite3_total_changes(db)) { if (mflags.verbosity == 2) printf("Updated %s\n", file); (*new_count)++; } else { /* Otherwise it was a hardlink. */ (*link_count)++; } } else { if (mflags.verbosity == 2) warnx("Could not update the meta data for %s", file); (*err_count)++; } sqlite3_finalize(inner_stmt); } /* update_db -- * Does an incremental updation of the database by checking the file_cache. * It parses and adds the pages which are present in file_cache, * but not in the database. * It also removes the pages which are present in the databse, * but not in the file_cache. */ static void update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec) { const char *sqlstr; sqlite3_stmt *stmt = NULL; char *file; char *parent; char *errmsg = NULL; char *md5sum; void *buf; size_t buflen; struct sql_row { struct sql_row *next; dev_t device; ino_t inode; time_t mtime; char *parent; char *file; } *rows, *row; int new_count = 0; /* Counter for newly indexed/updated pages */ int total_count = 0; /* Counter for total number of pages */ int err_count = 0; /* Counter for number of failed pages */ int link_count = 0; /* Counter for number of hard/sym links */ int md5_status; int rc; sqlstr = "SELECT device, inode, mtime, parent, file" " FROM metadb.file_cache fc" " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE" " device = fc.device AND inode = fc.inode AND " " mtime = fc.mtime AND file = fc.file)"; rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); close_db(db); errx(EXIT_FAILURE, "Could not query file cache"); } buf = NULL; rows = NULL; while (sqlite3_step(stmt) == SQLITE_ROW) { row = emalloc(sizeof(struct sql_row)); row->device = sqlite3_column_int64(stmt, 0); row->inode = sqlite3_column_int64(stmt, 1); row->mtime = sqlite3_column_int64(stmt, 2); row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3)); row->file = estrdup((const char *) sqlite3_column_text(stmt, 4)); row->next = rows; rows = row; total_count++; } sqlite3_finalize(stmt); for ( ; rows != NULL; free(parent), free(file), free(buf)) { int fd; row = rows; rows = rows->next; rec->device = row->device; rec->inode = row->inode; rec->mtime = row->mtime; parent = row->parent; file = row->file; free(row); // XXX: reading twice! if (read_and_decompress(file, &buf, &buflen)) { err_count++; continue; } if ((fd = mparse_open(mp, file)) == -1) { err_count++; continue; } md5_status = check_md5(file, db, &md5sum, buf, buflen); assert(md5sum != NULL); if (md5_status == -1) { if (mflags.verbosity) warnx("An error occurred in checking md5 value" " for file %s", file); err_count++; close(fd); continue; } if (md5_status == 0) { /* * The MD5 hash is already present in the database, * so simply update the metadata. */ update_existing_entry(db, file, md5sum, rec, &new_count, &link_count, &err_count); free(md5sum); close(fd); continue; } if (md5_status == 1) { /* * The MD5 hash was not present in the database. * This means is either a new file or an updated file. * We should go ahead with parsing. */ if (chdir(parent) == -1) { if (mflags.verbosity) warn("chdir failed for `%s', could " "not index `%s'", parent, file); err_count++; free(md5sum); close(fd); continue; } if (mflags.verbosity == 2) printf("Parsing: %s\n", file); rec->md5_hash = md5sum; rec->file_path = estrdup(file); // file_path is freed by insert_into_db itself. begin_parse(file, mp, rec, fd); if (insert_into_db(db, rec) < 0) { if (mflags.verbosity) warnx("Error in indexing `%s'", file); err_count++; } else { new_count++; } } close(fd); } if (mflags.verbosity == 2) { printf("Number of new or updated pages encountered: %d\n" "Number of hard links found: %d\n" "Number of pages that were successfully" " indexed or updated: %d\n" "Number of pages that could not be indexed" " due to errors: %d\n", total_count - link_count, link_count, new_count, err_count); } if (mflags.recreate) return; if (mflags.verbosity == 2) printf("Deleting stale index entries\n"); sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN" " (SELECT file FROM metadb.file_cache);" "DELETE FROM mandb_links WHERE md5_hash NOT IN" " (SELECT md5_hash from mandb_meta);" "DROP TABLE metadb.file_cache;" "DELETE FROM mandb WHERE rowid NOT IN" " (SELECT id FROM mandb_meta);"; sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); if (errmsg != NULL) { warnx("Removing old entries failed: %s", errmsg); warnx("Please rebuild database from scratch with -f."); free(errmsg); return; } } /* * begin_parse -- * parses the man page using libmandoc */ static void begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, int fd) { struct roff_meta *roff; mparse_reset(mp); rec->xr_found = 0; mparse_readfd(mp, fd, file); roff = mparse_result(mp); if (roff == NULL) { if (mflags.verbosity == 2) warnx("Not a roff(7) page"); return; } if (roff->macroset == MACROSET_MDOC) { rec->page_type = MDOC; proff_node(roff->first->child, rec, roff, mdocs); } else if (roff->macroset == MACROSET_MAN) { rec->page_type = MAN; proff_node(roff->first->child, rec, roff, mans); } else warnx("Unknown macroset %d", roff->macroset); set_machine(roff, rec); set_section(roff, rec); } /* * set_section -- * Extracts the section number and normalizes it to only the numeric part * (Which should be the first character of the string). */ static void set_section(const struct roff_meta *rm, mandb_rec *rec) { if (!rm) return; const char *s = rm->msec == NULL ? "?" : rm->msec; easprintf(&rec->section, "%s", s); if (rec->section[0] == '?' && mflags.verbosity == 2) warnx("%s: Missing section number", rec->file_path); } /* * get_machine -- * Extracts the machine architecture information if available. */ static void set_machine(const struct roff_meta *rm, mandb_rec *rec) { if (rm == NULL) return; if (rm->arch) rec->machine = estrdup(rm->arch); } /* * pmdoc_Nm -- * Extracts the Name of the manual page from the .Nm macro */ static void pmdoc_Nm(const struct roff_node *n, mandb_rec *rec) { if (n->sec != SEC_NAME) return; for (n = n->child; n; n = n->next) { if (n->type == ROFFT_TEXT) { char *escaped_name = parse_escape(n->string); concat(&rec->name, escaped_name); free(escaped_name); } } } /* * pmdoc_Nd -- * Extracts the one line description of the man page from the .Nd macro */ static void pmdoc_Nd(const struct roff_node *n, mandb_rec *rec) { if (n->type == ROFFT_BODY) deroff(&rec->name_desc, n); if (rec->name_desc) replace_hyph(rec->name_desc); } /* * pmdoc_macro_handler-- * This function is a single point of handling all the special macros that we * want to handle especially. For example the .Xr macro for properly parsing * the referenced page name along with the section number, or the .Pp macro * for adding a new line whenever we encounter it. */ static void pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct) { const struct roff_node *sn; assert(n); switch (doct) { /* Parse the man page references. * Basically the .Xr macros are used like: * .Xr ls 1 * and formatted like this: * ls(1) * Prepare a buffer to format the data like the above example and call * pmdoc_parse_section to append it. */ case MDOC_Xr: n = n->child; while (n->type != ROFFT_TEXT && n->next) n = n->next; if (n && n->type != ROFFT_TEXT) return; sn = n; if (n->next) n = n->next; while (n->type != ROFFT_TEXT && n->next) n = n->next; if (n && n->type == ROFFT_TEXT) { char *buf; easprintf(&buf, "%s(%s)", sn->string, n->string); mdoc_parse_section(n->sec, buf, rec); free(buf); } break; /* Parse the .Pp macro to add a new line */ case MDOC_Pp: if (n->type == ROFFT_TEXT) mdoc_parse_section(n->sec, "\n", rec); break; default: break; } } /* * pmdoc_Xr, pmdoc_Pp-- * Empty stubs. * The parser calls these functions each time it encounters * a .Xr or .Pp macro. We are parsing all the data from * the pmdoc_Sh function, so don't do anything here. * (See if else blocks in pmdoc_Sh.) */ static void pmdoc_Xr(const struct roff_node *n, mandb_rec *rec) { } static void pmdoc_Pp(const struct roff_node *n, mandb_rec *rec) { } /* * pmdoc_Sh -- * Called when a .Sh macro is encountered and tries to parse its body */ static void pmdoc_Sh(const struct roff_node *n, mandb_rec *rec) { if (n == NULL) return; switch (n->sec) { case SEC_NAME: case SEC_SYNOPSIS: case SEC_EXAMPLES: case SEC_STANDARDS: case SEC_HISTORY: case SEC_AUTHORS: case SEC_BUGS: /* * We don't care about text from these sections */ return; default: break; } if (n->type == ROFFT_BLOCK) mdoc_parse_Sh(n->body, rec); } /* * Called from pmdoc_Sh to parse body of a .Sh macro. It calls * mdoc_parse_section to append the data to the section specific buffer. * The .Xr macro needs special handling, thus the separate if branch for it. */ static void mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec) { if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX)) return; int xr_found = 0; if (n->type == ROFFT_TEXT) { mdoc_parse_section(n->sec, n->string, rec); } else if (n->tok >= MDOC_Dd && n->tok < MDOC_MAX) { const int tok_idx = n->tok - MDOC_Dd; if (mdocs[tok_idx] == pmdoc_Xr) { /* * When encountering other inline macros, * call pmdoc_macro_handler. */ pmdoc_macro_handler(n, rec, MDOC_Xr); xr_found = 1; } else if (mdocs[tok_idx] == pmdoc_Pp) { pmdoc_macro_handler(n, rec, MDOC_Pp); } } /* * If an Xr macro was encountered then the child node has * already been explored by pmdoc_macro_handler. */ if (xr_found == 0) mdoc_parse_Sh(n->child, rec); mdoc_parse_Sh(n->next, rec); } /* * mdoc_parse_section-- * Utility function for parsing sections of the mdoc type pages. * Takes two params: * 1. sec is an enum which indicates the section in which we are present * 2. string is the string which we need to append to the secbuff for this * particular section. * The function appends string to the global section buffer and returns. */ static void mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec) { /* * If the user specified the 'l' flag, then parse and store only the * NAME section. Ignore the rest. */ if (mflags.limit) return; switch (sec) { case SEC_LIBRARY: append(&rec->lib, string); break; case SEC_RETURN_VALUES: append(&rec->return_vals, string); break; case SEC_ENVIRONMENT: append(&rec->env, string); break; case SEC_FILES: append(&rec->files, string); break; case SEC_EXIT_STATUS: append(&rec->exit_status, string); break; case SEC_DIAGNOSTICS: append(&rec->diagnostics, string); break; case SEC_ERRORS: append(&rec->errors, string); break; default: append(&rec->desc, string); break; } } static void proff_node(const struct roff_node * n, mandb_rec * rec, struct roff_meta * roff, const proff_nf * func) { if (n == NULL) return; int tok_idx; switch (n->type) { case (ROFFT_BODY): /* FALLTHROUGH */ case (ROFFT_BLOCK): /* FALLTHROUGH */ case (ROFFT_ELEM): if (roff->macroset == MACROSET_MAN) tok_idx = n->tok - MAN_TH; else if (roff->macroset == MACROSET_MDOC) tok_idx = n->tok - MDOC_Dd; else tok_idx = -1; if (tok_idx >= 0 && func[tok_idx] != NULL) (*func[tok_idx]) (n, rec); break; default: break; } proff_node(n->child, rec, roff, func); proff_node(n->next, rec, roff, func); } /* * pman_parse_name -- * Parses the NAME section and puts the complete content in the name_desc * variable. */ static void pman_parse_name(const struct roff_node *n, mandb_rec *rec) { if (n == NULL) return; if (n->type == ROFFT_TEXT) { char *tmp = parse_escape(n->string); concat(&rec->name_desc, tmp); free(tmp); } if (n->child) pman_parse_name(n->child, rec); if(n->next) pman_parse_name(n->next, rec); } /* * A stub function to be able to parse the macros like .B embedded inside * a section. */ static void pman_block(const struct roff_node *n, mandb_rec *rec) { } /* * pman_sh -- * This function does one of the two things: * 1. If the present section is NAME, then it will: * (a) Extract the name of the page (in case of multiple comma separated * names, it will pick up the first one). * (b) Build a space spearated list of all the symlinks/hardlinks to * this page and store in the buffer 'links'. These are extracted from * the comma separated list of names in the NAME section as well. * (c) Move on to the one line description section, which is after the list * of names in the NAME section. * 2. Otherwise, it will check the section name and call the man_parse_section * function, passing the enum corresponding to that section. */ static void pman_sh(const struct roff_node *n, mandb_rec *rec) { static const struct { enum man_sec section; const char *header; } mapping[] = { { MANSEC_DESCRIPTION, "DESCRIPTION" }, { MANSEC_SYNOPSIS, "SYNOPSIS" }, { MANSEC_LIBRARY, "LIBRARY" }, { MANSEC_ERRORS, "ERRORS" }, { MANSEC_FILES, "FILES" }, { MANSEC_RETURN_VALUES, "RETURN VALUE" }, { MANSEC_RETURN_VALUES, "RETURN VALUES" }, { MANSEC_EXIT_STATUS, "EXIT STATUS" }, { MANSEC_EXAMPLES, "EXAMPLES" }, { MANSEC_EXAMPLES, "EXAMPLE" }, { MANSEC_STANDARDS, "STANDARDS" }, { MANSEC_HISTORY, "HISTORY" }, { MANSEC_BUGS, "BUGS" }, { MANSEC_AUTHORS, "AUTHORS" }, { MANSEC_COPYRIGHT, "COPYRIGHT" }, }; const struct roff_node *head; char *name_desc; size_t sz; size_t i; if ((head = n->parent->head) == NULL || (head = head->child) == NULL || head->type != ROFFT_TEXT) return; /* * Check if this section should be extracted and * where it should be stored. Handled the trival cases first. */ for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) { if (strcmp(head->string, mapping[i].header) == 0) { man_parse_section(mapping[i].section, n, rec); return; } } if (strcmp(head->string, "NAME") == 0) { /* * We are in the NAME section. * pman_parse_name will put the complete content in name_desc. */ pman_parse_name(n, rec); name_desc = rec->name_desc; if (name_desc == NULL) return; /* Remove any leading spaces. */ while (name_desc[0] == ' ') name_desc++; /* If the line begins with a "\&", avoid those */ if (name_desc[0] == '\\' && name_desc[1] == '&') name_desc += 2; /* Now name_desc should be left with a comma-space * separated list of names and the one line description * of the page: * "a, b, c \- sample description" * Take out the first name, before the first comma * (or space) and store it in rec->name. * If the page has aliases then they should be * in the form of a comma separated list. * Keep looping while there is a comma in name_desc, * extract the alias name and store in rec->links. * When there are no more commas left, break out. */ int has_alias = 0; // Any more aliases left? while (*name_desc) { /* Remove any leading spaces or hyphens. */ if (name_desc[0] == ' ' || name_desc[0] == '-') { name_desc++; continue; } sz = strcspn(name_desc, ", "); /* Extract the first term and store it in rec->name. */ if (rec->name == NULL) { if (name_desc[sz] == ',') has_alias = 1; rec->name = estrndup(name_desc, sz); /* XXX This would only happen with a poorly * written man page, maybe warn? */ if (name_desc[sz] == '\0') break; name_desc += sz + 1; continue; } /* * Once rec->name is set, rest of the names * are to be treated as links or aliases. */ if (rec->name && has_alias) { if (name_desc[sz] != ',') { /* No more commas left --> no more * aliases to take out */ has_alias = 0; } concat2(&rec->links, name_desc, sz); /* XXX This would only happen with a poorly * written man page, maybe warn? */ if (name_desc[sz] == '\0') break; name_desc += sz + 1; continue; } break; } /* Parse any escape sequences that might be there */ char *temp = parse_escape(name_desc); free(rec->name_desc); rec->name_desc = temp; temp = parse_escape(rec->name); free(rec->name); rec->name = temp; return; } /* The RETURN VALUE section might be specified in multiple ways */ if (strcmp(head->string, "RETURN") == 0 && head->next != NULL && head->next->type == ROFFT_TEXT && (strcmp(head->next->string, "VALUE") == 0 || strcmp(head->next->string, "VALUES") == 0)) { man_parse_section(MANSEC_RETURN_VALUES, n, rec); return; } /* * EXIT STATUS section can also be specified all on one line or on two * separate lines. */ if (strcmp(head->string, "EXIT") == 0 && head->next != NULL && head->next->type == ROFFT_TEXT && strcmp(head->next->string, "STATUS") == 0) { man_parse_section(MANSEC_EXIT_STATUS, n, rec); return; } /* Store the rest of the content in desc. */ man_parse_section(MANSEC_NONE, n, rec); } /* * pman_parse_node -- * Generic function to iterate through a node. Usually called from * man_parse_section to parse a particular section of the man page. */ static void pman_parse_node(const struct roff_node *n, secbuff *s) { if (n == NULL) return; if (n->type == ROFFT_TEXT) append(s, n->string); pman_parse_node(n->child, s); pman_parse_node(n->next, s); } /* * man_parse_section -- * Takes two parameters: * sec: Tells which section we are present in * n: Is the present node of the AST. * Depending on the section, we call pman_parse_node to parse that section and * concatenate the content from that section into the buffer for that section. */ static void man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec) { /* * If the user sepecified the 'l' flag then just parse * the NAME section, ignore the rest. */ if (mflags.limit) return; switch (sec) { case MANSEC_LIBRARY: pman_parse_node(n, &rec->lib); break; case MANSEC_RETURN_VALUES: pman_parse_node(n, &rec->return_vals); break; case MANSEC_ENVIRONMENT: pman_parse_node(n, &rec->env); break; case MANSEC_FILES: pman_parse_node(n, &rec->files); break; case MANSEC_EXIT_STATUS: pman_parse_node(n, &rec->exit_status); break; case MANSEC_DIAGNOSTICS: pman_parse_node(n, &rec->diagnostics); break; case MANSEC_ERRORS: pman_parse_node(n, &rec->errors); break; case MANSEC_NAME: case MANSEC_SYNOPSIS: case MANSEC_EXAMPLES: case MANSEC_STANDARDS: case MANSEC_HISTORY: case MANSEC_BUGS: case MANSEC_AUTHORS: case MANSEC_COPYRIGHT: break; default: pman_parse_node(n, &rec->desc); break; } } /* * insert_into_db -- * Inserts the parsed data of the man page in the Sqlite databse. * If any of the values is NULL, then we cleanup and return -1 indicating * an error. * Otherwise, store the data in the database and return 0. */ static int insert_into_db(sqlite3 *db, mandb_rec *rec) { int rc = 0; int idx = -1; const char *sqlstr = NULL; sqlite3_stmt *stmt = NULL; char *ln = NULL; char *errmsg = NULL; long int mandb_rowid; /* * At the very minimum we want to make sure that we store * the following data: * Name, one line description, and the MD5 hash */ if (rec->name == NULL || rec->name_desc == NULL || rec->md5_hash == NULL) { cleanup(rec); return -1; } /* Write null byte at the end of all the sec_buffs */ rec->desc.data[rec->desc.offset] = 0; rec->lib.data[rec->lib.offset] = 0; rec->env.data[rec->env.offset] = 0; rec->return_vals.data[rec->return_vals.offset] = 0; rec->exit_status.data[rec->exit_status.offset] = 0; rec->files.data[rec->files.offset] = 0; rec->diagnostics.data[rec->diagnostics.offset] = 0; rec->errors.data[rec->errors.offset] = 0; /* * In case of a mdoc page: (sorry, no better place to put this code) * parse the comma separated list of names of man pages, * the first name will be stored in the mandb table, rest will be * treated as links and put in the mandb_links table. */ if (rec->page_type == MDOC) { char *tmp; rec->links = estrdup(rec->name); free(rec->name); size_t sz = strcspn(rec->links, " \0"); rec->name = emalloc(sz + 1); memcpy(rec->name, rec->links, sz); if(rec->name[sz - 1] == ',') rec->name[sz - 1] = 0; else rec->name[sz] = 0; while (rec->links[sz] == ' ') ++sz; tmp = estrdup(rec->links + sz); free(rec->links); rec->links = tmp; } /*------------------------ Populate the mandb table---------------------------*/ sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc," " :lib, :return_vals, :env, :files, :exit_status," " :diagnostics, :errors, :md5_hash, :machine)"; rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); if (rc != SQLITE_OK) goto Out; idx = sqlite3_bind_parameter_index(stmt, ":name"); rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":section"); rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":name_desc"); rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":desc"); rc = sqlite3_bind_text(stmt, idx, rec->desc.data, rec->desc.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":lib"); rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":return_vals"); rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data, rec->return_vals.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":env"); rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":files"); rc = sqlite3_bind_text(stmt, idx, rec->files.data, rec->files.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":exit_status"); rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data, rec->exit_status.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":diagnostics"); rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data, rec->diagnostics.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":errors"); rc = sqlite3_bind_text(stmt, idx, rec->errors.data, rec->errors.offset + 1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":machine"); if (rec->machine) rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL); else rc = sqlite3_bind_null(stmt, idx); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } rc = sqlite3_step(stmt); if (rc != SQLITE_DONE) { sqlite3_finalize(stmt); goto Out; } sqlite3_finalize(stmt); /* Get the row id of the last inserted row */ mandb_rowid = sqlite3_last_insert_rowid(db); /*------------------------Populate the mandb_meta table-----------------------*/ sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime," " :file, :md5_hash, :id)"; rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); if (rc != SQLITE_OK) goto Out; idx = sqlite3_bind_parameter_index(stmt, ":device"); rc = sqlite3_bind_int64(stmt, idx, rec->device); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":inode"); rc = sqlite3_bind_int64(stmt, idx, rec->inode); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":mtime"); rc = sqlite3_bind_int64(stmt, idx, rec->mtime); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":file"); rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } idx = sqlite3_bind_parameter_index(stmt, ":id"); rc = sqlite3_bind_int64(stmt, idx, mandb_rowid); if (rc != SQLITE_OK) { sqlite3_finalize(stmt); goto Out; } rc = sqlite3_step(stmt); sqlite3_finalize(stmt); if (rc == SQLITE_CONSTRAINT_UNIQUE) { /* The *most* probable reason for reaching here is that * the UNIQUE contraint on the file column of the mandb_meta * table was violated. * This can happen when a file was updated/modified. * To fix this we need to do two things: * 1. Delete the row for the older version of this file * from mandb table. * 2. Run an UPDATE query to update the row for this file * in the mandb_meta table. */ warnx("Trying to update index for %s", rec->file_path); char *sql = sqlite3_mprintf("DELETE FROM mandb " "WHERE rowid = (SELECT id" " FROM mandb_meta" " WHERE file = %Q)", rec->file_path); sqlite3_exec(db, sql, NULL, NULL, &errmsg); sqlite3_free(sql); if (errmsg != NULL) { if (mflags.verbosity) warnx("%s", errmsg); free(errmsg); } sqlstr = "UPDATE mandb_meta SET device = :device," " inode = :inode, mtime = :mtime, id = :id," " md5_hash = :md5 WHERE file = :file"; rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("Update failed with error: %s", sqlite3_errmsg(db)); close_db(db); cleanup(rec); errx(EXIT_FAILURE, "Consider running makemandb with -f option"); } idx = sqlite3_bind_parameter_index(stmt, ":device"); sqlite3_bind_int64(stmt, idx, rec->device); idx = sqlite3_bind_parameter_index(stmt, ":inode"); sqlite3_bind_int64(stmt, idx, rec->inode); idx = sqlite3_bind_parameter_index(stmt, ":mtime"); sqlite3_bind_int64(stmt, idx, rec->mtime); idx = sqlite3_bind_parameter_index(stmt, ":id"); sqlite3_bind_int64(stmt, idx, mandb_rowid); idx = sqlite3_bind_parameter_index(stmt, ":md5"); sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); idx = sqlite3_bind_parameter_index(stmt, ":file"); sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); rc = sqlite3_step(stmt); sqlite3_finalize(stmt); if (rc != SQLITE_DONE) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); close_db(db); cleanup(rec); errx(EXIT_FAILURE, "Consider running makemandb with -f option"); } } else if (rc != SQLITE_DONE) { /* Otherwise make this error fatal */ warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db)); cleanup(rec); close_db(db); exit(EXIT_FAILURE); } /*------------------------ Populate the mandb_links table---------------------*/ char *str = NULL; char *links; if (rec->links && strlen(rec->links)) { links = rec->links; for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) { if (ln[0] == ',') ln++; if(ln[strlen(ln) - 1] == ',') ln[strlen(ln) - 1] = 0; str = sqlite3_mprintf("INSERT INTO mandb_links" " VALUES (%Q, %Q, %Q, %Q, %Q, %Q)", ln, rec->name, rec->section, rec->machine, rec->md5_hash, rec->name_desc); sqlite3_exec(db, str, NULL, NULL, &errmsg); sqlite3_free(str); if (errmsg != NULL) { warnx("%s", errmsg); cleanup(rec); free(errmsg); return -1; } } } cleanup(rec); return 0; Out: if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); cleanup(rec); return -1; } /* * check_md5-- * Generates the md5 hash of the file and checks if it already doesn't exist * in the table. * This function is being used to avoid hardlinks. * On successful completion it will also set the value of the fourth parameter * to the md5 hash of the file (computed previously). It is the responsibility * of the caller to free this buffer. * Return values: * -1: If an error occurs somewhere and sets the md5 return buffer to NULL. * 0: If the md5 hash does not exist in the table. * 1: If the hash exists in the database. */ static int check_md5(const char *file, sqlite3 *db, char **md5, void *buf, size_t buflen) { int rc = 0; int idx = -1; char *sqlstr = NULL; char *mymd5; sqlite3_stmt *stmt = NULL; *md5 = NULL; assert(file != NULL); if ((mymd5 = MD5Data(buf, buflen, NULL)) == NULL) { if (mflags.verbosity) warn("md5 failed: %s", file); return -1; } easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash"); rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); if (rc != SQLITE_OK) { free(sqlstr); free(mymd5); return -1; } idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); rc = sqlite3_bind_text(stmt, idx, mymd5, -1, NULL); if (rc != SQLITE_OK) { if (mflags.verbosity) warnx("%s", sqlite3_errmsg(db)); sqlite3_finalize(stmt); free(sqlstr); free(mymd5); return -1; } *md5 = mymd5; if (sqlite3_step(stmt) == SQLITE_ROW) { sqlite3_finalize(stmt); free(sqlstr); return 0; } sqlite3_finalize(stmt); free(sqlstr); return 1; } /* Optimize the index for faster search */ static void optimize(sqlite3 *db) { const char *sqlstr; char *errmsg = NULL; if (mflags.verbosity == 2) printf("Optimizing the database index\n"); sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');" "VACUUM"; sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); if (errmsg != NULL) { if (mflags.verbosity) warnx("%s", errmsg); free(errmsg); return; } } /* * cleanup -- * cleans up the global buffers */ static void cleanup(mandb_rec *rec) { rec->desc.offset = 0; rec->lib.offset = 0; rec->return_vals.offset = 0; rec->env.offset = 0; rec->exit_status.offset = 0; rec->diagnostics.offset = 0; rec->errors.offset = 0; rec->files.offset = 0; free(rec->machine); rec->machine = NULL; free(rec->links); rec->links = NULL; free(rec->file_path); rec->file_path = NULL; free(rec->name); rec->name = NULL; free(rec->name_desc); rec->name_desc = NULL; free(rec->md5_hash); rec->md5_hash = NULL; free(rec->section); rec->section = NULL; } /* * init_secbuffs-- * Sets the value of buflen for all the sec_buff field of rec. And then * allocate memory to each sec_buff member of rec. */ static void init_secbuffs(mandb_rec *rec) { /* * Some sec_buff might need more memory, for example desc, * which stores the data of the DESCRIPTION section, * while some might need very small amount of memory. * Therefore explicitly setting the value of buflen field for * each sec_buff. */ rec->desc.buflen = 10 * BUFLEN; rec->desc.data = emalloc(rec->desc.buflen); rec->desc.offset = 0; rec->lib.buflen = BUFLEN / 2; rec->lib.data = emalloc(rec->lib.buflen); rec->lib.offset = 0; rec->return_vals.buflen = BUFLEN; rec->return_vals.data = emalloc(rec->return_vals.buflen); rec->return_vals.offset = 0; rec->exit_status.buflen = BUFLEN; rec->exit_status.data = emalloc(rec->exit_status.buflen); rec->exit_status.offset = 0; rec->env.buflen = BUFLEN; rec->env.data = emalloc(rec->env.buflen); rec->env.offset = 0; rec->files.buflen = BUFLEN; rec->files.data = emalloc(rec->files.buflen); rec->files.offset = 0; rec->diagnostics.buflen = BUFLEN; rec->diagnostics.data = emalloc(rec->diagnostics.buflen); rec->diagnostics.offset = 0; rec->errors.buflen = BUFLEN; rec->errors.data = emalloc(rec->errors.buflen); rec->errors.offset = 0; } /* * free_secbuffs-- * This function should be called at the end, when all the pages have been * parsed. * It frees the memory allocated to sec_buffs by init_secbuffs in the starting. */ static void free_secbuffs(mandb_rec *rec) { free(rec->desc.data); free(rec->lib.data); free(rec->return_vals.data); free(rec->exit_status.data); free(rec->env.data); free(rec->files.data); free(rec->diagnostics.data); free(rec->errors.data); } static void replace_hyph(char *str) { char *iter = str; while ((iter = strchr(iter, ASCII_HYPH)) != NULL) *iter = '-'; iter = str; while ((iter = strchr(iter, ASCII_NBRSP)) != NULL) *iter = '-'; } static char * parse_escape(const char *str) { const char *backslash, *last_backslash; char *result, *iter; size_t len; assert(str); last_backslash = str; backslash = strchr(str, '\\'); if (backslash == NULL) { result = estrdup(str); replace_hyph(result); return result; } result = emalloc(strlen(str) + 1); iter = result; do { len = backslash - last_backslash; memcpy(iter, last_backslash, len); iter += len; if (backslash[1] == '-' || backslash[1] == ' ') { *iter++ = backslash[1]; last_backslash = backslash + 2; backslash = strchr(last_backslash, '\\'); } else { ++backslash; mandoc_escape(&backslash, NULL, NULL); last_backslash = backslash; if (backslash == NULL) break; backslash = strchr(last_backslash, '\\'); } } while (backslash != NULL); if (last_backslash != NULL) strcpy(iter, last_backslash); replace_hyph(result); return result; } /* * append-- * Concatenates a space and src at the end of sbuff->data (much like concat in * apropos-utils.c). * Rather than reallocating space for writing data, it uses the value of the * offset field of sec_buff to write new data at the free space left in the * buffer. * In case the size of the data to be appended exceeds the number of bytes left * in the buffer, it reallocates buflen number of bytes and then continues. * Value of offset field should be adjusted as new data is written. * * NOTE: This function does not write the null byte at the end of the buffers, * write a null byte at the position pointed to by offset before inserting data * in the db. */ static void append(secbuff *sbuff, const char *src) { short flag = 0; size_t srclen, newlen; char *temp; assert(src != NULL); temp = parse_escape(src); srclen = strlen(temp); if (sbuff->data == NULL) { sbuff->data = emalloc(sbuff->buflen); sbuff->offset = 0; } newlen = sbuff->offset + srclen + 2; if (newlen >= sbuff->buflen) { while (sbuff->buflen < newlen) sbuff->buflen += sbuff->buflen; sbuff->data = erealloc(sbuff->data, sbuff->buflen); flag = 1; } /* Append a space at the end of the buffer. */ if (sbuff->offset || flag) sbuff->data[sbuff->offset++] = ' '; /* Now, copy src at the end of the buffer. */ memcpy(sbuff->data + sbuff->offset, temp, srclen); sbuff->offset += srclen; free(temp); } static void usage(void) { fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname()); exit(1); }