diff options
| author | cashmere <cashmere@autistici.org> | 2026-09-04 09:05:24 +0200 |
|---|---|---|
| committer | cashmere <cashmere@autistici.org> | 2026-09-04 09:05:24 +0200 |
| commit | 4f49b16079b27d7ce633619e4cddfdf2bffbc952 (patch) | |
| tree | c296c1e7f3c3e2f6859c6d3a1b1b208779b89679 /src/search.c | |
| download | relic-master.tar.gz | |
Diffstat (limited to 'src/search.c')
| -rw-r--r-- | src/search.c | 2905 |
1 files changed, 2905 insertions, 0 deletions
diff --git a/src/search.c b/src/search.c new file mode 100644 index 0000000..1015fd1 --- /dev/null +++ b/src/search.c @@ -0,0 +1,2905 @@ +/* +** Copyright (c) 2009 D. Richard Hipp +** +** This program is free software; you can redistribute it and/or +** modify it under the terms of the Simplified BSD License (also +** known as the "2-Clause License" or "FreeBSD License".) + +** This program is distributed in the hope that it will be useful, +** but without any warranty; without even the implied warranty of +** merchantability or fitness for a particular purpose. +** +** Author contact information: +** drh@hwaci.com +** http://www.hwaci.com/drh/ +** +******************************************************************************* +** +** This file contains code to implement a search functions +** against timeline comments, check-in content, wiki pages, tickets, +** and/or forum posts. +** +** The search can be either a per-query "grep"-like search that scans +** the entire corpus. Or it can use the FTS5 search engine of SQLite. +** The choice is an administrator configuration option. +** +** The first option is referred to as "full-scan search". The second +** option is called "indexed search". +** +** The code in this file is ordered approximately as follows: +** +** (1) The full-scan search engine +** (2) The indexed search engine +** (3) Higher level interfaces that use either (1) or (b2) according +** to the current search configuration settings +*/ +#include "config.h" +#include "search.h" +#include <assert.h> + +#if INTERFACE + +/* Maximum number of search terms for full-scan search */ +#define SEARCH_MAX_TERM 8 + +/* +** A compiled search pattern used for full-scan search. +*/ +struct Search { + int nTerm; /* Number of search terms */ + struct srchTerm { /* For each search term */ + char *z; /* Text */ + int n; /* length */ + } a[SEARCH_MAX_TERM]; + /* Snippet controls */ + char *zPattern; /* The search pattern */ + char *zMarkBegin; /* Start of a match */ + char *zMarkEnd; /* End of a match */ + char *zMarkGap; /* A gap between two matches */ + unsigned fSrchFlg; /* Flags */ + int iScore; /* Score of the last match attempt */ + Blob snip; /* Snippet for the most recent match */ +}; + +#define SRCHFLG_HTML 0x01 /* Escape snippet text for HTML */ +#define SRCHFLG_STATIC 0x04 /* The static gSearch object */ + +#endif + +/* +** There is a single global Search object: +*/ +static Search gSearch; + + +/* +** Theses characters constitute a word boundary +*/ +static const char isBoundary[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define ISALNUM(x) (!isBoundary[(x)&0xff]) + + +/* +** Destroy a full-scan search context. +*/ +void search_end(Search *p){ + if( p ){ + fossil_free(p->zPattern); + fossil_free(p->zMarkBegin); + fossil_free(p->zMarkEnd); + fossil_free(p->zMarkGap); + if( p->iScore ) blob_reset(&p->snip); + memset(p, 0, sizeof(*p)); + if( p!=&gSearch ) fossil_free(p); + } +} + +/* +** Compile a full-scan search pattern +*/ +static Search *search_init( + const char *zPattern, /* The search pattern */ + const char *zMarkBegin, /* Start of a match */ + const char *zMarkEnd, /* End of a match */ + const char *zMarkGap, /* A gap between two matches */ + unsigned fSrchFlg /* Flags */ +){ + Search *p; + char *z; + int i; + + if( fSrchFlg & SRCHFLG_STATIC ){ + p = &gSearch; + search_end(p); + }else{ + p = fossil_malloc(sizeof(*p)); + memset(p, 0, sizeof(*p)); + } + p->zPattern = z = mprintf("%s",zPattern); + p->zMarkBegin = mprintf("%s",zMarkBegin); + p->zMarkEnd = mprintf("%s",zMarkEnd); + p->zMarkGap = mprintf("%s",zMarkGap); + p->fSrchFlg = fSrchFlg; + blob_init(&p->snip, 0, 0); + while( *z && p->nTerm<SEARCH_MAX_TERM ){ + while( *z && !ISALNUM(*z) ){ z++; } + if( *z==0 ) break; + p->a[p->nTerm].z = z; + for(i=1; ISALNUM(z[i]); i++){} + p->a[p->nTerm].n = i; + z += i; + p->nTerm++; + } + return p; +} + + +/* +** Append n bytes of text to snippet zTxt. Encode the text appropriately. +*/ +static void snippet_text_append( + Search *p, /* The search context */ + Blob *pSnip, /* Append to this snippet */ + const char *zTxt, /* Text to append */ + int n /* How many bytes to append */ +){ + if( n>0 ){ + if( p->fSrchFlg & SRCHFLG_HTML ){ + blob_appendf(pSnip, "%#h", n, zTxt); + }else{ + blob_append(pSnip, zTxt, n); + } + } +} + +/* This the core search engine for full-scan search. +** +** Compare a search pattern against one or more input strings which +** collectively comprise a document. Return a match score. Any +** postive value means there was a match. Zero means that one or +** more terms are missing. +** +** The score and a snippet are record for future use. +** +** Scoring: +** * All terms must match at least once or the score is zero +** * One point for each matching term +** * Extra points if consecutive words of the pattern are consecutive +** in the document +*/ +static int search_match( + Search *p, /* Search pattern and flags */ + int nDoc, /* Number of strings in this document */ + const char **azDoc /* Text of each string */ +){ + int score; /* Final score */ + int i; /* Offset into current document */ + int ii; /* Loop counter */ + int j; /* Loop over search terms */ + int k; /* Loop over prior terms */ + int iWord = 0; /* Current word number */ + int iDoc; /* Current document number */ + int wantGap = 0; /* True if a zMarkGap is wanted */ + const char *zDoc; /* Current document text */ + const int CTX = 50; /* Amount of snippet context */ + int anMatch[SEARCH_MAX_TERM]; /* Number of terms in best match */ + int aiBestDoc[SEARCH_MAX_TERM]; /* Document containing best match */ + int aiBestOfst[SEARCH_MAX_TERM]; /* Byte offset to start of best match */ + int aiLastDoc[SEARCH_MAX_TERM]; /* Document containing most recent match */ + int aiLastOfst[SEARCH_MAX_TERM]; /* Byte offset to the most recent match */ + int aiWordIdx[SEARCH_MAX_TERM]; /* Word index of most recent match */ + + memset(anMatch, 0, sizeof(anMatch)); + memset(aiWordIdx, 0xff, sizeof(aiWordIdx)); + for(iDoc=0; iDoc<nDoc; iDoc++){ + zDoc = azDoc[iDoc]; + if( zDoc==0 ) continue; + iWord++; + for(i=0; zDoc[i]; i++){ + if( !ISALNUM(zDoc[i]) ) continue; + iWord++; + for(j=0; j<p->nTerm; j++){ + int n = p->a[j].n; + if( sqlite3_strnicmp(p->a[j].z, &zDoc[i], n)==0 + && (!ISALNUM(zDoc[i+n]) || p->a[j].z[n]=='*') + ){ + aiWordIdx[j] = iWord; + aiLastDoc[j] = iDoc; + aiLastOfst[j] = i; + for(k=1; j-k>=0 && anMatch[j-k] && aiWordIdx[j-k]==iWord-k; k++){} + for(ii=0; ii<k; ii++){ + if( anMatch[j-ii]<k ){ + anMatch[j-ii] = k*(nDoc-iDoc); + aiBestDoc[j-ii] = aiLastDoc[j-ii]; + aiBestOfst[j-ii] = aiLastOfst[j-ii]; + } + } + break; + } + } + while( ISALNUM(zDoc[i]) ){ i++; } + if( zDoc[i]==0 ) break; + } + } + + /* Finished search all documents. + ** Every term must be seen or else the score is zero + */ + score = 1; + for(j=0; j<p->nTerm; j++) score *= anMatch[j]; + blob_reset(&p->snip); + p->iScore = score; + if( score==0 ) return score; + + + /* Prepare a snippet that describes the matching text. + */ + while(1){ + int iOfst; + int iTail; + int iBest; + for(ii=0; ii<p->nTerm && anMatch[ii]==0; ii++){} + if( ii>=p->nTerm ) break; /* This is where the loop exits */ + iBest = ii; + iDoc = aiBestDoc[ii]; + iOfst = aiBestOfst[ii]; + for(; ii<p->nTerm; ii++){ + if( anMatch[ii]==0 ) continue; + if( aiBestDoc[ii]>iDoc ) continue; + if( aiBestOfst[ii]>iOfst ) continue; + iDoc = aiBestDoc[ii]; + iOfst = aiBestOfst[ii]; + iBest = ii; + } + iTail = iOfst + p->a[iBest].n; + anMatch[iBest] = 0; + for(ii=0; ii<p->nTerm; ii++){ + if( anMatch[ii]==0 ) continue; + if( aiBestDoc[ii]!=iDoc ) continue; + if( aiBestOfst[ii]<=iTail+CTX*2 ){ + if( iTail<aiBestOfst[ii]+p->a[ii].n ){ + iTail = aiBestOfst[ii]+p->a[ii].n; + } + anMatch[ii] = 0; + ii = -1; + continue; + } + } + zDoc = azDoc[iDoc]; + iOfst -= CTX; + if( iOfst<0 ) iOfst = 0; + while( iOfst>0 && ISALNUM(zDoc[iOfst-1]) ) iOfst--; + while( zDoc[iOfst] && !ISALNUM(zDoc[iOfst]) ) iOfst++; + for(ii=0; ii<CTX && zDoc[iTail]; ii++, iTail++){} + while( ISALNUM(zDoc[iTail]) ) iTail++; + if( iOfst>0 || wantGap ) blob_append(&p->snip, p->zMarkGap, -1); + wantGap = zDoc[iTail]!=0; + zDoc += iOfst; + iTail -= iOfst; + + /* Add a snippet segment using characters iOfst..iOfst+iTail from zDoc */ + for(i=0; i<iTail; i++){ + if( !ISALNUM(zDoc[i]) ) continue; + for(j=0; j<p->nTerm; j++){ + int n = p->a[j].n; + if( sqlite3_strnicmp(p->a[j].z, &zDoc[i], n)==0 + && (!ISALNUM(zDoc[i+n]) || p->a[j].z[n]=='*') + ){ + snippet_text_append(p, &p->snip, zDoc, i); + zDoc += i; + iTail -= i; + blob_append(&p->snip, p->zMarkBegin, -1); + if( p->a[j].z[n]=='*' ){ + while( ISALNUM(zDoc[n]) ) n++; + } + snippet_text_append(p, &p->snip, zDoc, n); + zDoc += n; + iTail -= n; + blob_append(&p->snip, p->zMarkEnd, -1); + i = -1; + break; + } /* end-if */ + } /* end for(j) */ + if( j<p->nTerm ){ + while( ISALNUM(zDoc[i]) && i<iTail ){ i++; } + } + } /* end for(i) */ + snippet_text_append(p, &p->snip, zDoc, iTail); + } + if( wantGap ) blob_append(&p->snip, p->zMarkGap, -1); + return score; +} + +/* +** COMMAND: test-match +** +** Usage: %fossil test-match SEARCHSTRING FILE1 FILE2 ... +** +** Run the full-scan search algorithm using SEARCHSTRING against +** the text of the files listed. Output matches and snippets. +** +** Options: +** --begin TEXT Text to insert before each match +** --end TEXT Text to insert after each match +** --gap TEXT Text to indicate elided content +** --html Input is HTML +** --static Use the static Search object +*/ +void test_match_cmd(void){ + Search *p; + int i; + Blob x; + int score; + char *zDoc; + int flg = 0; + char *zBegin = (char*)find_option("begin",0,1); + char *zEnd = (char*)find_option("end",0,1); + char *zGap = (char*)find_option("gap",0,1); + if( find_option("html",0,0)!=0 ) flg |= SRCHFLG_HTML; + if( find_option("static",0,0)!=0 ) flg |= SRCHFLG_STATIC; + verify_all_options(); + if( g.argc<4 ) usage("SEARCHSTRING FILE1..."); + if( zBegin==0 ) zBegin = "[["; + if( zEnd==0 ) zEnd = "]]"; + if( zGap==0 ) zGap = " ... "; + p = search_init(g.argv[2], zBegin, zEnd, zGap, flg); + for(i=3; i<g.argc; i++){ + blob_read_from_file(&x, g.argv[i], ExtFILE); + zDoc = blob_str(&x); + score = search_match(p, 1, (const char**)&zDoc); + fossil_print("%s: %d\n", g.argv[i], p->iScore); + blob_reset(&x); + if( score ){ + fossil_print("%.78c\n%s\n%.78c\n\n", '=', blob_str(&p->snip), '='); + } + } + search_end(p); +} + +/* +** An SQL function to initialize the full-scan search pattern: +** +** search_init(PATTERN,BEGIN,END,GAP,FLAGS) +** +** All arguments are optional. PATTERN is the search pattern. If it +** is omitted, then the global search pattern is reset. BEGIN and END +** and GAP are the strings used to construct snippets. FLAGS is an +** integer bit pattern containing the various SRCH_CKIN, SRCH_DOC, +** SRCH_TKT, SRCH_FORUM, or SRCH_ALL bits to determine what is to be +** searched. +*/ +static void search_init_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPattern = 0; + const char *zBegin = "<mark>"; + const char *zEnd = "</mark>"; + const char *zGap = " ... "; + unsigned int flg = SRCHFLG_HTML; + switch( argc ){ + default: + flg = (unsigned int)sqlite3_value_int(argv[4]); + case 4: + zGap = (const char*)sqlite3_value_text(argv[3]); + case 3: + zEnd = (const char*)sqlite3_value_text(argv[2]); + case 2: + zBegin = (const char*)sqlite3_value_text(argv[1]); + case 1: + zPattern = (const char*)sqlite3_value_text(argv[0]); + } + if( zPattern && zPattern[0] ){ + search_init(zPattern, zBegin, zEnd, zGap, flg | SRCHFLG_STATIC); + }else{ + search_end(&gSearch); + } +} + +/* search_match(TEXT, TEXT, ....) +** +** Using the full-scan search engine created by the most recent call +** to search_init(), match the input the TEXT arguments. +** Remember the results in the global full-scan search object. +** Return non-zero on a match and zero on a miss. +*/ +static void search_match_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *azDoc[5]; + int nDoc; + int rc; + for(nDoc=0; nDoc<count(azDoc) && nDoc<argc; nDoc++){ + azDoc[nDoc] = (const char*)sqlite3_value_text(argv[nDoc]); + if( azDoc[nDoc]==0 ) azDoc[nDoc] = ""; + } + rc = search_match(&gSearch, nDoc, azDoc); + sqlite3_result_int(context, rc); +} + + +/* search_score() +** +** Return the match score for the last successful search_match call. +*/ +static void search_score_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_result_int(context, gSearch.iScore); +} + +/* search_snippet() +** +** Return a snippet for the last successful search_match() call. +*/ +static void search_snippet_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( blob_size(&gSearch.snip)>0 ){ + sqlite3_result_text(context, blob_str(&gSearch.snip), -1, fossil_free); + blob_init(&gSearch.snip, 0, 0); + } +} + +/* stext(TYPE, RID, ARG) +** +** This is an SQLite function that computes the searchable text. +** It is a wrapper around the search_stext() routine. See the +** search_stext() routine for further detail. +*/ +static void search_stext_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zType = (const char*)sqlite3_value_text(argv[0]); + int rid = sqlite3_value_int(argv[1]); + const char *zName = (const char*)sqlite3_value_text(argv[2]); + sqlite3_result_text(context, search_stext_cached(zType[0],rid,zName,0), -1, + SQLITE_TRANSIENT); +} + +/* title(TYPE, RID, ARG) +** +** Return the title of the document to be search. +*/ +static void search_title_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zType = (const char*)sqlite3_value_text(argv[0]); + int rid = sqlite3_value_int(argv[1]); + const char *zName = (const char*)sqlite3_value_text(argv[2]); + int nHdr = 0; + char *z = search_stext_cached(zType[0], rid, zName, &nHdr); + if( nHdr || zType[0]!='d' ){ + sqlite3_result_text(context, z, nHdr, SQLITE_TRANSIENT); + }else{ + sqlite3_result_value(context, argv[2]); + } +} + +/* body(TYPE, RID, ARG) +** +** Return the body of the document to be search. +*/ +static void search_body_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zType = (const char*)sqlite3_value_text(argv[0]); + int rid = sqlite3_value_int(argv[1]); + const char *zName = (const char*)sqlite3_value_text(argv[2]); + int nHdr = 0; + char *z = search_stext_cached(zType[0], rid, zName, &nHdr); + sqlite3_result_text(context, z+nHdr+1, -1, SQLITE_TRANSIENT); +} + +/* urlencode(X) +** +** Encode a string for use as a query parameter in a URL. This is +** the equivalent of printf("%T",X). +*/ +static void search_urlencode_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + char *z = mprintf("%T",sqlite3_value_text(argv[0])); + sqlite3_result_text(context, z, -1, fossil_free); +} + +/* +** Register the various SQL functions (defined above) needed to implement +** full-scan search. +*/ +void search_sql_setup(sqlite3 *db){ + static int once = 0; + static const int enc = SQLITE_UTF8|SQLITE_INNOCUOUS; + if( once++ ) return; + sqlite3_create_function(db, "search_match", -1, enc, 0, + search_match_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_score", 0, enc, 0, + search_score_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_snippet", 0, enc, 0, + search_snippet_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_init", -1, enc, 0, + search_init_sqlfunc, 0, 0); + sqlite3_create_function(db, "stext", 3, enc, 0, + search_stext_sqlfunc, 0, 0); + sqlite3_create_function(db, "title", 3, enc, 0, + search_title_sqlfunc, 0, 0); + sqlite3_create_function(db, "body", 3, enc, 0, + search_body_sqlfunc, 0, 0); + sqlite3_create_function(db, "urlencode", 1, enc, 0, + search_urlencode_sqlfunc, 0, 0); +} + +/* +** Testing the search function. +** +** COMMAND: search* +** +** Usage: %fossil search [OPTIONS] PATTERN... +** +** Search the repository for PATTERN and show matches. Depending on +** options and how the administrator has search configured for the +** repository, the search can cover: +** +** * check-in comments (-c) +** * embedded documentation (--docs) +** * forum posts (--forum) +** * tickets (--tickets) +** * tech notes (--technotes) +** * wiki pages (--wiki) +** * built-in fossil help text (-h) +** * all of the above (-a) +** +** Use options below to select the scope of the search. The +** default is check-in comments only (-c). +** +** Output is colorized if writing to a TTY and if the NO_COLOR environment +** variable is not set. Use the "--highlight 0" option to disable colorization +** or use "--highlight 91" to force it on. Change the argument to --highlight +** to change the color. +** +** Options: +** -a|--all Search everything +** -c|--checkins Search check-in comments +** --docs Search embedded documentation +** --forum Search forum posts +** -h|--bi-help Search built-in help +** --highlight N Used VT100 color N for matching text. 0 means "off". +** -n|--limit N Limit output to N matches +** --technotes Search tech notes +** --tickets Search tickets +** -W|--width WIDTH Set display width to WIDTH columns, 0 for +** unlimited. Defaults to the terminal's width. +** --wiki Search wiki +*/ +void search_cmd(void){ + Blob pattern; + int i; + Blob sql = empty_blob; + Stmt q; + int iBest; + int srchFlags = 0; + int bFts = 1; /* Use FTS search by default now */ + char fAll = NULL != find_option("all", "a", 0); + const char *zLimit = find_option("limit","n",1); + const char *zScope = 0; + const char *zWidth = find_option("width","W",1); + int bDebug = find_option("debug",0,0)!=0; /* Undocumented */ + int nLimit = zLimit ? atoi(zLimit) : -1000; + int width; + int nTty = 0; /* VT100 highlight color for matching text */ + const char *zHighlight = 0; + int bFlags = 0; /* DB open flags */ + + nTty = terminal_is_vt100(); + + /* Undocumented option to change highlight color */ + zHighlight = find_option("highlight",0,1); + if( zHighlight ) nTty = atoi(zHighlight); + + /* Undocumented option (legacy) */ + zScope = find_option("scope",0,1); + + if( find_option("fts",0,0)!=0 ) bFts = 1; /* Undocumented legacy */ + if( find_option("legacy",0,0)!=0 ) bFts = 0; /* Undocumented */ + + if( zWidth ){ + width = atoi(zWidth); + if( (width!=0) && (width<=20) ){ + fossil_fatal("-W|--width value must be >20 or 0"); + } + }else{ + width = -1; + } + if( zScope ){ + for(i=0; zScope[i]; i++){ + switch( zScope[i] ){ + case 'a': srchFlags = SRCH_ALL; break; + case 'c': srchFlags |= SRCH_CKIN; break; + case 'd': srchFlags |= SRCH_DOC; break; + case 'e': srchFlags |= SRCH_TECHNOTE; break; + case 'f': srchFlags |= SRCH_FORUM; break; + case 'h': srchFlags |= SRCH_HELP; break; + case 't': srchFlags |= SRCH_TKT; break; + case 'w': srchFlags |= SRCH_WIKI; break; + } + } + bFts = 1; + } + if( find_option("all","a",0) ){ srchFlags |= SRCH_ALL; bFts = 1; } + if( find_option("bi-help","h",0) ){ srchFlags |= SRCH_HELP; bFts = 1; } + if( find_option("checkins","c",0) ){ srchFlags |= SRCH_CKIN; bFts = 1; } + if( find_option("docs",0,0) ){ srchFlags |= SRCH_DOC; bFts = 1; } + if( find_option("forum",0,0) ){ srchFlags |= SRCH_FORUM; bFts = 1; } + if( find_option("technotes",0,0) ){ srchFlags |= SRCH_TECHNOTE; bFts = 1; } + if( find_option("tickets",0,0) ){ srchFlags |= SRCH_TKT; bFts = 1; } + if( find_option("wiki",0,0) ){ srchFlags |= SRCH_WIKI; bFts = 1; } + + /* If no search objects are specified, default to "check-in comments" */ + if( srchFlags==0 ) srchFlags = SRCH_CKIN; + + if( srchFlags==SRCH_HELP ) bFlags = OPEN_OK_NOT_FOUND|OPEN_SUBSTITUTE; + db_find_and_open_repository(bFlags, 0); + verify_all_options(); + if( g.argc<3 ) return; + login_set_capabilities("s", 0); + if( search_restrict(srchFlags)==0 && (srchFlags & SRCH_HELP)==0 ){ + const char *zC1 = 0, *zPlural = "s"; + if( srchFlags & SRCH_TECHNOTE ){ zC1 = "technote"; } + if( srchFlags & SRCH_TKT ){ zC1 = "ticket"; } + if( srchFlags & SRCH_FORUM ){ zC1 = "forum"; zPlural = ""; } + if( srchFlags & SRCH_DOC ){ zC1 = "document"; } + if( srchFlags & SRCH_WIKI ){ zC1 = "wiki"; zPlural = ""; } + if( srchFlags & SRCH_CKIN ){ zC1 = "check-in"; } + fossil_print( + "Search of %s%s is disabled on this repository.\n" + "Enable using \"fossil fts-config enable %s\".\n", + zC1, zPlural, zC1 + ); + return; + } + + blob_init(&pattern, g.argv[2], -1); + for(i=3; i<g.argc; i++){ + blob_appendf(&pattern, " %s", g.argv[i]); + } + if( bFts ){ + /* Search using FTS */ + Blob com; + Blob snip; + const char *zPattern = blob_str(&pattern); + search_sql_setup(g.db); + add_content_sql_commands(g.db); + db_multi_exec( + "CREATE TEMP TABLE x(label,url,score,id,date,snip);" + ); + if( !search_index_exists() ){ + search_fullscan(zPattern, srchFlags); /* Full-scan search */ + }else{ + search_update_index(srchFlags); /* Update the index */ + search_indexed(zPattern, srchFlags); /* Indexed search */ + if( srchFlags & SRCH_HELP ){ + search_fullscan(zPattern, SRCH_HELP); + } + } + db_prepare(&q, "SELECT snip, label, score, id, date" + " FROM x" + " ORDER BY score DESC, date DESC;"); + blob_init(&com, 0, 0); + blob_init(&snip, 0, 0); + if( width<0 ) width = terminal_get_width(80); + while( db_step(&q)==SQLITE_ROW ){ + const char *zSnippet = db_column_text(&q, 0); + const char *zLabel = db_column_text(&q, 1); + const char *zDate = db_column_text(&q, 4); + const char *zScore = db_column_text(&q, 2); + const char *zId = db_column_text(&q, 3); + char *zOrig; + blob_appendf(&snip, "%s", zSnippet); + zOrig = blob_materialize(&snip); + blob_init(&snip, 0, 0); + html_to_plaintext(zOrig, &snip, (nTty?HTOT_VT100:0)|HTOT_FLOW|HTOT_TRIM); + fossil_free(zOrig); + blob_appendf(&com, "%s\n%s\n%s", zLabel, blob_str(&snip), zDate); + if( bDebug ){ + blob_appendf(&com," score: %s id: %s", zScore, zId); + } + comment_print(blob_str(&com), 0, 5, width, + COMMENT_PRINT_TRIM_CRLF | + COMMENT_PRINT_WORD_BREAK | + COMMENT_PRINT_TRIM_SPACE); + blob_reset(&com); + blob_reset(&snip); + if( nLimit>=1 ){ + nLimit--; + if( nLimit==0 ) break; + } + } + db_finalize(&q); + blob_reset(&pattern); + }else{ + /* Legacy timeline search (the default) */ + (void)search_init(blob_str(&pattern),"*","*","...",SRCHFLG_STATIC); + blob_reset(&pattern); + search_sql_setup(g.db); + + db_multi_exec( + "CREATE TEMP TABLE srch(rid,uuid,date,comment,x);" + "CREATE INDEX srch_idx1 ON srch(x);" + "INSERT INTO srch(rid,uuid,date,comment,x)" + " SELECT blob.rid, uuid, datetime(event.mtime,toLocal())," + " coalesce(ecomment,comment)," + " search_score()" + " FROM event, blob" + " WHERE blob.rid=event.objid" + " AND search_match(coalesce(ecomment,comment));" + ); + iBest = db_int(0, "SELECT max(x) FROM srch"); + blob_append(&sql, + "SELECT rid, uuid, date, comment, 0, 0 FROM srch " + "WHERE 1 ", -1); + if(!fAll){ + blob_append_sql(&sql,"AND x>%d ", iBest/3); + } + blob_append(&sql, "ORDER BY x DESC, date DESC ", -1); + db_prepare(&q, "%s", blob_sql_text(&sql)); + blob_reset(&sql); + print_timeline(&q, nLimit, width, 0, 0); + db_finalize(&q); + } +} + +#if INTERFACE +/* What to search for */ +#define SRCH_CKIN 0x0001 /* Search over check-in comments */ +#define SRCH_DOC 0x0002 /* Search over embedded documents */ +#define SRCH_TKT 0x0004 /* Search over tickets */ +#define SRCH_WIKI 0x0008 /* Search over wiki */ +#define SRCH_TECHNOTE 0x0010 /* Search over tech notes */ +#define SRCH_FORUM 0x0020 /* Search over forum messages */ +#define SRCH_HELP 0x0040 /* Search built-in help (full-scan only) */ +#define SRCH_ALL 0x007f /* Search over everything */ +#endif + +/* +** Remove bits from srchFlags which are disallowed by either the +** current server configuration or by user permissions. Return +** the revised search flags mask. +** +** If bFlex is true, that means allow through the SRCH_HELP option +** even if it is not explicitly enabled. +*/ +unsigned int search_restrict(unsigned int srchFlags){ + static unsigned int knownGood = 0; + static unsigned int knownBad = 0; + static const struct { unsigned m; const char *zKey; } aSetng[] = { + { SRCH_CKIN, "search-ci" }, + { SRCH_DOC, "search-doc" }, + { SRCH_TKT, "search-tkt" }, + { SRCH_WIKI, "search-wiki" }, + { SRCH_TECHNOTE, "search-technote" }, + { SRCH_FORUM, "search-forum" }, + { SRCH_HELP, "search-help" }, + }; + int i; + if( g.perm.Read==0 ) srchFlags &= ~(SRCH_CKIN|SRCH_DOC|SRCH_TECHNOTE); + if( g.perm.RdTkt==0 ) srchFlags &= ~(SRCH_TKT); + if( g.perm.RdWiki==0 ) srchFlags &= ~(SRCH_WIKI); + if( g.perm.RdForum==0) srchFlags &= ~(SRCH_FORUM); + for(i=0; i<count(aSetng); i++){ + unsigned int m = aSetng[i].m; + if( (srchFlags & m)==0 ) continue; + if( ((knownGood|knownBad) & m)!=0 ) continue; + if( db_get_boolean(aSetng[i].zKey,0) ){ + knownGood |= m; + }else{ + knownBad |= m; + } + } + return srchFlags & ~knownBad; +} + +/* +** When this routine is called, there already exists a table +** +** x(label,url,score,id,snip). +** +** label: The "name" of the document containing the match +** url: A URL for the document +** score: How well the document matched +** id: The document id. Format: xNNNNN, x: type, N: number +** snip: A snippet for the match +** +** And the srchFlags parameter has been validated. This routine +** fills the X table with search results using a full-scan search. +** +** The companion indexed search routine is search_indexed(). +*/ +LOCAL void search_fullscan( + const char *zPattern, /* The query pattern */ + unsigned int srchFlags /* What to search over */ +){ + search_init(zPattern, "<mark>", "</mark>", " ... ", + SRCHFLG_STATIC|SRCHFLG_HTML); + if( (srchFlags & SRCH_DOC)!=0 ){ + char *zDocGlob = db_get("doc-glob",""); + const char *zMainBranch = db_main_branch(); + char *zDocBr = db_get("doc-branch", zMainBranch); + if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){ + Glob * pGlob = glob_create(zDocBr) + /* We're misusing a Glob as a list of comma-/space-delimited + ** tokens. We're not actually doing glob matches here. */; + int i; + db_multi_exec( + "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;" + ); + for( i = 0; i < pGlob->nPattern; ++i ){ + const char * zBranch = pGlob->azPattern[i]; + db_multi_exec( + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Document: %%s',title('d',blob.rid,foci.filename))," + " printf('/doc/%T/%%s',foci.filename)," + " search_score()," + " 'd'||blob.rid," + " (SELECT datetime(event.mtime) FROM event" + " WHERE objid=symbolic_name_to_rid(%Q))," + " search_snippet()" + " FROM foci CROSS JOIN blob" + " WHERE checkinID=symbolic_name_to_rid(%Q)" + " AND blob.uuid=foci.uuid" + " AND search_match(title('d',blob.rid,foci.filename)," + " body('d',blob.rid,foci.filename))" + " AND %z", + zBranch, zBranch, zBranch, glob_expr("foci.filename", zDocGlob) + ); + } + glob_free(pGlob); + } + fossil_free(zDocGlob); + fossil_free(zDocBr); + } + if( (srchFlags & SRCH_WIKI)!=0 ){ + db_multi_exec( + "WITH wiki(name,rid,mtime) AS (" + " SELECT substr(tagname,6), tagxref.rid, max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'wiki-*'" + " AND tagxref.tagid=tag.tagid" + " GROUP BY 1" + ")" + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Wiki: %%s',name)," + " printf('/wiki?name=%%s',urlencode(name))," + " search_score()," + " 'w'||rid," + " datetime(mtime)," + " search_snippet()" + " FROM wiki" + " WHERE search_match(title('w',rid,name),body('w',rid,name));" + ); + } + if( (srchFlags & SRCH_CKIN)!=0 ){ + db_multi_exec( + "WITH ckin(uuid,rid,mtime) AS (" + " SELECT blob.uuid, event.objid, event.mtime" + " FROM event, blob" + " WHERE event.type='ci'" + " AND blob.rid=event.objid" + ")" + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime))," + " printf('/timeline?c=%%s',uuid)," + " search_score()," + " 'c'||rid," + " datetime(mtime)," + " search_snippet()" + " FROM ckin" + " WHERE search_match('',body('c',rid,NULL));" + ); + } + if( (srchFlags & SRCH_TKT)!=0 ){ + db_multi_exec( + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL)," + "datetime(tkt_mtime))," + " printf('/tktview/%%.20s',tkt_uuid)," + " search_score()," + " 't'||tkt_id," + " datetime(tkt_mtime)," + " search_snippet()" + " FROM ticket" + " WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));" + ); + } + if( (srchFlags & SRCH_TECHNOTE)!=0 ){ + db_multi_exec( + "WITH technote(uuid,rid,mtime) AS (" + " SELECT substr(tagname,7), tagxref.rid, max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'event-*'" + " AND tagxref.tagid=tag.tagid" + " GROUP BY 1" + ")" + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Tech Note: %%s',uuid)," + " printf('/technote/%%s',uuid)," + " search_score()," + " 'e'||rid," + " datetime(mtime)," + " search_snippet()" + " FROM technote" + " WHERE search_match('',body('e',rid,NULL));" + ); + } + if( (srchFlags & SRCH_FORUM)!=0 ){ + db_multi_exec( + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT 'Forum '||comment," + " '/forumpost/'||uuid," + " search_score()," + " 'f'||rid," + " datetime(event.mtime)," + " search_snippet()" + " FROM event JOIN blob on event.objid=blob.rid" + " WHERE search_match('',body('f',rid,NULL));" + ); + } + if( (srchFlags & SRCH_HELP)!=0 ){ + const char *zPrefix; + helptext_vtab_register(g.db); + if( srchFlags==SRCH_HELP ){ + zPrefix = "The"; + }else{ + zPrefix = "Built-in help for the"; + } + db_multi_exec( + "INSERT INTO x(label,url,score,id,snip)" + " SELECT format('%q \"%%s\" %%s',name,type)," + " '/help/'||name," + " search_score()," + " 'h'||rowid," + " search_snippet()" + " FROM helptext" + " WHERE search_match(format('the \"%%s\" %%s',name,type)," + " helptext.helptext);", + zPrefix + ); + } +} + +/* +** Number of significant bits in a u32 +*/ +static int nbits(u32 x){ + int n = 0; + while( x ){ n++; x >>= 1; } + return n; +} + +/* +** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')). +*/ +static void search_rank_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned *aVal = (unsigned int*)sqlite3_value_blob(argv[0]); + int nVal = sqlite3_value_bytes(argv[0])/4; + int nCol; /* Number of columns in the index */ + int nTerm; /* Number of search terms in the query */ + int i, j; /* Loop counter */ + double r = 0.0; /* Score */ + const unsigned *aX, *aS; + + if( nVal<2 ) return; + nTerm = aVal[0]; + nCol = aVal[1]; + if( nVal<2+3*nCol*nTerm+nCol ) return; + aS = aVal+2; + aX = aS+nCol; + for(j=0; j<nCol; j++){ + double x; + if( aS[j]>0 ){ + x = 0.0; + for(i=0; i<nTerm; i++){ + int hits_this_row; + int hits_all_rows; + int rows_with_hit; + double avg_hits_per_row; + + hits_this_row = aX[j + i*nCol*3]; + if( hits_this_row==0 )continue; + hits_all_rows = aX[j + i*nCol*3 + 1]; + rows_with_hit = aX[j + i*nCol*3 + 2]; + if( rows_with_hit==0 ) continue; + avg_hits_per_row = hits_all_rows/(double)rows_with_hit; + x += hits_this_row/(avg_hits_per_row*nbits(rows_with_hit)); + } + x *= (1<<((30*(aS[j]-1))/nTerm)); + }else{ + x = 0.0; + } + r = r*10.0 + x; + } +#define SEARCH_DEBUG_RANK 0 +#if SEARCH_DEBUG_RANK + { + Blob x; + blob_init(&x,0,0); + blob_appendf(&x,"%08x", (int)r); + for(i=0; i<nVal; i++){ + blob_appendf(&x," %d", aVal[i]); + } + blob_appendf(&x," r=%g", r); + sqlite3_result_text(context, blob_str(&x), -1, fossil_free); + } +#else + sqlite3_result_double(context, r); +#endif +} + +/* +** Expects a search pattern string. Makes a copy of the string, +** replaces all non-alphanum ASCII characters with a space, and +** lower-cases all upper-case ASCII characters. The intent is to avoid +** causing errors in FTS5 searches with inputs which contain AND, OR, +** and symbols like #. The caller is responsible for passing the +** result to fossil_free(). +*/ +char *search_simplify_pattern(const char * zPattern){ + char *zPat = mprintf("%s",zPattern); + int i; + for(i=0; zPat[i]; i++){ + if( (zPat[i]&0x80)==0 && !fossil_isalnum(zPat[i]) ) zPat[i] = ' '; + if( fossil_isupper(zPat[i]) ) zPat[i] = fossil_tolower(zPat[i]); + } + for(i--; i>=0 && zPat[i]==' '; i--){} + if( i<0 ){ + fossil_free(zPat); + zPat = mprintf("\"\""); + } + return zPat; +} + +/* +** When this routine is called, there already exists a table +** +** x(label,url,score,id,snip). +** +** label: The "name" of the document containing the match +** url: A URL for the document +** score: How well the document matched +** id: The document id. Format: xNNNNN, x: type, N: number +** snip: A snippet for the match +** +** And the srchFlags parameter has been validated. This routine +** fills the X table with search results using FTS indexed search. +** +** The companion full-scan search routine is search_fullscan(). +*/ +LOCAL void search_indexed( + const char *zPattern, /* The query pattern */ + unsigned int srchFlags /* What to search over */ +){ + Blob sql; + char *zPat; + static const char *zSnippetCall; + if( srchFlags==0 ) return; + sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + search_rank_sqlfunc, 0, 0); + zPat = search_simplify_pattern(zPattern); + blob_init(&sql, 0, 0); + if( search_index_type(0)==4 ){ + /* If this repo is still using the legacy FTS5 search index, then + ** the snippet() function is slightly different */ + zSnippetCall = "snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"; + }else{ + /* This is the common case - Using newer FTS5 search index */ + zSnippetCall = "snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)"; + } + blob_appendf(&sql, + "INSERT INTO x(label,url,score,id,date,snip) " + " SELECT ftsdocs.label," + " ftsdocs.url," + " rank(matchinfo(ftsidx,'pcsx'))," + " ftsdocs.type || ftsdocs.rid," + " datetime(ftsdocs.mtime)," + " %s" + " FROM ftsidx CROSS JOIN ftsdocs" + " WHERE ftsidx MATCH %Q" + " AND ftsdocs.rowid=ftsidx.rowid", + zSnippetCall /*safe-for-%s*/, zPat + ); + fossil_free(zPat); + if( srchFlags!=SRCH_ALL ){ + const char *zSep = " AND ("; + static const struct { unsigned m; char c; } aMask[] = { + { SRCH_CKIN, 'c' }, + { SRCH_DOC, 'd' }, + { SRCH_TKT, 't' }, + { SRCH_WIKI, 'w' }, + { SRCH_TECHNOTE, 'e' }, + { SRCH_FORUM, 'f' }, + { SRCH_HELP, 'h' }, + }; + int i; + for(i=0; i<count(aMask); i++){ + if( srchFlags & aMask[i].m ){ + blob_appendf(&sql, "%sftsdocs.type='%c'", zSep, aMask[i].c); + zSep = " OR "; + } + } + blob_append(&sql,")",1); + } + db_multi_exec("%s",blob_str(&sql)/*safe-for-%s*/); +#if SEARCH_DEBUG_RANK + db_multi_exec("UPDATE x SET label=printf('%%s (score=%%s)',label,score)"); +#endif +} + +/* +** If z[] is of the form "<mark>TEXT</mark>" where TEXT contains +** no white-space or punctuation, then return the length of the mark. +*/ +static int isSnippetMark(const char *z){ + int n; + if( strncmp(z,"<mark>",6)!=0 ) return 0; + n = 6; + while( fossil_isalnum(z[n]) ) n++; + if( strncmp(&z[n],"</mark>",7)!=0 ) return 0; + return n+7; +} + +/* +** Return a copy of zSnip (in memory obtained from fossil_malloc()) that +** has all "<" characters, other than those on <mark> and </mark>, +** converted into "<". This is similar to htmlize() except that +** <mark> and </mark> are preserved. +*/ +static char *cleanSnippet(const char *zSnip){ + int i; + int n = 0; + char *z; + if( zSnip==0 ) zSnip = ""; + for(i=0; zSnip[i]; i++) if( zSnip[i]=='<' ) n++; + z = fossil_malloc( i+n*4+1 ); + i = 0; + while( zSnip[0] ){ + if( zSnip[0]=='<' ){ + n = isSnippetMark(zSnip); + if( n ){ + memcpy(&z[i], zSnip, n); + zSnip += n; + i += n; + continue; + }else{ + memcpy(&z[i], "<", 4); + i += 4; + zSnip++; + } + }else{ + z[i++] = zSnip[0]; + zSnip++; + } + } + z[i] = 0; + return z; +} + + +/* +** This routine generates web-page output for a search operation. +** Other web-pages can invoke this routine to add search results +** in the middle of the page. +** +** This routine works for both full-scan and indexed search. The +** appropriate low-level search routine is called according to the +** current configuration. +** +** Return the number of rows. +*/ +int search_run_and_output( + const char *zPattern, /* The query pattern */ + unsigned int srchFlags, /* What to search over */ + int fDebug /* Extra debugging output */ +){ + Stmt q; + int nRow = 0; + int nLimit = db_get_int("search-limit", 100); + + if( P("searchlimit")!=0 ){ + nLimit = atoi(P("searchlimit")); + } + srchFlags = search_restrict(srchFlags) | (srchFlags & SRCH_HELP); + if( srchFlags==0 ) return 0; + search_sql_setup(g.db); + add_content_sql_commands(g.db); + db_multi_exec( + "CREATE TEMP TABLE x(label,url,score,id,date,snip);" + ); + if( !search_index_exists() ){ + search_fullscan(zPattern, srchFlags); /* Full-scan search */ + }else{ + search_update_index(srchFlags); /* Update the index, if necessary */ + search_indexed(zPattern, srchFlags); /* Indexed search */ + if( srchFlags & SRCH_HELP ){ + search_fullscan(zPattern, SRCH_HELP); + } + } + db_prepare(&q, "SELECT url, snip, label, score, id, substr(date,1,10)" + " FROM x" + " ORDER BY score DESC, date DESC;"); + while( db_step(&q)==SQLITE_ROW ){ + const char *zUrl = db_column_text(&q, 0); + const char *zSnippet = db_column_text(&q, 1); + const char *zLabel = db_column_text(&q, 2); + const char *zDate = db_column_text(&q, 5); + if( nRow==0 ){ + @ <ol> + } + nRow++; + @ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a> + if( fDebug ){ + @ (%e(db_column_double(&q,3)), %s(db_column_text(&q,4)) + } + @ <br><span class='snippet'>%z(cleanSnippet(zSnippet)) \ + if( zLabel && zDate && zDate[0] && strstr(zLabel,zDate)==0 ){ + @ <small>(%h(zDate))</small> + } + @ </span></li> + if( nLimit && nRow>=nLimit ) break; + } + db_finalize(&q); + if( nRow ){ + @ </ol> + } + return nRow; +} + +/* +** Generate some HTML for doing search. At a minimum include the +** Search-Text entry form. If the "s" query parameter is present, also +** show search results. +** +** The srchFlags parameter restricts the set of documents to be searched. +** srchFlags should normally be either a single search category or all +** categories. Any srchFlags with two or more bits set +** is treated like SRCH_ALL for display purposes. +** +** This routine automatically restricts srchFlag according to user +** permissions and the server configuration. The entry box is shown +** disabled if srchFlags is 0 after these restrictions are applied. +** +** The mFlags value controls options: +** +** 0x01 If the y= query parameter is present, use it as an addition +** restriction what to search. +** +** 0x02 Show nothing if search is disabled. +** +** Return true if there are search results. +*/ +int search_screen(unsigned srchAllowed, int mFlags){ + const char *zType = 0; + const char *zClass = 0; + const char *zDisable1; + const char *zDisable2; + const char *zPattern; + int fDebug = PB("debug"); + int haveResult = 0; + int srchThisTime; + const char *zY = PD("y","all"); + if( zY[0]=='h' && zY[1]==0 ){ + srchAllowed = search_restrict(srchAllowed) | (srchAllowed & SRCH_HELP); + }else{ + srchAllowed = search_restrict(srchAllowed); + } + switch( srchAllowed ){ + case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break; + case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break; + case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break; + case SRCH_WIKI: zType = " Wiki"; zClass = "Wiki"; break; + case SRCH_TECHNOTE: zType = " Tech Notes"; zClass = "Note"; break; + case SRCH_FORUM: zType = " Forum"; zClass = "Frm"; break; + case SRCH_HELP: zType = " Help"; zClass = "Hlp"; break; + } + if( srchAllowed==0 ){ + if( mFlags & 0x02 ) return 0; + zDisable1 = " disabled"; + zDisable2 = " disabled"; + zPattern = ""; + }else{ + zDisable1 = ""; /* Was: " autofocus" */ + zDisable2 = ""; + zPattern = PD("s",""); + } + @ <form method='GET' action='%R/%T(g.zPath)'> + if( zClass ){ + @ <div class='searchForm searchForm%s(zClass)'> + }else{ + @ <div class='searchForm'> + } + @ <input type="text" name="s" size="40" value="%h(zPattern)"%s(zDisable1)> + srchThisTime = srchAllowed; + if( (mFlags & 0x01)!=0 && (srchAllowed & (srchAllowed-1))!=0 ){ + static const struct { + const char *z; + const char *zNm; + unsigned m; + } aY[] = { + { "all", "All", SRCH_ALL }, + { "c", "Check-ins", SRCH_CKIN }, + { "d", "Docs", SRCH_DOC }, + { "t", "Tickets", SRCH_TKT }, + { "w", "Wiki", SRCH_WIKI }, + { "e", "Tech Notes", SRCH_TECHNOTE }, + { "f", "Forum", SRCH_FORUM }, + { "h", "Help", SRCH_HELP }, + }; + int i; + @ <select size='1' name='y'> + for(i=0; i<count(aY); i++){ + if( (aY[i].m & srchAllowed)==0 ) continue; + if( aY[i].m==SRCH_HELP && fossil_strcmp(zY,"h")!=0 + && search_restrict(SRCH_HELP)==0 ) continue; + cgi_printf("<option value='%s'", aY[i].z); + if( fossil_strcmp(zY,aY[i].z)==0 ){ + srchThisTime &= aY[i].m; + cgi_printf(" selected"); + } + cgi_printf(">%s</option>\n", aY[i].zNm); + } + @ </select> + } + if( fDebug ){ + @ <input type="hidden" name="debug" value="1"> + } + @ <input type="submit" value="Search%s(zType)"%s(zDisable2)> + if( srchAllowed==0 && srchThisTime==0 ){ + @ <p class="generalError">Search is disabled</p> + } + @ </div></form> + while( fossil_isspace(zPattern[0]) ) zPattern++; + if( zPattern[0] ){ + if( zClass ){ + @ <div class='searchResult searchResult%s(zClass)'> + }else{ + @ <div class='searchResult'> + } + if( search_run_and_output(zPattern, srchThisTime, fDebug)==0 ){ + @ <p class='searchEmpty'>No matches for: <span>%h(zPattern)</span></p> + } + @ </div> + haveResult = 1; + } + return haveResult; +} + +/* +** WEBPAGE: search +** +** Search for check-in comments, documents, tickets, or wiki that +** match a user-supplied pattern. +** +** s=PATTERN Specify the full-text pattern to search for +** y=TYPE What to search. +** c -> check-ins, +** d -> documentation, +** t -> tickets, +** w -> wiki, +** e -> tech notes, +** f -> forum, +** h -> built-in help, +** all -> everything. +*/ +void search_page(void){ + const int isSearch = P("s")!=0; + login_check_credentials(); + style_header("Search%s", isSearch ? " Results" : ""); + cgi_check_for_malice(); + search_screen(SRCH_ALL, 1); + style_finish_page(); +} + + +/* +** This is a helper function for search_stext(). Writing into pOut +** the search text obtained from pIn according to zMimetype. +** +** If a title is not specified in zTitle (e.g. for wiki pages that do not +** include the title in the body), it is determined from the page content. +** +** The title of the document is the first line of text. All subsequent +** lines are the body. If the document has no title, the first line +** is blank. +*/ +static void get_stext_by_mimetype( + Blob *pIn, + const char *zMimetype, + const char *zTitle, + Blob *pOut +){ + Blob html, title; + Blob *pHtml = &html; + blob_init(&html, 0, 0); + if( zTitle==0 ){ + blob_init(&title, 0, 0); + }else{ + blob_init(&title, zTitle, -1); + } + if( zMimetype==0 ) zMimetype = "text/plain"; + if( fossil_strcmp(zMimetype,"text/x-fossil-wiki")==0 ){ + if( blob_size(&title) ){ + wiki_convert(pIn, &html, 0); + }else{ + Blob tail; + blob_init(&tail, 0, 0); + if( wiki_find_title(pIn, &title, &tail) ){ + blob_appendf(pOut, "%s\n", blob_str(&title)); + wiki_convert(&tail, &html, 0); + blob_reset(&tail); + }else{ + blob_append(pOut, "\n", 1); + wiki_convert(pIn, &html, 0); + } + } + html_to_plaintext(blob_str(&html), pOut, 0); + }else if( fossil_strcmp(zMimetype,"text/x-markdown")==0 ){ + markdown_to_html(pIn, blob_size(&title) ? NULL : &title, &html); + }else if( fossil_strcmp(zMimetype,"text/x-org")==0 ){ + if( blob_size(&title) ){ + org_to_html_wiki(pIn, &html); + }else{ + org_to_html(pIn, &title, &html); + } + }else if( fossil_strcmp(zMimetype,"text/html")==0 ){ + if( blob_size(&title)==0 ) doc_is_embedded_html(pIn, &title); + pHtml = pIn; + } + blob_appendf(pOut, "%s\n", blob_str(&title)); + if( blob_size(pHtml) ){ + html_to_plaintext(blob_str(pHtml), pOut, 0); + }else{ + blob_append(pOut, blob_buffer(pIn), blob_size(pIn)); + } + blob_reset(&html); + blob_reset(&title); +} + +/* +** Query pQuery is pointing at a single row of output. Append a text +** representation of every text-compatible column to pAccum. +*/ +static void append_all_ticket_fields(Blob *pAccum, Stmt *pQuery, int iTitle){ + int n = db_column_count(pQuery); + int i; + const char *zMime = 0; + if( iTitle>=0 && iTitle<n ){ + if( db_column_type(pQuery,iTitle)==SQLITE_TEXT ){ + blob_append(pAccum, db_column_text(pQuery,iTitle), -1); + } + blob_append(pAccum, "\n", 1); + } + for(i=0; i<n; i++){ + const char *zColName = db_column_name(pQuery,i); + int eType = db_column_type(pQuery,i); + if( i==iTitle ) continue; + if( fossil_strnicmp(zColName,"tkt_",4)==0 ) continue; + if( fossil_strnicmp(zColName,"private_",8)==0 ) continue; + if( eType==SQLITE_BLOB || eType==SQLITE_NULL ) continue; + if( fossil_stricmp(zColName,"mimetype")==0 ){ + zMime = db_column_text(pQuery,i); + if( fossil_strcmp(zMime,"text/plain")==0 ) zMime = 0; + }else if( zMime==0 || eType!=SQLITE_TEXT ){ + blob_appendf(pAccum, "%s: %s |\n", zColName, db_column_text(pQuery,i)); + }else{ + Blob txt; + blob_init(&txt, db_column_text(pQuery,i), -1); + blob_appendf(pAccum, "%s: ", zColName); + get_stext_by_mimetype(&txt, zMime, NULL, pAccum); + blob_append(pAccum, " |", 2); + blob_reset(&txt); + } + } +} + + +/* +** Return "search text" - a reduced version of a document appropriate for +** full text search and/or for constructing a search result snippet. +** +** cType: d Embedded documentation +** w Wiki page +** c Check-in comment +** t Ticket text +** e Tech note +** f Forum +** +** rid The RID of an artifact that defines the object +** being searched. +** +** zName Name of the object being searched. This is used +** only to help figure out the mimetype (text/plain, +** test/html, test/x-fossil-wiki, or text/x-markdown) +** so that the code can know how to simplify the text. +*/ +void search_stext( + char cType, /* Type of document */ + int rid, /* BLOB.RID or TAG.TAGID value for document */ + const char *zName, /* Auxiliary information */ + Blob *pOut /* OUT: Initialize to the search text */ +){ + blob_init(pOut, 0, 0); + switch( cType ){ + case 'd': { /* Documents */ + Blob doc; + content_get(rid, &doc); + blob_to_utf8_no_bom(&doc, 0); + get_stext_by_mimetype(&doc, mimetype_from_name(zName), NULL, pOut); + blob_reset(&doc); + break; + } + case 'f': /* Forum messages */ + case 'e': /* Tech Notes */ + case 'w': { /* Wiki */ + Manifest *pWiki = manifest_get(rid, + cType == 'e' ? CFTYPE_EVENT : + cType == 'f' ? CFTYPE_FORUM : CFTYPE_WIKI, 0); + Blob wiki; + if( pWiki==0 ) break; + if( cType=='f' ){ + blob_init(&wiki, 0, 0); + if( pWiki->zThreadTitle ){ + blob_appendf(&wiki, "<h1>%h</h1>\n", pWiki->zThreadTitle); + } + blob_appendf(&wiki, "From %s:\n\n%s", pWiki->zUser, pWiki->zWiki); + }else{ + blob_init(&wiki, pWiki->zWiki, -1); + } + get_stext_by_mimetype(&wiki, wiki_filter_mimetypes(pWiki->zMimetype), + cType=='w' ? pWiki->zWikiTitle : NULL, pOut); + blob_reset(&wiki); + manifest_destroy(pWiki); + break; + } + case 'c': { /* Check-in Comments */ + static Stmt q; + static int isPlainText = -1; + db_static_prepare(&q, + "SELECT coalesce(ecomment,comment)" + " ||' (user: '||coalesce(euser,user,'?')" + " ||', tags: '||" + " (SELECT group_concat(substr(tag.tagname,5),',')" + " FROM tag, tagxref" + " WHERE tagname GLOB 'sym-*' AND tag.tagid=tagxref.tagid" + " AND tagxref.rid=event.objid AND tagxref.tagtype>0)" + " ||')'" + " FROM event WHERE objid=:x AND type='ci'"); + if( isPlainText<0 ){ + isPlainText = db_get_boolean("timeline-plaintext",0); + } + db_bind_int(&q, ":x", rid); + if( db_step(&q)==SQLITE_ROW ){ + blob_append(pOut, "\n", 1); + if( isPlainText ){ + db_column_blob(&q, 0, pOut); + }else{ + Blob x; + blob_init(&x,0,0); + db_column_blob(&q, 0, &x); + get_stext_by_mimetype(&x, "text/x-fossil-wiki", NULL, pOut); + blob_reset(&x); + } + } + db_reset(&q); + break; + } + case 't': { /* Tickets */ + static Stmt q1; + static int iTitle = -1; + db_static_prepare(&q1, "SELECT * FROM ticket WHERE tkt_id=:rid"); + db_bind_int(&q1, ":rid", rid); + if( db_step(&q1)==SQLITE_ROW ){ + if( iTitle<0 ){ + int n = db_column_count(&q1); + for(iTitle=0; iTitle<n; iTitle++){ + if( fossil_stricmp(db_column_name(&q1,iTitle),"title")==0 ) break; + } + } + append_all_ticket_fields(pOut, &q1, iTitle); + } + db_reset(&q1); + if( db_table_exists("repository","ticketchng") ){ + static Stmt q2; + db_static_prepare(&q2, "SELECT * FROM ticketchng WHERE tkt_id=:rid" + " ORDER BY tkt_mtime"); + db_bind_int(&q2, ":rid", rid); + while( db_step(&q2)==SQLITE_ROW ){ + append_all_ticket_fields(pOut, &q2, -1); + } + db_reset(&q2); + } + break; + } + } +} + +/* +** This routine is a wrapper around search_stext(). +** +** This routine looks up the search text, stores it in an internal +** buffer, and returns a pointer to the text. Subsequent requests +** for the same document return the same pointer. The returned pointer +** is valid until the next invocation of this routine. Call this routine +** with an eType of 0 to clear the cache. +*/ +char *search_stext_cached( + char cType, /* Type of document */ + int rid, /* BLOB.RID or TAG.TAGID value for document */ + const char *zName, /* Auxiliary information, for mimetype */ + int *pnTitle /* OUT: length of title in bytes excluding \n */ +){ + static struct { + Blob stext; /* Cached search text */ + char cType; /* The type */ + int rid; /* The RID */ + int nTitle; /* Number of bytes in title */ + } cache; + int i; + char *z; + if( cType!=cache.cType || rid!=cache.rid ){ + if( cache.rid>0 ){ + blob_reset(&cache.stext); + }else{ + blob_init(&cache.stext,0,0); + } + cache.cType = cType; + cache.rid = rid; + if( cType==0 ) return 0; + search_stext(cType, rid, zName, &cache.stext); + z = blob_str(&cache.stext); + for(i=0; z[i] && z[i]!='\n'; i++){} + cache.nTitle = i; + } + if( pnTitle ) *pnTitle = cache.nTitle; + return blob_str(&cache.stext); +} + +/* +** COMMAND: test-search-stext +** +** Usage: fossil test-search-stext TYPE RID NAME +** +** Compute the search text for document TYPE-RID whose name is NAME. +** The TYPE is one of "c", "d", "t", "w", or "e". The RID is the document +** ID. The NAME is used to figure out a mimetype to use for formatting +** the raw document text. +*/ +void test_search_stext(void){ + Blob out; + db_find_and_open_repository(0,0); + if( g.argc!=5 ) usage("TYPE RID NAME"); + search_stext(g.argv[2][0], atoi(g.argv[3]), g.argv[4], &out); + fossil_print("%s\n",blob_str(&out)); + blob_reset(&out); +} + +/* +** COMMAND: test-convert-stext +** +** Usage: fossil test-convert-stext FILE MIMETYPE +** +** Read the content of FILE and convert it to stext according to MIMETYPE. +** Send the result to standard output. +*/ +void test_convert_stext(void){ + Blob in, out; + db_find_and_open_repository(0,0); + if( g.argc!=4 ) usage("FILENAME MIMETYPE"); + blob_read_from_file(&in, g.argv[2], ExtFILE); + blob_init(&out, 0, 0); + get_stext_by_mimetype(&in, g.argv[3], NULL, &out); + fossil_print("%s\n",blob_str(&out)); + blob_reset(&in); + blob_reset(&out); +} + +/* +** The schema for the full-text index. The %s part must be an empty +** string or a comma followed by additional flags for the FTS virtual +** table. +*/ +static const char zFtsSchema[] = +@ -- One entry for each possible search result +@ CREATE TABLE IF NOT EXISTS repository.ftsdocs( +@ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid +@ type CHAR(1), -- Type of document +@ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document +@ name TEXT, -- Additional document description +@ idxed BOOLEAN, -- True if currently in the index +@ label TEXT, -- Label to print on search results +@ url TEXT, -- URL to access this document +@ mtime DATE, -- Date when document created +@ bx TEXT, -- Temporary "body" content cache +@ UNIQUE(type,rid) +@ ); +@ CREATE INDEX repository.ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0; +@ CREATE INDEX repository.ftsdocName ON ftsdocs(name) WHERE type='w'; +@ CREATE VIEW IF NOT EXISTS repository.ftscontent AS +@ SELECT rowid, type, rid, name, idxed, label, url, mtime, +@ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body' +@ FROM ftsdocs; +@ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx +@ USING fts5(content="ftscontent", title, body%s); +; +static const char zFtsDrop[] = +@ DROP TABLE IF EXISTS repository.ftsidx; +@ DROP VIEW IF EXISTS repository.ftscontent; +@ DROP TABLE IF EXISTS repository.ftsdocs; +@ DROP TABLE IF EXISTS repository.chatfts1; +; + +#if INTERFACE +/* +** Values for the search-tokenizer config option. +*/ +#define FTS5TOK_NONE 0 /* disabled */ +#define FTS5TOK_PORTER 1 /* porter stemmer */ +#define FTS5TOK_UNICODE61 2 /* unicode61 tokenizer */ +#define FTS5TOK_TRIGRAM 3 /* trigram tokenizer */ +#endif + +/* +** Cached FTS5TOK_xyz value for search_tokenizer_type() and +** friends. +*/ +static int iFtsTokenizer = -1; + +/* +** Returns one of the FTS5TOK_xyz values, depending on the value of +** the search-tokenizer config entry, defaulting to FTS5TOK_NONE. The +** result of the first call is cached for subsequent calls unless +** bRecheck is true. +*/ +int search_tokenizer_type(int bRecheck){ + char *z; + if( iFtsTokenizer>=0 && bRecheck==0 ){ + return iFtsTokenizer; + } + z = db_get("search-tokenizer",0); + if( 0==z ){ + iFtsTokenizer = FTS5TOK_NONE; + }else if(0==fossil_strcmp(z,"porter")){ + iFtsTokenizer = FTS5TOK_PORTER; + }else if(0==fossil_strcmp(z,"unicode61")){ + iFtsTokenizer = FTS5TOK_UNICODE61; + }else if(0==fossil_strcmp(z,"trigram")){ + iFtsTokenizer = FTS5TOK_TRIGRAM; + }else{ + iFtsTokenizer = is_truth(z) ? FTS5TOK_PORTER : FTS5TOK_NONE; + } + fossil_free(z); + return iFtsTokenizer; +} + +/* +** Returns a string in the form ",tokenize=X", where X is the string +** counterpart of the given FTS5TOK_xyz value. Returns "" if tokType +** does not correspond to a known FTS5 tokenizer. +*/ +const char * search_tokenize_arg_for_type(int tokType){ + switch( tokType ){ + case FTS5TOK_PORTER: return ",tokenize=porter"; + case FTS5TOK_UNICODE61: return ",tokenize=unicode61"; + case FTS5TOK_TRIGRAM: return ",tokenize=trigram"; + case FTS5TOK_NONE: + default: return ""; + } +} + +/* +** Returns a string value suitable for use as the search-tokenizer +** setting's value, depending on the value of z. If z is 0 then the +** current search-tokenizer value is used as the basis for formulating +** the result (which may differ from the current value but will have +** the same meaning). Any unknown/unsupported value is interpreted as +** "off". +*/ +const char *search_tokenizer_for_string(const char *z){ + char * zTmp = 0; + const char *zRc = 0; + + if( 0==z ){ + z = zTmp = db_get("search-tokenizer",0); + } + if( 0==z ){ + zRc = "off"; + }else if( 0==fossil_strcmp(z,"porter") ){ + zRc = "porter"; + }else if( 0==fossil_strcmp(z,"unicode61") ){ + zRc = "unicode61"; + }else if( 0==fossil_strcmp(z,"trigram") ){ + zRc = "trigram"; + }else{ + zRc = is_truth(z) ? "porter" : "off"; + } + fossil_free(zTmp); + return zRc; +} + +/* +** Sets the search-tokenizer config setting to the value of +** search_tokenizer_for_string(zName). +*/ +void search_set_tokenizer(const char *zName){ + db_set("search-tokenizer", search_tokenizer_for_string( zName ), 0); + iFtsTokenizer = -1; +} + +/* +** Create or drop the tables associated with a full-text index. +*/ +static int searchIdxExists = -1; +void search_create_index(void){ + const char *zExtra = + search_tokenize_arg_for_type(search_tokenizer_type(0)); + assert( zExtra ); + search_sql_setup(g.db); + db_multi_exec(zFtsSchema/*works-like:"%s"*/, zExtra/*safe-for-%s*/); + searchIdxExists = 1; +} +void search_drop_index(void){ + db_multi_exec(zFtsDrop/*works-like:""*/); + searchIdxExists = 0; +} + +/* +** Return true if the full-text search index exists. See also the +** search_index_type() function. +*/ +int search_index_exists(void){ + if( searchIdxExists<0 ){ + searchIdxExists = db_table_exists("repository","ftsdocs"); + } + return searchIdxExists; +} + +/* +** Determine which full-text search index is currently being used to +** add searching. Return values: +** +** 0 No search index is available +** 4 FTS3/4 +** 5 FTS5 +** +** Results are cached. Make the argument 1 to reset the cache. See +** also the search_index_exists() routine. +*/ +int search_index_type(int bReset){ + static int idxType = -1; + if( idxType<0 || bReset ){ + idxType = db_int(0, + "SELECT CASE WHEN sql GLOB '*fts4*' THEN 4 ELSE 5 END" + " FROM repository.sqlite_schema WHERE name='ftsidx'" + ); + } + return idxType; +} + +/* +** Fill the FTSDOCS table with unindexed entries for everything +** in the repository. This uses INSERT OR IGNORE so entries already +** in FTSDOCS are unchanged. +*/ +void search_fill_index(void){ + if( !search_index_exists() ) return; + search_sql_setup(g.db); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,idxed)" + " SELECT 'c', objid, 0 FROM event WHERE type='ci';" + ); + db_multi_exec( + "WITH latest_wiki(rid,name,mtime) AS (" + " SELECT tagxref.rid, substr(tag.tagname,6), max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'wiki-*'" + " AND tagxref.tagid=tag.tagid" + " AND tagxref.value>0" + " GROUP BY 2" + ") INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed)" + " SELECT 'w', rid, name, 0 FROM latest_wiki;" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,idxed)" + " SELECT 't', tkt_id, 0 FROM ticket;" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed)" + " SELECT type, objid, comment, 0 FROM event WHERE type IN ('e','f');" + ); +} + +/* +** The document described by cType,rid,zName is about to be added or +** updated. If the document has already been indexed, then unindex it +** now while we still have access to the old content. Add the document +** to the queue of documents that need to be indexed or reindexed. +*/ +void search_doc_touch(char cType, int rid, const char *zName){ + if( search_index_exists() && !content_is_private(rid) ){ + char zType[2]; + zType[0] = cType; + zType[1] = 0; + search_sql_setup(g.db); + db_multi_exec( + "DELETE FROM ftsidx WHERE rowid IN" + " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)", + zType, rid + ); + db_multi_exec( + "REPLACE INTO ftsdocs(type,rid,name,idxed)" + " VALUES(%Q,%d,%Q,0)", + zType, rid, zName + ); + if( cType=='w' || cType=='e' ){ + db_multi_exec( + "DELETE FROM ftsidx WHERE rowid IN" + " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)", + cType, zName + ); + db_multi_exec( + "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d", + cType, zName, rid + ); + } + /* All forum posts are always indexed */ + } +} + +/* +** If the doc-glob and doc-br settings are valid for document search +** and if the latest check-in on doc-br is in the unindexed set of +** check-ins, then update all 'd' entries in FTSDOCS that have +** changed. +*/ +static void search_update_doc_index(void){ + const char *zMainBranch = db_main_branch(); + const char *zDocBranches = db_get("doc-branch", zMainBranch); + int i; + Glob * pGlob = glob_create(zDocBranches) + /* We're misusing a Glob as a list of comma-/space-delimited + ** tokens. We're not actually doing glob matches here. */; + if( !pGlob ) return; + db_multi_exec( + "CREATE TEMP TABLE current_docs(rid INTEGER PRIMARY KEY, name);" + "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;" + ); + for( i = 0; i < pGlob->nPattern; ++i ){ + const char *zDocBr = pGlob->azPattern[i]; + int ckid = symbolic_name_to_rid(zDocBr,"ci"); + double rTime; + if( !db_exists("SELECT 1 FROM ftsdocs WHERE type='c' AND rid=%d" + " AND NOT idxed", ckid) ) continue; + /* If we get this far, it means that changes to 'd' entries are + ** required. */ + rTime = db_double(0.0, "SELECT mtime FROM event WHERE objid=%d", ckid); + db_multi_exec( + "INSERT OR IGNORE INTO current_docs(rid, name)" + " SELECT blob.rid, foci.filename FROM foci, blob" + " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid" + " AND %z", + ckid, glob_expr("foci.filename", db_get("doc-glob","")) + ); + db_multi_exec( + "DELETE FROM ftsidx WHERE rowid IN" + " (SELECT rowid FROM ftsdocs WHERE type='d'" + " AND rid NOT IN (SELECT rid FROM current_docs))" + ); + db_multi_exec( + "DELETE FROM ftsdocs WHERE type='d'" + " AND rid NOT IN (SELECT rid FROM current_docs)" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,bx,url,mtime)" + " SELECT 'd', rid, name, 0," + " title('d',rid,name)," + " body('d',rid,name)," + " printf('/doc/%T/%%s',urlencode(name))," + " %.17g" + " FROM current_docs", + zDocBr, rTime + ); + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed" + ); + db_multi_exec( + "UPDATE ftsdocs SET" + " idxed=1," + " bx=NULL," + " label='Document: '||label" + " WHERE type='d' AND NOT idxed" + ); + } + glob_free(pGlob); +} + +/* +** Deal with all of the unindexed 'c' terms in FTSDOCS +*/ +static void search_update_checkin_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs" + " WHERE type='c' AND NOT idxed;" + ); + db_multi_exec( + "UPDATE ftsdocs SET idxed=1, name=NULL," + " (label,url,mtime) = " + " (SELECT printf('Check-in [%%.16s] on %%s',blob.uuid," + " datetime(event.mtime))," + " printf('/timeline?y=ci&c=%%.20s',blob.uuid)," + " event.mtime" + " FROM event, blob" + " WHERE event.objid=ftsdocs.rid" + " AND blob.rid=ftsdocs.rid)" + "WHERE ftsdocs.type='c' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 't' terms in FTSDOCS +*/ +static void search_update_ticket_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs" + " WHERE type='t' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1, name=NULL," + " (label,url,mtime) =" + " (SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,null)," + " datetime(tkt_mtime))," + " printf('/tktview/%%.20s',tkt_uuid)," + " tkt_mtime" + " FROM ticket" + " WHERE tkt_id=ftsdocs.rid)" + "WHERE ftsdocs.type='t' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 'w' terms in FTSDOCS +*/ +static void search_update_wiki_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs" + " WHERE type='w' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1," + " (name,label,url,mtime) = " + " (SELECT ftsdocs.name," + " 'Wiki: '||ftsdocs.name," + " '/wiki?name='||urlencode(ftsdocs.name)," + " tagxref.mtime" + " FROM tagxref WHERE tagxref.rid=ftsdocs.rid)" + " WHERE ftsdocs.type='w' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 'f' terms in FTSDOCS +*/ +static void search_update_forum_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs" + " WHERE type='f' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1, name=NULL," + " (label,url,mtime) = " + " (SELECT 'Forum '||event.comment," + " '/forumpost/'||blob.uuid," + " event.mtime" + " FROM event, blob" + " WHERE event.objid=ftsdocs.rid" + " AND blob.rid=ftsdocs.rid)" + "WHERE ftsdocs.type='f' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 'e' terms in FTSDOCS +*/ +static void search_update_technote_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs" + " WHERE type='e' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1," + " (name,label,url,mtime) = " + " (SELECT ftsdocs.name," + " 'Tech Note: '||ftsdocs.name," + " '/technote/'||substr(tag.tagname,7)," + " tagxref.mtime" + " FROM tagxref, tag USING (tagid)" + " WHERE tagxref.rid=ftsdocs.rid" + " AND tagname GLOB 'event-*')" + " WHERE ftsdocs.type='e' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed entries in the FTSDOCS table - that +** is to say, all the entries with FTSDOCS.IDXED=0. Add them to the +** index. +*/ +void search_update_index(unsigned int srchFlags){ + if( !search_index_exists() ) return; + if( !db_exists("SELECT 1 FROM ftsdocs WHERE NOT idxed") ) return; + search_sql_setup(g.db); + db_unprotect(PROTECT_READONLY); + if( srchFlags & (SRCH_CKIN|SRCH_DOC) ){ + search_update_doc_index(); + search_update_checkin_index(); + } + if( srchFlags & SRCH_TKT ){ + search_update_ticket_index(); + } + if( srchFlags & SRCH_WIKI ){ + search_update_wiki_index(); + } + if( srchFlags & SRCH_TECHNOTE ){ + search_update_technote_index(); + } + if( srchFlags & SRCH_FORUM ){ + search_update_forum_index(); + } + db_protect_pop(); +} + +/* +** Construct, prepopulate, and then update the full-text index. +*/ +void search_rebuild_index(void){ + fossil_print("rebuilding the search index..."); + fflush(stdout); + search_create_index(); + search_fill_index(); + search_update_index(search_restrict(SRCH_ALL)); + if( db_table_exists("repository","chat") ){ + chat_rebuild_index(1); + } + fossil_print(" done\n"); +} + +/* +** COMMAND: fts-config* abbrv-subcom +** +** Usage: fossil fts-config ?SUBCOMMAND? ?ARGUMENT? +** +** The "fossil fts-config" command configures the full-text search capabilities +** of the repository. Subcommands: +** +** reindex Rebuild the search index. This is a no-op if +** index search is disabled +** +** index (on|off) Turn the search index on or off +** +** enable TYPE .. Enable search for TYPE. TYPE is one of: +** check-in, document, ticket, wiki, technote, +** forum, help, or all +** +** disable TYPE ... Disable search for TYPE +** +** tokenizer VALUE Select a tokenizer for indexed search. VALUE +** may be one of (porter, on, off, trigram, unicode61), +** and "on" is equivalent to "porter". Unindexed +** search never uses tokenization or stemming. +** +** The current search settings are displayed after any changes are applied. +** Run this command with no arguments to simply see the settings. +*/ +void fts_config_cmd(void){ + static const struct { + int iCmd; + const char *z; + } aCmd[] = { + { 1, "reindex" }, + { 2, "index" }, + { 3, "disable" }, + { 4, "enable" }, + { 5, "tokenizer"}, + }; + static const struct { + const char *zSetting; + const char *zName; + const char *zSw; + } aSetng[] = { + { "search-ci", "check-in search:", "c" }, + { "search-doc", "document search:", "d" }, + { "search-tkt", "ticket search:", "t" }, + { "search-wiki", "wiki search:", "w" }, + { "search-technote", "technote search:", "e" }, + { "search-forum", "forum search:", "f" }, + { "search-help", "built-in help search:", "h" }, + }; + char *zSubCmd = 0; + int i, j, n; + int iCmd = 0; + int iAction = 0; + db_find_and_open_repository(0, 0); + if( g.argc>2 ){ + zSubCmd = g.argv[2]; + n = (int)strlen(zSubCmd); + for(i=0; i<count(aCmd); i++){ + if( fossil_strncmp(aCmd[i].z, zSubCmd, n)==0 ) break; + } + if( i>=count(aCmd) ){ + Blob all; + blob_init(&all,0,0); + for(i=0; i<count(aCmd); i++) blob_appendf(&all, " %s", aCmd[i].z); + fossil_fatal("unknown \"%s\" - should be one of:%s", + zSubCmd, blob_str(&all)); + return; + } + iCmd = aCmd[i].iCmd; + } + g.perm.Read = 1; + g.perm.RdTkt = 1; + g.perm.RdWiki = 1; + if( iCmd==1 ){ + if( search_index_exists() ) iAction = 2; + } + if( iCmd==2 ){ + if( g.argc<3 ) usage("index (on|off)"); + iAction = 1 + is_truth(g.argv[3]); + } + db_begin_transaction(); + + /* Adjust search settings */ + if( iCmd==3 || iCmd==4 ){ + int k; + const char *zCtrl; + for(k=2; k<g.argc; k++){ + if( k==2 ){ + if( g.argc<4 ){ + zCtrl = "all"; + }else{ + zCtrl = g.argv[3]; + k++; + } + }else{ + zCtrl = g.argv[k]; + } + if( fossil_strcmp(zCtrl,"all")==0 ){ + zCtrl = "cdtwefh"; + } + if( strlen(zCtrl)>=4 ){ + /* If the argument to "enable" or "disable" is a string of at least + ** 4 characters which matches part of any aSetng.zName, then use that + ** one aSetng value only. */ + char *zGlob = mprintf("*%s*", zCtrl); + for(j=0; j<count(aSetng); j++){ + if( sqlite3_strglob(zGlob, aSetng[j].zName)==0 ){ + db_set_int(aSetng[j].zSetting/*works-like:"x"*/, iCmd-3, 0); + zCtrl = 0; + break; + } + } + fossil_free(zGlob); + } + if( zCtrl ){ + for(j=0; j<count(aSetng); j++){ + if( strchr(zCtrl, aSetng[j].zSw[0])!=0 ){ + db_set_int(aSetng[j].zSetting/*works-like:"x"*/, iCmd-3, 0); + } + } + } + } + }else if( iCmd==5 ){ + int iOldTokenizer, iNewTokenizer; + if( g.argc<4 ) usage("tokenizer porter|on|off|trigram|unicode61"); + iOldTokenizer = search_tokenizer_type(0); + db_set("search-tokenizer", + search_tokenizer_for_string(g.argv[3]), 0); + iNewTokenizer = search_tokenizer_type(1); + if( iOldTokenizer!=iNewTokenizer ){ + /* Drop or rebuild index if tokenizer changes. */ + iAction = 1 + ((iOldTokenizer && iNewTokenizer) + ? 1 : (iNewTokenizer ? 1 : 0)); + } + } + + /* destroy or rebuild the index, if requested */ + if( iAction>=1 ){ + search_drop_index(); + } + if( iAction>=2 ){ + search_rebuild_index(); + } + + /* Always show the status before ending */ + for(i=0; i<count(aSetng); i++){ + fossil_print("%-21s %s\n", aSetng[i].zName, + db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off"); + } + fossil_print("%-21s %s\n", "tokenizer:", + search_tokenizer_for_string(0)); + if( search_index_exists() ){ + int pgsz = db_int64(0, "PRAGMA repository.page_size;"); + i64 nTotal = db_int64(0, "PRAGMA repository.page_count;")*pgsz; + i64 nFts = db_int64(0, "SELECT count(*) FROM dbstat" + " WHERE schema='repository'" + " AND name LIKE 'fts%%'")*pgsz; + char zSize[50]; + fossil_print("%-21s FTS%d\n", "full-text index:", search_index_type(1)); + fossil_print("%-21s %d\n", "documents:", + db_int(0, "SELECT count(*) FROM ftsdocs")); + approxSizeName(sizeof(zSize), zSize, nFts); + fossil_print("%-21s %s (%.1f%% of repository)\n", "space used", + zSize, 100.0*((double)nFts/(double)nTotal)); + }else{ + fossil_print("%-21s disabled\n", "full-text index:"); + } + db_end_transaction(0); +} + +/* +** WEBPAGE: test-ftsdocs +** +** Show a table of all documents currently in the search index. +*/ +void search_data_page(void){ + Stmt q; + const char *zId = P("id"); + const char *zType = P("y"); + const char *zIdxed = P("ixed"); + int id; + int cnt1 = 0, cnt2 = 0, cnt3 = 0; + login_check_credentials(); + if( !g.perm.Admin ){ login_needed(0); return; } + style_set_current_feature("test"); + if( !search_index_exists() ){ + @ <p>Indexed search is disabled + style_finish_page(); + return; + } + search_sql_setup(g.db); + style_submenu_element("Setup","%R/srchsetup"); + if( zId!=0 && (id = atoi(zId))>0 ){ + /* Show information about a single ftsdocs entry */ + style_header("Information about ftsdoc entry %d", id); + style_submenu_element("Summary","%R/test-ftsdocs"); + db_prepare(&q, + "SELECT type||rid, name, idxed, label, url, datetime(mtime)" + " FROM ftsdocs WHERE rowid=%d", id + ); + if( db_step(&q)==SQLITE_ROW ){ + const char *zUrl = db_column_text(&q,4); + const char *zDocId = db_column_text(&q,0); + char *zName; + char *z; + @ <table border=0> + @ <tr><td align='right'>rowid:<td> <td>%d(id) + @ <tr><td align='right'>id:<td><td>%s(zDocId) + @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1)) + @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2)) + @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3)) + @ <tr><td align='right'>url:<td><td> + @ <a href='%R%s(zUrl)'>%h(zUrl)</a> + @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5)) + z = db_text(0, "SELECT title FROM ftsidx WHERE rowid=%d",id); + if( z && z[0] ){ + @ <tr><td align="right">title:<td><td>%h(z) + fossil_free(z); + } + z = db_text(0, "SELECT body FROM ftsidx WHERE rowid=%d",id); + if( z && z[0] ){ + @ <tr><td align="right" valign="top">body:<td><td>%h(z) + fossil_free(z); + } + @ </table> + zName = mprintf("Indexed '%c' docs",zDocId[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=1",zDocId[0]); + zName = mprintf("Unindexed '%c' docs",zDocId[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=0",zDocId[0]); + } + db_finalize(&q); + style_finish_page(); + return; + } + if( zType!=0 && zType[0]!=0 && zType[1]==0 && + zIdxed!=0 && (zIdxed[0]=='1' || zIdxed[0]=='0') && zIdxed[1]==0 + ){ + int ixed = zIdxed[0]=='1'; + char *zName; + style_header("List of '%c' documents that are%s indexed", + zType[0], ixed ? "" : " not"); + style_submenu_element("Summary","%R/test-ftsdocs"); + if( ixed==0 ){ + zName = mprintf("Indexed '%c' docs",zType[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=1",zType[0]); + }else{ + zName = mprintf("Unindexed '%c' docs",zType[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=0",zType[0]); + } + db_prepare(&q, + "SELECT rowid, type||rid ||' '|| coalesce(label,'')" + " FROM ftsdocs WHERE type='%c' AND %s idxed", + zType[0], ixed ? "" : "NOT" + ); + @ <ul> + while( db_step(&q)==SQLITE_ROW ){ + @ <li> <a href='test-ftsdocs?id=%d(db_column_int(&q,0))'> + @ %h(db_column_text(&q,1))</a> + } + @ </ul> + db_finalize(&q); + style_finish_page(); + return; + } + style_header("Summary of ftsdocs"); + db_prepare(&q, + "SELECT type, sum(idxed IS TRUE), sum(idxed IS FALSE), count(*)" + " FROM ftsdocs" + " GROUP BY 1 ORDER BY 4 DESC" + ); + @ <table border=1 cellpadding=3 cellspacing=0> + @ <thead> + @ <tr><th>Type<th>Indexed<th>Unindexed<th>Total + @ </thead> + @ <tbody> + while( db_step(&q)==SQLITE_ROW ){ + const char *zType = db_column_text(&q,0); + int nIndexed = db_column_int(&q, 1); + int nUnindexed = db_column_int(&q, 2); + int nTotal = db_column_int(&q, 3); + @ <tr><td>%h(zType) + if( nIndexed>0 ){ + @ <td align="right"><a href='%R/test-ftsdocs?y=%s(zType)&ixed=1'>\ + @ %d(nIndexed)</a> + }else{ + @ <td align="right">0 + } + if( nUnindexed>0 ){ + @ <td align="right"><a href='%R/test-ftsdocs?y=%s(zType)&ixed=0'>\ + @ %d(nUnindexed)</a> + }else{ + @ <td align="right">0 + } + @ <td align="right">%d(nTotal) + @ </tr> + cnt1 += nIndexed; + cnt2 += nUnindexed; + cnt3 += nTotal; + } + db_finalize(&q); + @ </tbody><tfooter> + @ <tr><th>Total<th align="right">%d(cnt1)<th align="right">%d(cnt2) + @ <th align="right">%d(cnt3) + @ </tfooter> + @ </table> + style_finish_page(); +} + + +/* +** The Fts5MatchinfoCtx bits were all taken verbatim from: +** +** https://sqlite.org/src/finfo?name=ext/fts5/fts5_test_mi.c +*/ + +typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx; + +#if INTERFACE +#ifndef SQLITE_AMALGAMATION +typedef unsigned int u32; +#endif +#endif + +struct Fts5MatchinfoCtx { + int nCol; /* Number of cols in FTS5 table */ + int nPhrase; /* Number of phrases in FTS5 query */ + char *zArg; /* nul-term'd copy of 2nd arg */ + int nRet; /* Number of elements in aRet[] */ + u32 *aRet; /* Array of 32-bit unsigned ints to return */ +}; + + +/* +** Return a pointer to the fts5_api pointer for database connection db. +** If an error occurs, return NULL and leave an error in the database +** handle (accessible using sqlite3_errcode()/errmsg()). +*/ +static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){ + sqlite3_stmt *pStmt = 0; + int rc; + + *ppApi = 0; + rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0); + (void)sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + } + + return rc; +} + + +/* +** Argument f should be a flag accepted by matchinfo() (a valid character +** in the string passed as the second argument). If it is not, -1 is +** returned. Otherwise, if f is a valid matchinfo flag, the value returned +** is the number of 32-bit integers added to the output array if the +** table has nCol columns and the query nPhrase phrases. +*/ +static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){ + int ret = -1; + switch( f ){ + case 'p': ret = 1; break; + case 'c': ret = 1; break; + case 'x': ret = 3 * nCol * nPhrase; break; + case 'y': ret = nCol * nPhrase; break; + case 'b': ret = ((nCol + 31) / 32) * nPhrase; break; + case 'n': ret = 1; break; + case 'a': ret = nCol; break; + case 'l': ret = nCol; break; + case 's': ret = nCol; break; + } + return ret; +} + +static int fts5MatchinfoIter( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + Fts5MatchinfoCtx *p, + int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*) +){ + int i; + int n = 0; + int rc = SQLITE_OK; + char f; + for(i=0; (f = p->zArg[i]); i++){ + rc = x(pApi, pFts, p, f, &p->aRet[n]); + if( rc!=SQLITE_OK ) break; + n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f); + } + return rc; +} + +static int fts5MatchinfoXCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pUserData +){ + Fts5PhraseIter iter; + int iCol, iOff; + u32 *aOut = (u32*)pUserData; + int iPrev = -1; + + for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); + iCol>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + aOut[iCol*3+1]++; + if( iCol!=iPrev ) aOut[iCol*3 + 2]++; + iPrev = iCol; + } + + return SQLITE_OK; +} + +static int fts5MatchinfoGlobalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int rc = SQLITE_OK; + switch( f ){ + case 'p': + aOut[0] = p->nPhrase; + break; + + case 'c': + aOut[0] = p->nCol; + break; + + case 'x': { + int i; + for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){ + void *pPtr = (void*)&aOut[i * p->nCol * 3]; + rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb); + } + break; + } + + case 'n': { + sqlite3_int64 nRow; + rc = pApi->xRowCount(pFts, &nRow); + aOut[0] = (u32)nRow; + break; + } + + case 'a': { + sqlite3_int64 nRow = 0; + rc = pApi->xRowCount(pFts, &nRow); + if( nRow==0 ){ + memset(aOut, 0, sizeof(u32) * p->nCol); + }else{ + int i; + for(i=0; rc==SQLITE_OK && i<p->nCol; i++){ + sqlite3_int64 nToken; + rc = pApi->xColumnTotalSize(pFts, i, &nToken); + if( rc==SQLITE_OK){ + aOut[i] = (u32)((2*nToken + nRow) / (2*nRow)); + } + } + } + break; + } + + } + return rc; +} + +static int fts5MatchinfoLocalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int i; + int rc = SQLITE_OK; + + switch( f ){ + case 'b': { + int iPhrase; + int nInt = ((p->nCol + 31) / 32) * p->nPhrase; + for(i=0; i<nInt; i++) aOut[i] = 0; + + for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iCol; + for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); + iCol>=0; + pApi->xPhraseNextColumn(pFts, &iter, &iCol) + ){ + aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); + } + } + + break; + } + + case 'x': + case 'y': { + int nMul = (f=='x' ? 3 : 1); + int iPhrase; + + for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; + + for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iOff, iCol; + for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); + iOff>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + aOut[nMul * (iCol + iPhrase * p->nCol)]++; + } + } + + break; + } + + case 'l': { + for(i=0; rc==SQLITE_OK && i<p->nCol; i++){ + int nToken; + rc = pApi->xColumnSize(pFts, i, &nToken); + aOut[i] = (u32)nToken; + } + break; + } + + case 's': { + int nInst; + + memset(aOut, 0, sizeof(u32) * p->nCol); + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; rc==SQLITE_OK && i<nInst; i++){ + int iPhrase, iOff, iCol = 0; + int iNextPhrase; + int iNextOff; + u32 nSeq = 1; + int j; + + rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff); + iNextPhrase = iPhrase+1; + iNextOff = iOff+pApi->xPhraseSize(pFts, 0); + for(j=i+1; rc==SQLITE_OK && j<nInst; j++){ + int ip, ic, io; + rc = pApi->xInst(pFts, j, &ip, &ic, &io); + if( ic!=iCol || io>iNextOff ) break; + if( ip==iNextPhrase && io==iNextOff ){ + nSeq++; + iNextPhrase = ip+1; + iNextOff = io + pApi->xPhraseSize(pFts, ip); + } + } + + if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq; + } + + break; + } + } + return rc; +} + +static Fts5MatchinfoCtx *fts5MatchinfoNew( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning error message */ + const char *zArg /* Matchinfo flag string */ +){ + Fts5MatchinfoCtx *p; + int nCol; + int nPhrase; + int i; + int nInt; + sqlite3_int64 nByte; + int rc; + + nCol = pApi->xColumnCount(pFts); + nPhrase = pApi->xPhraseCount(pFts); + + nInt = 0; + for(i=0; zArg[i]; i++){ + int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]); + if( n<0 ){ + char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return 0; + } + nInt += n; + } + + nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */ + + sizeof(u32) * nInt /* The p->aRet[] array */ + + (i+1); /* The p->zArg string */ + p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, nByte); + + p->nCol = nCol; + p->nPhrase = nPhrase; + p->aRet = (u32*)&p[1]; + p->nRet = nInt; + p->zArg = (char*)&p->aRet[nInt]; + memcpy(p->zArg, zArg, i); + + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(p); + p = 0; + } + + return p; +} + +static void fts5MatchinfoFunc( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + const char *zArg; + Fts5MatchinfoCtx *p; + int rc = SQLITE_OK; + + if( nVal>0 ){ + zArg = (const char*)sqlite3_value_text(apVal[0]); + }else{ + zArg = "pcx"; + } + + p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0); + if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){ + p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } + } + + if( rc==SQLITE_OK ){ + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + }else{ + /* No error has occurred, so return a copy of the array of integers. */ + int nByte = p->nRet * sizeof(u32); + sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT); + } +} + +int db_register_fts5(sqlite3 *db){ + int rc; /* Return code */ + fts5_api *pApi; /* FTS5 API functions */ + + /* Extract the FTS5 API pointer from the database handle. The + ** fts5_api_from_db() function above is copied verbatim from the + ** FTS5 documentation. Refer there for details. */ + rc = fts5_api_from_db(db, &pApi); + if( rc!=SQLITE_OK ) return rc; + + /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered + ** with this database handle, or an error (OOM perhaps?) has occurred. + ** + ** Also check that the fts5_api object is version 2 or newer. + */ + if( pApi==0 || pApi->iVersion<2 ){ + return SQLITE_ERROR; + } + + /* Register the implementation of matchinfo() */ + rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); + + return rc; +} |