From 4f49b16079b27d7ce633619e4cddfdf2bffbc952 Mon Sep 17 00:00:00 2001 From: cashmere Date: Fri, 4 Sep 2026 09:05:24 +0200 Subject: initial commit --- src/search.c | 2905 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2905 insertions(+) create mode 100644 src/search.c (limited to 'src/search.c') diff --git a/src/search.c b/src/search.c new file mode 100644 index 0000000..1015fd1 --- /dev/null +++ b/src/search.c @@ -0,0 +1,2905 @@ +/* +** Copyright (c) 2009 D. Richard Hipp +** +** This program is free software; you can redistribute it and/or +** modify it under the terms of the Simplified BSD License (also +** known as the "2-Clause License" or "FreeBSD License".) + +** This program is distributed in the hope that it will be useful, +** but without any warranty; without even the implied warranty of +** merchantability or fitness for a particular purpose. +** +** Author contact information: +** drh@hwaci.com +** http://www.hwaci.com/drh/ +** +******************************************************************************* +** +** This file contains code to implement a search functions +** against timeline comments, check-in content, wiki pages, tickets, +** and/or forum posts. +** +** The search can be either a per-query "grep"-like search that scans +** the entire corpus. Or it can use the FTS5 search engine of SQLite. +** The choice is an administrator configuration option. +** +** The first option is referred to as "full-scan search". The second +** option is called "indexed search". +** +** The code in this file is ordered approximately as follows: +** +** (1) The full-scan search engine +** (2) The indexed search engine +** (3) Higher level interfaces that use either (1) or (b2) according +** to the current search configuration settings +*/ +#include "config.h" +#include "search.h" +#include + +#if INTERFACE + +/* Maximum number of search terms for full-scan search */ +#define SEARCH_MAX_TERM 8 + +/* +** A compiled search pattern used for full-scan search. +*/ +struct Search { + int nTerm; /* Number of search terms */ + struct srchTerm { /* For each search term */ + char *z; /* Text */ + int n; /* length */ + } a[SEARCH_MAX_TERM]; + /* Snippet controls */ + char *zPattern; /* The search pattern */ + char *zMarkBegin; /* Start of a match */ + char *zMarkEnd; /* End of a match */ + char *zMarkGap; /* A gap between two matches */ + unsigned fSrchFlg; /* Flags */ + int iScore; /* Score of the last match attempt */ + Blob snip; /* Snippet for the most recent match */ +}; + +#define SRCHFLG_HTML 0x01 /* Escape snippet text for HTML */ +#define SRCHFLG_STATIC 0x04 /* The static gSearch object */ + +#endif + +/* +** There is a single global Search object: +*/ +static Search gSearch; + + +/* +** Theses characters constitute a word boundary +*/ +static const char isBoundary[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define ISALNUM(x) (!isBoundary[(x)&0xff]) + + +/* +** Destroy a full-scan search context. +*/ +void search_end(Search *p){ + if( p ){ + fossil_free(p->zPattern); + fossil_free(p->zMarkBegin); + fossil_free(p->zMarkEnd); + fossil_free(p->zMarkGap); + if( p->iScore ) blob_reset(&p->snip); + memset(p, 0, sizeof(*p)); + if( p!=&gSearch ) fossil_free(p); + } +} + +/* +** Compile a full-scan search pattern +*/ +static Search *search_init( + const char *zPattern, /* The search pattern */ + const char *zMarkBegin, /* Start of a match */ + const char *zMarkEnd, /* End of a match */ + const char *zMarkGap, /* A gap between two matches */ + unsigned fSrchFlg /* Flags */ +){ + Search *p; + char *z; + int i; + + if( fSrchFlg & SRCHFLG_STATIC ){ + p = &gSearch; + search_end(p); + }else{ + p = fossil_malloc(sizeof(*p)); + memset(p, 0, sizeof(*p)); + } + p->zPattern = z = mprintf("%s",zPattern); + p->zMarkBegin = mprintf("%s",zMarkBegin); + p->zMarkEnd = mprintf("%s",zMarkEnd); + p->zMarkGap = mprintf("%s",zMarkGap); + p->fSrchFlg = fSrchFlg; + blob_init(&p->snip, 0, 0); + while( *z && p->nTerma[p->nTerm].z = z; + for(i=1; ISALNUM(z[i]); i++){} + p->a[p->nTerm].n = i; + z += i; + p->nTerm++; + } + return p; +} + + +/* +** Append n bytes of text to snippet zTxt. Encode the text appropriately. +*/ +static void snippet_text_append( + Search *p, /* The search context */ + Blob *pSnip, /* Append to this snippet */ + const char *zTxt, /* Text to append */ + int n /* How many bytes to append */ +){ + if( n>0 ){ + if( p->fSrchFlg & SRCHFLG_HTML ){ + blob_appendf(pSnip, "%#h", n, zTxt); + }else{ + blob_append(pSnip, zTxt, n); + } + } +} + +/* This the core search engine for full-scan search. +** +** Compare a search pattern against one or more input strings which +** collectively comprise a document. Return a match score. Any +** postive value means there was a match. Zero means that one or +** more terms are missing. +** +** The score and a snippet are record for future use. +** +** Scoring: +** * All terms must match at least once or the score is zero +** * One point for each matching term +** * Extra points if consecutive words of the pattern are consecutive +** in the document +*/ +static int search_match( + Search *p, /* Search pattern and flags */ + int nDoc, /* Number of strings in this document */ + const char **azDoc /* Text of each string */ +){ + int score; /* Final score */ + int i; /* Offset into current document */ + int ii; /* Loop counter */ + int j; /* Loop over search terms */ + int k; /* Loop over prior terms */ + int iWord = 0; /* Current word number */ + int iDoc; /* Current document number */ + int wantGap = 0; /* True if a zMarkGap is wanted */ + const char *zDoc; /* Current document text */ + const int CTX = 50; /* Amount of snippet context */ + int anMatch[SEARCH_MAX_TERM]; /* Number of terms in best match */ + int aiBestDoc[SEARCH_MAX_TERM]; /* Document containing best match */ + int aiBestOfst[SEARCH_MAX_TERM]; /* Byte offset to start of best match */ + int aiLastDoc[SEARCH_MAX_TERM]; /* Document containing most recent match */ + int aiLastOfst[SEARCH_MAX_TERM]; /* Byte offset to the most recent match */ + int aiWordIdx[SEARCH_MAX_TERM]; /* Word index of most recent match */ + + memset(anMatch, 0, sizeof(anMatch)); + memset(aiWordIdx, 0xff, sizeof(aiWordIdx)); + for(iDoc=0; iDocnTerm; j++){ + int n = p->a[j].n; + if( sqlite3_strnicmp(p->a[j].z, &zDoc[i], n)==0 + && (!ISALNUM(zDoc[i+n]) || p->a[j].z[n]=='*') + ){ + aiWordIdx[j] = iWord; + aiLastDoc[j] = iDoc; + aiLastOfst[j] = i; + for(k=1; j-k>=0 && anMatch[j-k] && aiWordIdx[j-k]==iWord-k; k++){} + for(ii=0; iinTerm; j++) score *= anMatch[j]; + blob_reset(&p->snip); + p->iScore = score; + if( score==0 ) return score; + + + /* Prepare a snippet that describes the matching text. + */ + while(1){ + int iOfst; + int iTail; + int iBest; + for(ii=0; iinTerm && anMatch[ii]==0; ii++){} + if( ii>=p->nTerm ) break; /* This is where the loop exits */ + iBest = ii; + iDoc = aiBestDoc[ii]; + iOfst = aiBestOfst[ii]; + for(; iinTerm; ii++){ + if( anMatch[ii]==0 ) continue; + if( aiBestDoc[ii]>iDoc ) continue; + if( aiBestOfst[ii]>iOfst ) continue; + iDoc = aiBestDoc[ii]; + iOfst = aiBestOfst[ii]; + iBest = ii; + } + iTail = iOfst + p->a[iBest].n; + anMatch[iBest] = 0; + for(ii=0; iinTerm; ii++){ + if( anMatch[ii]==0 ) continue; + if( aiBestDoc[ii]!=iDoc ) continue; + if( aiBestOfst[ii]<=iTail+CTX*2 ){ + if( iTaila[ii].n ){ + iTail = aiBestOfst[ii]+p->a[ii].n; + } + anMatch[ii] = 0; + ii = -1; + continue; + } + } + zDoc = azDoc[iDoc]; + iOfst -= CTX; + if( iOfst<0 ) iOfst = 0; + while( iOfst>0 && ISALNUM(zDoc[iOfst-1]) ) iOfst--; + while( zDoc[iOfst] && !ISALNUM(zDoc[iOfst]) ) iOfst++; + for(ii=0; ii0 || wantGap ) blob_append(&p->snip, p->zMarkGap, -1); + wantGap = zDoc[iTail]!=0; + zDoc += iOfst; + iTail -= iOfst; + + /* Add a snippet segment using characters iOfst..iOfst+iTail from zDoc */ + for(i=0; inTerm; j++){ + int n = p->a[j].n; + if( sqlite3_strnicmp(p->a[j].z, &zDoc[i], n)==0 + && (!ISALNUM(zDoc[i+n]) || p->a[j].z[n]=='*') + ){ + snippet_text_append(p, &p->snip, zDoc, i); + zDoc += i; + iTail -= i; + blob_append(&p->snip, p->zMarkBegin, -1); + if( p->a[j].z[n]=='*' ){ + while( ISALNUM(zDoc[n]) ) n++; + } + snippet_text_append(p, &p->snip, zDoc, n); + zDoc += n; + iTail -= n; + blob_append(&p->snip, p->zMarkEnd, -1); + i = -1; + break; + } /* end-if */ + } /* end for(j) */ + if( jnTerm ){ + while( ISALNUM(zDoc[i]) && isnip, zDoc, iTail); + } + if( wantGap ) blob_append(&p->snip, p->zMarkGap, -1); + return score; +} + +/* +** COMMAND: test-match +** +** Usage: %fossil test-match SEARCHSTRING FILE1 FILE2 ... +** +** Run the full-scan search algorithm using SEARCHSTRING against +** the text of the files listed. Output matches and snippets. +** +** Options: +** --begin TEXT Text to insert before each match +** --end TEXT Text to insert after each match +** --gap TEXT Text to indicate elided content +** --html Input is HTML +** --static Use the static Search object +*/ +void test_match_cmd(void){ + Search *p; + int i; + Blob x; + int score; + char *zDoc; + int flg = 0; + char *zBegin = (char*)find_option("begin",0,1); + char *zEnd = (char*)find_option("end",0,1); + char *zGap = (char*)find_option("gap",0,1); + if( find_option("html",0,0)!=0 ) flg |= SRCHFLG_HTML; + if( find_option("static",0,0)!=0 ) flg |= SRCHFLG_STATIC; + verify_all_options(); + if( g.argc<4 ) usage("SEARCHSTRING FILE1..."); + if( zBegin==0 ) zBegin = "[["; + if( zEnd==0 ) zEnd = "]]"; + if( zGap==0 ) zGap = " ... "; + p = search_init(g.argv[2], zBegin, zEnd, zGap, flg); + for(i=3; iiScore); + blob_reset(&x); + if( score ){ + fossil_print("%.78c\n%s\n%.78c\n\n", '=', blob_str(&p->snip), '='); + } + } + search_end(p); +} + +/* +** An SQL function to initialize the full-scan search pattern: +** +** search_init(PATTERN,BEGIN,END,GAP,FLAGS) +** +** All arguments are optional. PATTERN is the search pattern. If it +** is omitted, then the global search pattern is reset. BEGIN and END +** and GAP are the strings used to construct snippets. FLAGS is an +** integer bit pattern containing the various SRCH_CKIN, SRCH_DOC, +** SRCH_TKT, SRCH_FORUM, or SRCH_ALL bits to determine what is to be +** searched. +*/ +static void search_init_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPattern = 0; + const char *zBegin = ""; + const char *zEnd = ""; + const char *zGap = " ... "; + unsigned int flg = SRCHFLG_HTML; + switch( argc ){ + default: + flg = (unsigned int)sqlite3_value_int(argv[4]); + case 4: + zGap = (const char*)sqlite3_value_text(argv[3]); + case 3: + zEnd = (const char*)sqlite3_value_text(argv[2]); + case 2: + zBegin = (const char*)sqlite3_value_text(argv[1]); + case 1: + zPattern = (const char*)sqlite3_value_text(argv[0]); + } + if( zPattern && zPattern[0] ){ + search_init(zPattern, zBegin, zEnd, zGap, flg | SRCHFLG_STATIC); + }else{ + search_end(&gSearch); + } +} + +/* search_match(TEXT, TEXT, ....) +** +** Using the full-scan search engine created by the most recent call +** to search_init(), match the input the TEXT arguments. +** Remember the results in the global full-scan search object. +** Return non-zero on a match and zero on a miss. +*/ +static void search_match_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *azDoc[5]; + int nDoc; + int rc; + for(nDoc=0; nDoc0 ){ + sqlite3_result_text(context, blob_str(&gSearch.snip), -1, fossil_free); + blob_init(&gSearch.snip, 0, 0); + } +} + +/* stext(TYPE, RID, ARG) +** +** This is an SQLite function that computes the searchable text. +** It is a wrapper around the search_stext() routine. See the +** search_stext() routine for further detail. +*/ +static void search_stext_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zType = (const char*)sqlite3_value_text(argv[0]); + int rid = sqlite3_value_int(argv[1]); + const char *zName = (const char*)sqlite3_value_text(argv[2]); + sqlite3_result_text(context, search_stext_cached(zType[0],rid,zName,0), -1, + SQLITE_TRANSIENT); +} + +/* title(TYPE, RID, ARG) +** +** Return the title of the document to be search. +*/ +static void search_title_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zType = (const char*)sqlite3_value_text(argv[0]); + int rid = sqlite3_value_int(argv[1]); + const char *zName = (const char*)sqlite3_value_text(argv[2]); + int nHdr = 0; + char *z = search_stext_cached(zType[0], rid, zName, &nHdr); + if( nHdr || zType[0]!='d' ){ + sqlite3_result_text(context, z, nHdr, SQLITE_TRANSIENT); + }else{ + sqlite3_result_value(context, argv[2]); + } +} + +/* body(TYPE, RID, ARG) +** +** Return the body of the document to be search. +*/ +static void search_body_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zType = (const char*)sqlite3_value_text(argv[0]); + int rid = sqlite3_value_int(argv[1]); + const char *zName = (const char*)sqlite3_value_text(argv[2]); + int nHdr = 0; + char *z = search_stext_cached(zType[0], rid, zName, &nHdr); + sqlite3_result_text(context, z+nHdr+1, -1, SQLITE_TRANSIENT); +} + +/* urlencode(X) +** +** Encode a string for use as a query parameter in a URL. This is +** the equivalent of printf("%T",X). +*/ +static void search_urlencode_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + char *z = mprintf("%T",sqlite3_value_text(argv[0])); + sqlite3_result_text(context, z, -1, fossil_free); +} + +/* +** Register the various SQL functions (defined above) needed to implement +** full-scan search. +*/ +void search_sql_setup(sqlite3 *db){ + static int once = 0; + static const int enc = SQLITE_UTF8|SQLITE_INNOCUOUS; + if( once++ ) return; + sqlite3_create_function(db, "search_match", -1, enc, 0, + search_match_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_score", 0, enc, 0, + search_score_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_snippet", 0, enc, 0, + search_snippet_sqlfunc, 0, 0); + sqlite3_create_function(db, "search_init", -1, enc, 0, + search_init_sqlfunc, 0, 0); + sqlite3_create_function(db, "stext", 3, enc, 0, + search_stext_sqlfunc, 0, 0); + sqlite3_create_function(db, "title", 3, enc, 0, + search_title_sqlfunc, 0, 0); + sqlite3_create_function(db, "body", 3, enc, 0, + search_body_sqlfunc, 0, 0); + sqlite3_create_function(db, "urlencode", 1, enc, 0, + search_urlencode_sqlfunc, 0, 0); +} + +/* +** Testing the search function. +** +** COMMAND: search* +** +** Usage: %fossil search [OPTIONS] PATTERN... +** +** Search the repository for PATTERN and show matches. Depending on +** options and how the administrator has search configured for the +** repository, the search can cover: +** +** * check-in comments (-c) +** * embedded documentation (--docs) +** * forum posts (--forum) +** * tickets (--tickets) +** * tech notes (--technotes) +** * wiki pages (--wiki) +** * built-in fossil help text (-h) +** * all of the above (-a) +** +** Use options below to select the scope of the search. The +** default is check-in comments only (-c). +** +** Output is colorized if writing to a TTY and if the NO_COLOR environment +** variable is not set. Use the "--highlight 0" option to disable colorization +** or use "--highlight 91" to force it on. Change the argument to --highlight +** to change the color. +** +** Options: +** -a|--all Search everything +** -c|--checkins Search check-in comments +** --docs Search embedded documentation +** --forum Search forum posts +** -h|--bi-help Search built-in help +** --highlight N Used VT100 color N for matching text. 0 means "off". +** -n|--limit N Limit output to N matches +** --technotes Search tech notes +** --tickets Search tickets +** -W|--width WIDTH Set display width to WIDTH columns, 0 for +** unlimited. Defaults to the terminal's width. +** --wiki Search wiki +*/ +void search_cmd(void){ + Blob pattern; + int i; + Blob sql = empty_blob; + Stmt q; + int iBest; + int srchFlags = 0; + int bFts = 1; /* Use FTS search by default now */ + char fAll = NULL != find_option("all", "a", 0); + const char *zLimit = find_option("limit","n",1); + const char *zScope = 0; + const char *zWidth = find_option("width","W",1); + int bDebug = find_option("debug",0,0)!=0; /* Undocumented */ + int nLimit = zLimit ? atoi(zLimit) : -1000; + int width; + int nTty = 0; /* VT100 highlight color for matching text */ + const char *zHighlight = 0; + int bFlags = 0; /* DB open flags */ + + nTty = terminal_is_vt100(); + + /* Undocumented option to change highlight color */ + zHighlight = find_option("highlight",0,1); + if( zHighlight ) nTty = atoi(zHighlight); + + /* Undocumented option (legacy) */ + zScope = find_option("scope",0,1); + + if( find_option("fts",0,0)!=0 ) bFts = 1; /* Undocumented legacy */ + if( find_option("legacy",0,0)!=0 ) bFts = 0; /* Undocumented */ + + if( zWidth ){ + width = atoi(zWidth); + if( (width!=0) && (width<=20) ){ + fossil_fatal("-W|--width value must be >20 or 0"); + } + }else{ + width = -1; + } + if( zScope ){ + for(i=0; zScope[i]; i++){ + switch( zScope[i] ){ + case 'a': srchFlags = SRCH_ALL; break; + case 'c': srchFlags |= SRCH_CKIN; break; + case 'd': srchFlags |= SRCH_DOC; break; + case 'e': srchFlags |= SRCH_TECHNOTE; break; + case 'f': srchFlags |= SRCH_FORUM; break; + case 'h': srchFlags |= SRCH_HELP; break; + case 't': srchFlags |= SRCH_TKT; break; + case 'w': srchFlags |= SRCH_WIKI; break; + } + } + bFts = 1; + } + if( find_option("all","a",0) ){ srchFlags |= SRCH_ALL; bFts = 1; } + if( find_option("bi-help","h",0) ){ srchFlags |= SRCH_HELP; bFts = 1; } + if( find_option("checkins","c",0) ){ srchFlags |= SRCH_CKIN; bFts = 1; } + if( find_option("docs",0,0) ){ srchFlags |= SRCH_DOC; bFts = 1; } + if( find_option("forum",0,0) ){ srchFlags |= SRCH_FORUM; bFts = 1; } + if( find_option("technotes",0,0) ){ srchFlags |= SRCH_TECHNOTE; bFts = 1; } + if( find_option("tickets",0,0) ){ srchFlags |= SRCH_TKT; bFts = 1; } + if( find_option("wiki",0,0) ){ srchFlags |= SRCH_WIKI; bFts = 1; } + + /* If no search objects are specified, default to "check-in comments" */ + if( srchFlags==0 ) srchFlags = SRCH_CKIN; + + if( srchFlags==SRCH_HELP ) bFlags = OPEN_OK_NOT_FOUND|OPEN_SUBSTITUTE; + db_find_and_open_repository(bFlags, 0); + verify_all_options(); + if( g.argc<3 ) return; + login_set_capabilities("s", 0); + if( search_restrict(srchFlags)==0 && (srchFlags & SRCH_HELP)==0 ){ + const char *zC1 = 0, *zPlural = "s"; + if( srchFlags & SRCH_TECHNOTE ){ zC1 = "technote"; } + if( srchFlags & SRCH_TKT ){ zC1 = "ticket"; } + if( srchFlags & SRCH_FORUM ){ zC1 = "forum"; zPlural = ""; } + if( srchFlags & SRCH_DOC ){ zC1 = "document"; } + if( srchFlags & SRCH_WIKI ){ zC1 = "wiki"; zPlural = ""; } + if( srchFlags & SRCH_CKIN ){ zC1 = "check-in"; } + fossil_print( + "Search of %s%s is disabled on this repository.\n" + "Enable using \"fossil fts-config enable %s\".\n", + zC1, zPlural, zC1 + ); + return; + } + + blob_init(&pattern, g.argv[2], -1); + for(i=3; i=1 ){ + nLimit--; + if( nLimit==0 ) break; + } + } + db_finalize(&q); + blob_reset(&pattern); + }else{ + /* Legacy timeline search (the default) */ + (void)search_init(blob_str(&pattern),"*","*","...",SRCHFLG_STATIC); + blob_reset(&pattern); + search_sql_setup(g.db); + + db_multi_exec( + "CREATE TEMP TABLE srch(rid,uuid,date,comment,x);" + "CREATE INDEX srch_idx1 ON srch(x);" + "INSERT INTO srch(rid,uuid,date,comment,x)" + " SELECT blob.rid, uuid, datetime(event.mtime,toLocal())," + " coalesce(ecomment,comment)," + " search_score()" + " FROM event, blob" + " WHERE blob.rid=event.objid" + " AND search_match(coalesce(ecomment,comment));" + ); + iBest = db_int(0, "SELECT max(x) FROM srch"); + blob_append(&sql, + "SELECT rid, uuid, date, comment, 0, 0 FROM srch " + "WHERE 1 ", -1); + if(!fAll){ + blob_append_sql(&sql,"AND x>%d ", iBest/3); + } + blob_append(&sql, "ORDER BY x DESC, date DESC ", -1); + db_prepare(&q, "%s", blob_sql_text(&sql)); + blob_reset(&sql); + print_timeline(&q, nLimit, width, 0, 0); + db_finalize(&q); + } +} + +#if INTERFACE +/* What to search for */ +#define SRCH_CKIN 0x0001 /* Search over check-in comments */ +#define SRCH_DOC 0x0002 /* Search over embedded documents */ +#define SRCH_TKT 0x0004 /* Search over tickets */ +#define SRCH_WIKI 0x0008 /* Search over wiki */ +#define SRCH_TECHNOTE 0x0010 /* Search over tech notes */ +#define SRCH_FORUM 0x0020 /* Search over forum messages */ +#define SRCH_HELP 0x0040 /* Search built-in help (full-scan only) */ +#define SRCH_ALL 0x007f /* Search over everything */ +#endif + +/* +** Remove bits from srchFlags which are disallowed by either the +** current server configuration or by user permissions. Return +** the revised search flags mask. +** +** If bFlex is true, that means allow through the SRCH_HELP option +** even if it is not explicitly enabled. +*/ +unsigned int search_restrict(unsigned int srchFlags){ + static unsigned int knownGood = 0; + static unsigned int knownBad = 0; + static const struct { unsigned m; const char *zKey; } aSetng[] = { + { SRCH_CKIN, "search-ci" }, + { SRCH_DOC, "search-doc" }, + { SRCH_TKT, "search-tkt" }, + { SRCH_WIKI, "search-wiki" }, + { SRCH_TECHNOTE, "search-technote" }, + { SRCH_FORUM, "search-forum" }, + { SRCH_HELP, "search-help" }, + }; + int i; + if( g.perm.Read==0 ) srchFlags &= ~(SRCH_CKIN|SRCH_DOC|SRCH_TECHNOTE); + if( g.perm.RdTkt==0 ) srchFlags &= ~(SRCH_TKT); + if( g.perm.RdWiki==0 ) srchFlags &= ~(SRCH_WIKI); + if( g.perm.RdForum==0) srchFlags &= ~(SRCH_FORUM); + for(i=0; i", "", " ... ", + SRCHFLG_STATIC|SRCHFLG_HTML); + if( (srchFlags & SRCH_DOC)!=0 ){ + char *zDocGlob = db_get("doc-glob",""); + const char *zMainBranch = db_main_branch(); + char *zDocBr = db_get("doc-branch", zMainBranch); + if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){ + Glob * pGlob = glob_create(zDocBr) + /* We're misusing a Glob as a list of comma-/space-delimited + ** tokens. We're not actually doing glob matches here. */; + int i; + db_multi_exec( + "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;" + ); + for( i = 0; i < pGlob->nPattern; ++i ){ + const char * zBranch = pGlob->azPattern[i]; + db_multi_exec( + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Document: %%s',title('d',blob.rid,foci.filename))," + " printf('/doc/%T/%%s',foci.filename)," + " search_score()," + " 'd'||blob.rid," + " (SELECT datetime(event.mtime) FROM event" + " WHERE objid=symbolic_name_to_rid(%Q))," + " search_snippet()" + " FROM foci CROSS JOIN blob" + " WHERE checkinID=symbolic_name_to_rid(%Q)" + " AND blob.uuid=foci.uuid" + " AND search_match(title('d',blob.rid,foci.filename)," + " body('d',blob.rid,foci.filename))" + " AND %z", + zBranch, zBranch, zBranch, glob_expr("foci.filename", zDocGlob) + ); + } + glob_free(pGlob); + } + fossil_free(zDocGlob); + fossil_free(zDocBr); + } + if( (srchFlags & SRCH_WIKI)!=0 ){ + db_multi_exec( + "WITH wiki(name,rid,mtime) AS (" + " SELECT substr(tagname,6), tagxref.rid, max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'wiki-*'" + " AND tagxref.tagid=tag.tagid" + " GROUP BY 1" + ")" + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Wiki: %%s',name)," + " printf('/wiki?name=%%s',urlencode(name))," + " search_score()," + " 'w'||rid," + " datetime(mtime)," + " search_snippet()" + " FROM wiki" + " WHERE search_match(title('w',rid,name),body('w',rid,name));" + ); + } + if( (srchFlags & SRCH_CKIN)!=0 ){ + db_multi_exec( + "WITH ckin(uuid,rid,mtime) AS (" + " SELECT blob.uuid, event.objid, event.mtime" + " FROM event, blob" + " WHERE event.type='ci'" + " AND blob.rid=event.objid" + ")" + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime))," + " printf('/timeline?c=%%s',uuid)," + " search_score()," + " 'c'||rid," + " datetime(mtime)," + " search_snippet()" + " FROM ckin" + " WHERE search_match('',body('c',rid,NULL));" + ); + } + if( (srchFlags & SRCH_TKT)!=0 ){ + db_multi_exec( + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL)," + "datetime(tkt_mtime))," + " printf('/tktview/%%.20s',tkt_uuid)," + " search_score()," + " 't'||tkt_id," + " datetime(tkt_mtime)," + " search_snippet()" + " FROM ticket" + " WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));" + ); + } + if( (srchFlags & SRCH_TECHNOTE)!=0 ){ + db_multi_exec( + "WITH technote(uuid,rid,mtime) AS (" + " SELECT substr(tagname,7), tagxref.rid, max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'event-*'" + " AND tagxref.tagid=tag.tagid" + " GROUP BY 1" + ")" + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT printf('Tech Note: %%s',uuid)," + " printf('/technote/%%s',uuid)," + " search_score()," + " 'e'||rid," + " datetime(mtime)," + " search_snippet()" + " FROM technote" + " WHERE search_match('',body('e',rid,NULL));" + ); + } + if( (srchFlags & SRCH_FORUM)!=0 ){ + db_multi_exec( + "INSERT INTO x(label,url,score,id,date,snip)" + " SELECT 'Forum '||comment," + " '/forumpost/'||uuid," + " search_score()," + " 'f'||rid," + " datetime(event.mtime)," + " search_snippet()" + " FROM event JOIN blob on event.objid=blob.rid" + " WHERE search_match('',body('f',rid,NULL));" + ); + } + if( (srchFlags & SRCH_HELP)!=0 ){ + const char *zPrefix; + helptext_vtab_register(g.db); + if( srchFlags==SRCH_HELP ){ + zPrefix = "The"; + }else{ + zPrefix = "Built-in help for the"; + } + db_multi_exec( + "INSERT INTO x(label,url,score,id,snip)" + " SELECT format('%q \"%%s\" %%s',name,type)," + " '/help/'||name," + " search_score()," + " 'h'||rowid," + " search_snippet()" + " FROM helptext" + " WHERE search_match(format('the \"%%s\" %%s',name,type)," + " helptext.helptext);", + zPrefix + ); + } +} + +/* +** Number of significant bits in a u32 +*/ +static int nbits(u32 x){ + int n = 0; + while( x ){ n++; x >>= 1; } + return n; +} + +/* +** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')). +*/ +static void search_rank_sqlfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned *aVal = (unsigned int*)sqlite3_value_blob(argv[0]); + int nVal = sqlite3_value_bytes(argv[0])/4; + int nCol; /* Number of columns in the index */ + int nTerm; /* Number of search terms in the query */ + int i, j; /* Loop counter */ + double r = 0.0; /* Score */ + const unsigned *aX, *aS; + + if( nVal<2 ) return; + nTerm = aVal[0]; + nCol = aVal[1]; + if( nVal<2+3*nCol*nTerm+nCol ) return; + aS = aVal+2; + aX = aS+nCol; + for(j=0; j0 ){ + x = 0.0; + for(i=0; i=0 && zPat[i]==' '; i--){} + if( i<0 ){ + fossil_free(zPat); + zPat = mprintf("\"\""); + } + return zPat; +} + +/* +** When this routine is called, there already exists a table +** +** x(label,url,score,id,snip). +** +** label: The "name" of the document containing the match +** url: A URL for the document +** score: How well the document matched +** id: The document id. Format: xNNNNN, x: type, N: number +** snip: A snippet for the match +** +** And the srchFlags parameter has been validated. This routine +** fills the X table with search results using FTS indexed search. +** +** The companion full-scan search routine is search_fullscan(). +*/ +LOCAL void search_indexed( + const char *zPattern, /* The query pattern */ + unsigned int srchFlags /* What to search over */ +){ + Blob sql; + char *zPat; + static const char *zSnippetCall; + if( srchFlags==0 ) return; + sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + search_rank_sqlfunc, 0, 0); + zPat = search_simplify_pattern(zPattern); + blob_init(&sql, 0, 0); + if( search_index_type(0)==4 ){ + /* If this repo is still using the legacy FTS5 search index, then + ** the snippet() function is slightly different */ + zSnippetCall = "snippet(ftsidx,'','',' ... ',-1,35)"; + }else{ + /* This is the common case - Using newer FTS5 search index */ + zSnippetCall = "snippet(ftsidx,-1,'','',' ... ',35)"; + } + blob_appendf(&sql, + "INSERT INTO x(label,url,score,id,date,snip) " + " SELECT ftsdocs.label," + " ftsdocs.url," + " rank(matchinfo(ftsidx,'pcsx'))," + " ftsdocs.type || ftsdocs.rid," + " datetime(ftsdocs.mtime)," + " %s" + " FROM ftsidx CROSS JOIN ftsdocs" + " WHERE ftsidx MATCH %Q" + " AND ftsdocs.rowid=ftsidx.rowid", + zSnippetCall /*safe-for-%s*/, zPat + ); + fossil_free(zPat); + if( srchFlags!=SRCH_ALL ){ + const char *zSep = " AND ("; + static const struct { unsigned m; char c; } aMask[] = { + { SRCH_CKIN, 'c' }, + { SRCH_DOC, 'd' }, + { SRCH_TKT, 't' }, + { SRCH_WIKI, 'w' }, + { SRCH_TECHNOTE, 'e' }, + { SRCH_FORUM, 'f' }, + { SRCH_HELP, 'h' }, + }; + int i; + for(i=0; iTEXT" where TEXT contains +** no white-space or punctuation, then return the length of the mark. +*/ +static int isSnippetMark(const char *z){ + int n; + if( strncmp(z,"",6)!=0 ) return 0; + n = 6; + while( fossil_isalnum(z[n]) ) n++; + if( strncmp(&z[n],"",7)!=0 ) return 0; + return n+7; +} + +/* +** Return a copy of zSnip (in memory obtained from fossil_malloc()) that +** has all "<" characters, other than those on and , +** converted into "<". This is similar to htmlize() except that +** and are preserved. +*/ +static char *cleanSnippet(const char *zSnip){ + int i; + int n = 0; + char *z; + if( zSnip==0 ) zSnip = ""; + for(i=0; zSnip[i]; i++) if( zSnip[i]=='<' ) n++; + z = fossil_malloc( i+n*4+1 ); + i = 0; + while( zSnip[0] ){ + if( zSnip[0]=='<' ){ + n = isSnippetMark(zSnip); + if( n ){ + memcpy(&z[i], zSnip, n); + zSnip += n; + i += n; + continue; + }else{ + memcpy(&z[i], "<", 4); + i += 4; + zSnip++; + } + }else{ + z[i++] = zSnip[0]; + zSnip++; + } + } + z[i] = 0; + return z; +} + + +/* +** This routine generates web-page output for a search operation. +** Other web-pages can invoke this routine to add search results +** in the middle of the page. +** +** This routine works for both full-scan and indexed search. The +** appropriate low-level search routine is called according to the +** current configuration. +** +** Return the number of rows. +*/ +int search_run_and_output( + const char *zPattern, /* The query pattern */ + unsigned int srchFlags, /* What to search over */ + int fDebug /* Extra debugging output */ +){ + Stmt q; + int nRow = 0; + int nLimit = db_get_int("search-limit", 100); + + if( P("searchlimit")!=0 ){ + nLimit = atoi(P("searchlimit")); + } + srchFlags = search_restrict(srchFlags) | (srchFlags & SRCH_HELP); + if( srchFlags==0 ) return 0; + search_sql_setup(g.db); + add_content_sql_commands(g.db); + db_multi_exec( + "CREATE TEMP TABLE x(label,url,score,id,date,snip);" + ); + if( !search_index_exists() ){ + search_fullscan(zPattern, srchFlags); /* Full-scan search */ + }else{ + search_update_index(srchFlags); /* Update the index, if necessary */ + search_indexed(zPattern, srchFlags); /* Indexed search */ + if( srchFlags & SRCH_HELP ){ + search_fullscan(zPattern, SRCH_HELP); + } + } + db_prepare(&q, "SELECT url, snip, label, score, id, substr(date,1,10)" + " FROM x" + " ORDER BY score DESC, date DESC;"); + while( db_step(&q)==SQLITE_ROW ){ + const char *zUrl = db_column_text(&q, 0); + const char *zSnippet = db_column_text(&q, 1); + const char *zLabel = db_column_text(&q, 2); + const char *zDate = db_column_text(&q, 5); + if( nRow==0 ){ + @
    + } + nRow++; + @

  1. %h(zLabel) + if( fDebug ){ + @ (%e(db_column_double(&q,3)), %s(db_column_text(&q,4)) + } + @
    %z(cleanSnippet(zSnippet)) \ + if( zLabel && zDate && zDate[0] && strstr(zLabel,zDate)==0 ){ + @ (%h(zDate)) + } + @

    2. + if( nLimit && nRow>=nLimit ) break; + } + db_finalize(&q); + if( nRow ){ + @
+ } + return nRow; +} + +/* +** Generate some HTML for doing search. At a minimum include the +** Search-Text entry form. If the "s" query parameter is present, also +** show search results. +** +** The srchFlags parameter restricts the set of documents to be searched. +** srchFlags should normally be either a single search category or all +** categories. Any srchFlags with two or more bits set +** is treated like SRCH_ALL for display purposes. +** +** This routine automatically restricts srchFlag according to user +** permissions and the server configuration. The entry box is shown +** disabled if srchFlags is 0 after these restrictions are applied. +** +** The mFlags value controls options: +** +** 0x01 If the y= query parameter is present, use it as an addition +** restriction what to search. +** +** 0x02 Show nothing if search is disabled. +** +** Return true if there are search results. +*/ +int search_screen(unsigned srchAllowed, int mFlags){ + const char *zType = 0; + const char *zClass = 0; + const char *zDisable1; + const char *zDisable2; + const char *zPattern; + int fDebug = PB("debug"); + int haveResult = 0; + int srchThisTime; + const char *zY = PD("y","all"); + if( zY[0]=='h' && zY[1]==0 ){ + srchAllowed = search_restrict(srchAllowed) | (srchAllowed & SRCH_HELP); + }else{ + srchAllowed = search_restrict(srchAllowed); + } + switch( srchAllowed ){ + case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break; + case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break; + case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break; + case SRCH_WIKI: zType = " Wiki"; zClass = "Wiki"; break; + case SRCH_TECHNOTE: zType = " Tech Notes"; zClass = "Note"; break; + case SRCH_FORUM: zType = " Forum"; zClass = "Frm"; break; + case SRCH_HELP: zType = " Help"; zClass = "Hlp"; break; + } + if( srchAllowed==0 ){ + if( mFlags & 0x02 ) return 0; + zDisable1 = " disabled"; + zDisable2 = " disabled"; + zPattern = ""; + }else{ + zDisable1 = ""; /* Was: " autofocus" */ + zDisable2 = ""; + zPattern = PD("s",""); + } + @
+ if( zClass ){ + @
+ }else{ + @
+ } + @ + srchThisTime = srchAllowed; + if( (mFlags & 0x01)!=0 && (srchAllowed & (srchAllowed-1))!=0 ){ + static const struct { + const char *z; + const char *zNm; + unsigned m; + } aY[] = { + { "all", "All", SRCH_ALL }, + { "c", "Check-ins", SRCH_CKIN }, + { "d", "Docs", SRCH_DOC }, + { "t", "Tickets", SRCH_TKT }, + { "w", "Wiki", SRCH_WIKI }, + { "e", "Tech Notes", SRCH_TECHNOTE }, + { "f", "Forum", SRCH_FORUM }, + { "h", "Help", SRCH_HELP }, + }; + int i; + @ + } + if( fDebug ){ + @ + } + @ + if( srchAllowed==0 && srchThisTime==0 ){ + @

Search is disabled

+ } + @
+ while( fossil_isspace(zPattern[0]) ) zPattern++; + if( zPattern[0] ){ + if( zClass ){ + @
+ }else{ + @
+ } + if( search_run_and_output(zPattern, srchThisTime, fDebug)==0 ){ + @

No matches for: %h(zPattern)

+ } + @
+ haveResult = 1; + } + return haveResult; +} + +/* +** WEBPAGE: search +** +** Search for check-in comments, documents, tickets, or wiki that +** match a user-supplied pattern. +** +** s=PATTERN Specify the full-text pattern to search for +** y=TYPE What to search. +** c -> check-ins, +** d -> documentation, +** t -> tickets, +** w -> wiki, +** e -> tech notes, +** f -> forum, +** h -> built-in help, +** all -> everything. +*/ +void search_page(void){ + const int isSearch = P("s")!=0; + login_check_credentials(); + style_header("Search%s", isSearch ? " Results" : ""); + cgi_check_for_malice(); + search_screen(SRCH_ALL, 1); + style_finish_page(); +} + + +/* +** This is a helper function for search_stext(). Writing into pOut +** the search text obtained from pIn according to zMimetype. +** +** If a title is not specified in zTitle (e.g. for wiki pages that do not +** include the title in the body), it is determined from the page content. +** +** The title of the document is the first line of text. All subsequent +** lines are the body. If the document has no title, the first line +** is blank. +*/ +static void get_stext_by_mimetype( + Blob *pIn, + const char *zMimetype, + const char *zTitle, + Blob *pOut +){ + Blob html, title; + Blob *pHtml = &html; + blob_init(&html, 0, 0); + if( zTitle==0 ){ + blob_init(&title, 0, 0); + }else{ + blob_init(&title, zTitle, -1); + } + if( zMimetype==0 ) zMimetype = "text/plain"; + if( fossil_strcmp(zMimetype,"text/x-fossil-wiki")==0 ){ + if( blob_size(&title) ){ + wiki_convert(pIn, &html, 0); + }else{ + Blob tail; + blob_init(&tail, 0, 0); + if( wiki_find_title(pIn, &title, &tail) ){ + blob_appendf(pOut, "%s\n", blob_str(&title)); + wiki_convert(&tail, &html, 0); + blob_reset(&tail); + }else{ + blob_append(pOut, "\n", 1); + wiki_convert(pIn, &html, 0); + } + } + html_to_plaintext(blob_str(&html), pOut, 0); + }else if( fossil_strcmp(zMimetype,"text/x-markdown")==0 ){ + markdown_to_html(pIn, blob_size(&title) ? NULL : &title, &html); + }else if( fossil_strcmp(zMimetype,"text/x-org")==0 ){ + if( blob_size(&title) ){ + org_to_html_wiki(pIn, &html); + }else{ + org_to_html(pIn, &title, &html); + } + }else if( fossil_strcmp(zMimetype,"text/html")==0 ){ + if( blob_size(&title)==0 ) doc_is_embedded_html(pIn, &title); + pHtml = pIn; + } + blob_appendf(pOut, "%s\n", blob_str(&title)); + if( blob_size(pHtml) ){ + html_to_plaintext(blob_str(pHtml), pOut, 0); + }else{ + blob_append(pOut, blob_buffer(pIn), blob_size(pIn)); + } + blob_reset(&html); + blob_reset(&title); +} + +/* +** Query pQuery is pointing at a single row of output. Append a text +** representation of every text-compatible column to pAccum. +*/ +static void append_all_ticket_fields(Blob *pAccum, Stmt *pQuery, int iTitle){ + int n = db_column_count(pQuery); + int i; + const char *zMime = 0; + if( iTitle>=0 && iTitlezThreadTitle ){ + blob_appendf(&wiki, "

%h

\n", pWiki->zThreadTitle); + } + blob_appendf(&wiki, "From %s:\n\n%s", pWiki->zUser, pWiki->zWiki); + }else{ + blob_init(&wiki, pWiki->zWiki, -1); + } + get_stext_by_mimetype(&wiki, wiki_filter_mimetypes(pWiki->zMimetype), + cType=='w' ? pWiki->zWikiTitle : NULL, pOut); + blob_reset(&wiki); + manifest_destroy(pWiki); + break; + } + case 'c': { /* Check-in Comments */ + static Stmt q; + static int isPlainText = -1; + db_static_prepare(&q, + "SELECT coalesce(ecomment,comment)" + " ||' (user: '||coalesce(euser,user,'?')" + " ||', tags: '||" + " (SELECT group_concat(substr(tag.tagname,5),',')" + " FROM tag, tagxref" + " WHERE tagname GLOB 'sym-*' AND tag.tagid=tagxref.tagid" + " AND tagxref.rid=event.objid AND tagxref.tagtype>0)" + " ||')'" + " FROM event WHERE objid=:x AND type='ci'"); + if( isPlainText<0 ){ + isPlainText = db_get_boolean("timeline-plaintext",0); + } + db_bind_int(&q, ":x", rid); + if( db_step(&q)==SQLITE_ROW ){ + blob_append(pOut, "\n", 1); + if( isPlainText ){ + db_column_blob(&q, 0, pOut); + }else{ + Blob x; + blob_init(&x,0,0); + db_column_blob(&q, 0, &x); + get_stext_by_mimetype(&x, "text/x-fossil-wiki", NULL, pOut); + blob_reset(&x); + } + } + db_reset(&q); + break; + } + case 't': { /* Tickets */ + static Stmt q1; + static int iTitle = -1; + db_static_prepare(&q1, "SELECT * FROM ticket WHERE tkt_id=:rid"); + db_bind_int(&q1, ":rid", rid); + if( db_step(&q1)==SQLITE_ROW ){ + if( iTitle<0 ){ + int n = db_column_count(&q1); + for(iTitle=0; iTitle0 ){ + blob_reset(&cache.stext); + }else{ + blob_init(&cache.stext,0,0); + } + cache.cType = cType; + cache.rid = rid; + if( cType==0 ) return 0; + search_stext(cType, rid, zName, &cache.stext); + z = blob_str(&cache.stext); + for(i=0; z[i] && z[i]!='\n'; i++){} + cache.nTitle = i; + } + if( pnTitle ) *pnTitle = cache.nTitle; + return blob_str(&cache.stext); +} + +/* +** COMMAND: test-search-stext +** +** Usage: fossil test-search-stext TYPE RID NAME +** +** Compute the search text for document TYPE-RID whose name is NAME. +** The TYPE is one of "c", "d", "t", "w", or "e". The RID is the document +** ID. The NAME is used to figure out a mimetype to use for formatting +** the raw document text. +*/ +void test_search_stext(void){ + Blob out; + db_find_and_open_repository(0,0); + if( g.argc!=5 ) usage("TYPE RID NAME"); + search_stext(g.argv[2][0], atoi(g.argv[3]), g.argv[4], &out); + fossil_print("%s\n",blob_str(&out)); + blob_reset(&out); +} + +/* +** COMMAND: test-convert-stext +** +** Usage: fossil test-convert-stext FILE MIMETYPE +** +** Read the content of FILE and convert it to stext according to MIMETYPE. +** Send the result to standard output. +*/ +void test_convert_stext(void){ + Blob in, out; + db_find_and_open_repository(0,0); + if( g.argc!=4 ) usage("FILENAME MIMETYPE"); + blob_read_from_file(&in, g.argv[2], ExtFILE); + blob_init(&out, 0, 0); + get_stext_by_mimetype(&in, g.argv[3], NULL, &out); + fossil_print("%s\n",blob_str(&out)); + blob_reset(&in); + blob_reset(&out); +} + +/* +** The schema for the full-text index. The %s part must be an empty +** string or a comma followed by additional flags for the FTS virtual +** table. +*/ +static const char zFtsSchema[] = +@ -- One entry for each possible search result +@ CREATE TABLE IF NOT EXISTS repository.ftsdocs( +@ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid +@ type CHAR(1), -- Type of document +@ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document +@ name TEXT, -- Additional document description +@ idxed BOOLEAN, -- True if currently in the index +@ label TEXT, -- Label to print on search results +@ url TEXT, -- URL to access this document +@ mtime DATE, -- Date when document created +@ bx TEXT, -- Temporary "body" content cache +@ UNIQUE(type,rid) +@ ); +@ CREATE INDEX repository.ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0; +@ CREATE INDEX repository.ftsdocName ON ftsdocs(name) WHERE type='w'; +@ CREATE VIEW IF NOT EXISTS repository.ftscontent AS +@ SELECT rowid, type, rid, name, idxed, label, url, mtime, +@ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body' +@ FROM ftsdocs; +@ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx +@ USING fts5(content="ftscontent", title, body%s); +; +static const char zFtsDrop[] = +@ DROP TABLE IF EXISTS repository.ftsidx; +@ DROP VIEW IF EXISTS repository.ftscontent; +@ DROP TABLE IF EXISTS repository.ftsdocs; +@ DROP TABLE IF EXISTS repository.chatfts1; +; + +#if INTERFACE +/* +** Values for the search-tokenizer config option. +*/ +#define FTS5TOK_NONE 0 /* disabled */ +#define FTS5TOK_PORTER 1 /* porter stemmer */ +#define FTS5TOK_UNICODE61 2 /* unicode61 tokenizer */ +#define FTS5TOK_TRIGRAM 3 /* trigram tokenizer */ +#endif + +/* +** Cached FTS5TOK_xyz value for search_tokenizer_type() and +** friends. +*/ +static int iFtsTokenizer = -1; + +/* +** Returns one of the FTS5TOK_xyz values, depending on the value of +** the search-tokenizer config entry, defaulting to FTS5TOK_NONE. The +** result of the first call is cached for subsequent calls unless +** bRecheck is true. +*/ +int search_tokenizer_type(int bRecheck){ + char *z; + if( iFtsTokenizer>=0 && bRecheck==0 ){ + return iFtsTokenizer; + } + z = db_get("search-tokenizer",0); + if( 0==z ){ + iFtsTokenizer = FTS5TOK_NONE; + }else if(0==fossil_strcmp(z,"porter")){ + iFtsTokenizer = FTS5TOK_PORTER; + }else if(0==fossil_strcmp(z,"unicode61")){ + iFtsTokenizer = FTS5TOK_UNICODE61; + }else if(0==fossil_strcmp(z,"trigram")){ + iFtsTokenizer = FTS5TOK_TRIGRAM; + }else{ + iFtsTokenizer = is_truth(z) ? FTS5TOK_PORTER : FTS5TOK_NONE; + } + fossil_free(z); + return iFtsTokenizer; +} + +/* +** Returns a string in the form ",tokenize=X", where X is the string +** counterpart of the given FTS5TOK_xyz value. Returns "" if tokType +** does not correspond to a known FTS5 tokenizer. +*/ +const char * search_tokenize_arg_for_type(int tokType){ + switch( tokType ){ + case FTS5TOK_PORTER: return ",tokenize=porter"; + case FTS5TOK_UNICODE61: return ",tokenize=unicode61"; + case FTS5TOK_TRIGRAM: return ",tokenize=trigram"; + case FTS5TOK_NONE: + default: return ""; + } +} + +/* +** Returns a string value suitable for use as the search-tokenizer +** setting's value, depending on the value of z. If z is 0 then the +** current search-tokenizer value is used as the basis for formulating +** the result (which may differ from the current value but will have +** the same meaning). Any unknown/unsupported value is interpreted as +** "off". +*/ +const char *search_tokenizer_for_string(const char *z){ + char * zTmp = 0; + const char *zRc = 0; + + if( 0==z ){ + z = zTmp = db_get("search-tokenizer",0); + } + if( 0==z ){ + zRc = "off"; + }else if( 0==fossil_strcmp(z,"porter") ){ + zRc = "porter"; + }else if( 0==fossil_strcmp(z,"unicode61") ){ + zRc = "unicode61"; + }else if( 0==fossil_strcmp(z,"trigram") ){ + zRc = "trigram"; + }else{ + zRc = is_truth(z) ? "porter" : "off"; + } + fossil_free(zTmp); + return zRc; +} + +/* +** Sets the search-tokenizer config setting to the value of +** search_tokenizer_for_string(zName). +*/ +void search_set_tokenizer(const char *zName){ + db_set("search-tokenizer", search_tokenizer_for_string( zName ), 0); + iFtsTokenizer = -1; +} + +/* +** Create or drop the tables associated with a full-text index. +*/ +static int searchIdxExists = -1; +void search_create_index(void){ + const char *zExtra = + search_tokenize_arg_for_type(search_tokenizer_type(0)); + assert( zExtra ); + search_sql_setup(g.db); + db_multi_exec(zFtsSchema/*works-like:"%s"*/, zExtra/*safe-for-%s*/); + searchIdxExists = 1; +} +void search_drop_index(void){ + db_multi_exec(zFtsDrop/*works-like:""*/); + searchIdxExists = 0; +} + +/* +** Return true if the full-text search index exists. See also the +** search_index_type() function. +*/ +int search_index_exists(void){ + if( searchIdxExists<0 ){ + searchIdxExists = db_table_exists("repository","ftsdocs"); + } + return searchIdxExists; +} + +/* +** Determine which full-text search index is currently being used to +** add searching. Return values: +** +** 0 No search index is available +** 4 FTS3/4 +** 5 FTS5 +** +** Results are cached. Make the argument 1 to reset the cache. See +** also the search_index_exists() routine. +*/ +int search_index_type(int bReset){ + static int idxType = -1; + if( idxType<0 || bReset ){ + idxType = db_int(0, + "SELECT CASE WHEN sql GLOB '*fts4*' THEN 4 ELSE 5 END" + " FROM repository.sqlite_schema WHERE name='ftsidx'" + ); + } + return idxType; +} + +/* +** Fill the FTSDOCS table with unindexed entries for everything +** in the repository. This uses INSERT OR IGNORE so entries already +** in FTSDOCS are unchanged. +*/ +void search_fill_index(void){ + if( !search_index_exists() ) return; + search_sql_setup(g.db); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,idxed)" + " SELECT 'c', objid, 0 FROM event WHERE type='ci';" + ); + db_multi_exec( + "WITH latest_wiki(rid,name,mtime) AS (" + " SELECT tagxref.rid, substr(tag.tagname,6), max(tagxref.mtime)" + " FROM tag, tagxref" + " WHERE tag.tagname GLOB 'wiki-*'" + " AND tagxref.tagid=tag.tagid" + " AND tagxref.value>0" + " GROUP BY 2" + ") INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed)" + " SELECT 'w', rid, name, 0 FROM latest_wiki;" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,idxed)" + " SELECT 't', tkt_id, 0 FROM ticket;" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed)" + " SELECT type, objid, comment, 0 FROM event WHERE type IN ('e','f');" + ); +} + +/* +** The document described by cType,rid,zName is about to be added or +** updated. If the document has already been indexed, then unindex it +** now while we still have access to the old content. Add the document +** to the queue of documents that need to be indexed or reindexed. +*/ +void search_doc_touch(char cType, int rid, const char *zName){ + if( search_index_exists() && !content_is_private(rid) ){ + char zType[2]; + zType[0] = cType; + zType[1] = 0; + search_sql_setup(g.db); + db_multi_exec( + "DELETE FROM ftsidx WHERE rowid IN" + " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)", + zType, rid + ); + db_multi_exec( + "REPLACE INTO ftsdocs(type,rid,name,idxed)" + " VALUES(%Q,%d,%Q,0)", + zType, rid, zName + ); + if( cType=='w' || cType=='e' ){ + db_multi_exec( + "DELETE FROM ftsidx WHERE rowid IN" + " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)", + cType, zName + ); + db_multi_exec( + "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d", + cType, zName, rid + ); + } + /* All forum posts are always indexed */ + } +} + +/* +** If the doc-glob and doc-br settings are valid for document search +** and if the latest check-in on doc-br is in the unindexed set of +** check-ins, then update all 'd' entries in FTSDOCS that have +** changed. +*/ +static void search_update_doc_index(void){ + const char *zMainBranch = db_main_branch(); + const char *zDocBranches = db_get("doc-branch", zMainBranch); + int i; + Glob * pGlob = glob_create(zDocBranches) + /* We're misusing a Glob as a list of comma-/space-delimited + ** tokens. We're not actually doing glob matches here. */; + if( !pGlob ) return; + db_multi_exec( + "CREATE TEMP TABLE current_docs(rid INTEGER PRIMARY KEY, name);" + "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;" + ); + for( i = 0; i < pGlob->nPattern; ++i ){ + const char *zDocBr = pGlob->azPattern[i]; + int ckid = symbolic_name_to_rid(zDocBr,"ci"); + double rTime; + if( !db_exists("SELECT 1 FROM ftsdocs WHERE type='c' AND rid=%d" + " AND NOT idxed", ckid) ) continue; + /* If we get this far, it means that changes to 'd' entries are + ** required. */ + rTime = db_double(0.0, "SELECT mtime FROM event WHERE objid=%d", ckid); + db_multi_exec( + "INSERT OR IGNORE INTO current_docs(rid, name)" + " SELECT blob.rid, foci.filename FROM foci, blob" + " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid" + " AND %z", + ckid, glob_expr("foci.filename", db_get("doc-glob","")) + ); + db_multi_exec( + "DELETE FROM ftsidx WHERE rowid IN" + " (SELECT rowid FROM ftsdocs WHERE type='d'" + " AND rid NOT IN (SELECT rid FROM current_docs))" + ); + db_multi_exec( + "DELETE FROM ftsdocs WHERE type='d'" + " AND rid NOT IN (SELECT rid FROM current_docs)" + ); + db_multi_exec( + "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,bx,url,mtime)" + " SELECT 'd', rid, name, 0," + " title('d',rid,name)," + " body('d',rid,name)," + " printf('/doc/%T/%%s',urlencode(name))," + " %.17g" + " FROM current_docs", + zDocBr, rTime + ); + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed" + ); + db_multi_exec( + "UPDATE ftsdocs SET" + " idxed=1," + " bx=NULL," + " label='Document: '||label" + " WHERE type='d' AND NOT idxed" + ); + } + glob_free(pGlob); +} + +/* +** Deal with all of the unindexed 'c' terms in FTSDOCS +*/ +static void search_update_checkin_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs" + " WHERE type='c' AND NOT idxed;" + ); + db_multi_exec( + "UPDATE ftsdocs SET idxed=1, name=NULL," + " (label,url,mtime) = " + " (SELECT printf('Check-in [%%.16s] on %%s',blob.uuid," + " datetime(event.mtime))," + " printf('/timeline?y=ci&c=%%.20s',blob.uuid)," + " event.mtime" + " FROM event, blob" + " WHERE event.objid=ftsdocs.rid" + " AND blob.rid=ftsdocs.rid)" + "WHERE ftsdocs.type='c' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 't' terms in FTSDOCS +*/ +static void search_update_ticket_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs" + " WHERE type='t' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1, name=NULL," + " (label,url,mtime) =" + " (SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,null)," + " datetime(tkt_mtime))," + " printf('/tktview/%%.20s',tkt_uuid)," + " tkt_mtime" + " FROM ticket" + " WHERE tkt_id=ftsdocs.rid)" + "WHERE ftsdocs.type='t' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 'w' terms in FTSDOCS +*/ +static void search_update_wiki_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs" + " WHERE type='w' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1," + " (name,label,url,mtime) = " + " (SELECT ftsdocs.name," + " 'Wiki: '||ftsdocs.name," + " '/wiki?name='||urlencode(ftsdocs.name)," + " tagxref.mtime" + " FROM tagxref WHERE tagxref.rid=ftsdocs.rid)" + " WHERE ftsdocs.type='w' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 'f' terms in FTSDOCS +*/ +static void search_update_forum_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs" + " WHERE type='f' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1, name=NULL," + " (label,url,mtime) = " + " (SELECT 'Forum '||event.comment," + " '/forumpost/'||blob.uuid," + " event.mtime" + " FROM event, blob" + " WHERE event.objid=ftsdocs.rid" + " AND blob.rid=ftsdocs.rid)" + "WHERE ftsdocs.type='f' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed 'e' terms in FTSDOCS +*/ +static void search_update_technote_index(void){ + db_multi_exec( + "INSERT INTO ftsidx(rowid,title,body)" + " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs" + " WHERE type='e' AND NOT idxed;" + ); + if( db_changes()==0 ) return; + db_multi_exec( + "UPDATE ftsdocs SET idxed=1," + " (name,label,url,mtime) = " + " (SELECT ftsdocs.name," + " 'Tech Note: '||ftsdocs.name," + " '/technote/'||substr(tag.tagname,7)," + " tagxref.mtime" + " FROM tagxref, tag USING (tagid)" + " WHERE tagxref.rid=ftsdocs.rid" + " AND tagname GLOB 'event-*')" + " WHERE ftsdocs.type='e' AND NOT ftsdocs.idxed" + ); +} + +/* +** Deal with all of the unindexed entries in the FTSDOCS table - that +** is to say, all the entries with FTSDOCS.IDXED=0. Add them to the +** index. +*/ +void search_update_index(unsigned int srchFlags){ + if( !search_index_exists() ) return; + if( !db_exists("SELECT 1 FROM ftsdocs WHERE NOT idxed") ) return; + search_sql_setup(g.db); + db_unprotect(PROTECT_READONLY); + if( srchFlags & (SRCH_CKIN|SRCH_DOC) ){ + search_update_doc_index(); + search_update_checkin_index(); + } + if( srchFlags & SRCH_TKT ){ + search_update_ticket_index(); + } + if( srchFlags & SRCH_WIKI ){ + search_update_wiki_index(); + } + if( srchFlags & SRCH_TECHNOTE ){ + search_update_technote_index(); + } + if( srchFlags & SRCH_FORUM ){ + search_update_forum_index(); + } + db_protect_pop(); +} + +/* +** Construct, prepopulate, and then update the full-text index. +*/ +void search_rebuild_index(void){ + fossil_print("rebuilding the search index..."); + fflush(stdout); + search_create_index(); + search_fill_index(); + search_update_index(search_restrict(SRCH_ALL)); + if( db_table_exists("repository","chat") ){ + chat_rebuild_index(1); + } + fossil_print(" done\n"); +} + +/* +** COMMAND: fts-config* abbrv-subcom +** +** Usage: fossil fts-config ?SUBCOMMAND? ?ARGUMENT? +** +** The "fossil fts-config" command configures the full-text search capabilities +** of the repository. Subcommands: +** +** reindex Rebuild the search index. This is a no-op if +** index search is disabled +** +** index (on|off) Turn the search index on or off +** +** enable TYPE .. Enable search for TYPE. TYPE is one of: +** check-in, document, ticket, wiki, technote, +** forum, help, or all +** +** disable TYPE ... Disable search for TYPE +** +** tokenizer VALUE Select a tokenizer for indexed search. VALUE +** may be one of (porter, on, off, trigram, unicode61), +** and "on" is equivalent to "porter". Unindexed +** search never uses tokenization or stemming. +** +** The current search settings are displayed after any changes are applied. +** Run this command with no arguments to simply see the settings. +*/ +void fts_config_cmd(void){ + static const struct { + int iCmd; + const char *z; + } aCmd[] = { + { 1, "reindex" }, + { 2, "index" }, + { 3, "disable" }, + { 4, "enable" }, + { 5, "tokenizer"}, + }; + static const struct { + const char *zSetting; + const char *zName; + const char *zSw; + } aSetng[] = { + { "search-ci", "check-in search:", "c" }, + { "search-doc", "document search:", "d" }, + { "search-tkt", "ticket search:", "t" }, + { "search-wiki", "wiki search:", "w" }, + { "search-technote", "technote search:", "e" }, + { "search-forum", "forum search:", "f" }, + { "search-help", "built-in help search:", "h" }, + }; + char *zSubCmd = 0; + int i, j, n; + int iCmd = 0; + int iAction = 0; + db_find_and_open_repository(0, 0); + if( g.argc>2 ){ + zSubCmd = g.argv[2]; + n = (int)strlen(zSubCmd); + for(i=0; i=count(aCmd) ){ + Blob all; + blob_init(&all,0,0); + for(i=0; i=4 ){ + /* If the argument to "enable" or "disable" is a string of at least + ** 4 characters which matches part of any aSetng.zName, then use that + ** one aSetng value only. */ + char *zGlob = mprintf("*%s*", zCtrl); + for(j=0; j=1 ){ + search_drop_index(); + } + if( iAction>=2 ){ + search_rebuild_index(); + } + + /* Always show the status before ending */ + for(i=0; iIndexed search is disabled + style_finish_page(); + return; + } + search_sql_setup(g.db); + style_submenu_element("Setup","%R/srchsetup"); + if( zId!=0 && (id = atoi(zId))>0 ){ + /* Show information about a single ftsdocs entry */ + style_header("Information about ftsdoc entry %d", id); + style_submenu_element("Summary","%R/test-ftsdocs"); + db_prepare(&q, + "SELECT type||rid, name, idxed, label, url, datetime(mtime)" + " FROM ftsdocs WHERE rowid=%d", id + ); + if( db_step(&q)==SQLITE_ROW ){ + const char *zUrl = db_column_text(&q,4); + const char *zDocId = db_column_text(&q,0); + char *zName; + char *z; + @ + @
rowid:  %d(id) + @
id:%s(zDocId) + @
name:%h(db_column_text(&q,1)) + @
idxed:%d(db_column_int(&q,2)) + @
label:%h(db_column_text(&q,3)) + @
url: + @ %h(zUrl) + @
mtime:%s(db_column_text(&q,5)) + z = db_text(0, "SELECT title FROM ftsidx WHERE rowid=%d",id); + if( z && z[0] ){ + @
title:%h(z) + fossil_free(z); + } + z = db_text(0, "SELECT body FROM ftsidx WHERE rowid=%d",id); + if( z && z[0] ){ + @
body:%h(z) + fossil_free(z); + } + @
+ zName = mprintf("Indexed '%c' docs",zDocId[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=1",zDocId[0]); + zName = mprintf("Unindexed '%c' docs",zDocId[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=0",zDocId[0]); + } + db_finalize(&q); + style_finish_page(); + return; + } + if( zType!=0 && zType[0]!=0 && zType[1]==0 && + zIdxed!=0 && (zIdxed[0]=='1' || zIdxed[0]=='0') && zIdxed[1]==0 + ){ + int ixed = zIdxed[0]=='1'; + char *zName; + style_header("List of '%c' documents that are%s indexed", + zType[0], ixed ? "" : " not"); + style_submenu_element("Summary","%R/test-ftsdocs"); + if( ixed==0 ){ + zName = mprintf("Indexed '%c' docs",zType[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=1",zType[0]); + }else{ + zName = mprintf("Unindexed '%c' docs",zType[0]); + style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=0",zType[0]); + } + db_prepare(&q, + "SELECT rowid, type||rid ||' '|| coalesce(label,'')" + " FROM ftsdocs WHERE type='%c' AND %s idxed", + zType[0], ixed ? "" : "NOT" + ); + @ + db_finalize(&q); + style_finish_page(); + return; + } + style_header("Summary of ftsdocs"); + db_prepare(&q, + "SELECT type, sum(idxed IS TRUE), sum(idxed IS FALSE), count(*)" + " FROM ftsdocs" + " GROUP BY 1 ORDER BY 4 DESC" + ); + @ + @ + @ + @ + while( db_step(&q)==SQLITE_ROW ){ + const char *zType = db_column_text(&q,0); + int nIndexed = db_column_int(&q, 1); + int nUnindexed = db_column_int(&q, 2); + int nTotal = db_column_int(&q, 3); + @ + cnt1 += nIndexed; + cnt2 += nUnindexed; + cnt3 += nTotal; + } + db_finalize(&q); + @ + @
TypeIndexedUnindexedTotal + @
%h(zType) + if( nIndexed>0 ){ + @ \ + @ %d(nIndexed) + }else{ + @ 0 + } + if( nUnindexed>0 ){ + @ \ + @ %d(nUnindexed) + }else{ + @ 0 + } + @ %d(nTotal) + @
Total%d(cnt1)%d(cnt2) + @ %d(cnt3) + @ + @
+ style_finish_page(); +} + + +/* +** The Fts5MatchinfoCtx bits were all taken verbatim from: +** +** https://sqlite.org/src/finfo?name=ext/fts5/fts5_test_mi.c +*/ + +typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx; + +#if INTERFACE +#ifndef SQLITE_AMALGAMATION +typedef unsigned int u32; +#endif +#endif + +struct Fts5MatchinfoCtx { + int nCol; /* Number of cols in FTS5 table */ + int nPhrase; /* Number of phrases in FTS5 query */ + char *zArg; /* nul-term'd copy of 2nd arg */ + int nRet; /* Number of elements in aRet[] */ + u32 *aRet; /* Array of 32-bit unsigned ints to return */ +}; + + +/* +** Return a pointer to the fts5_api pointer for database connection db. +** If an error occurs, return NULL and leave an error in the database +** handle (accessible using sqlite3_errcode()/errmsg()). +*/ +static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){ + sqlite3_stmt *pStmt = 0; + int rc; + + *ppApi = 0; + rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0); + (void)sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + } + + return rc; +} + + +/* +** Argument f should be a flag accepted by matchinfo() (a valid character +** in the string passed as the second argument). If it is not, -1 is +** returned. Otherwise, if f is a valid matchinfo flag, the value returned +** is the number of 32-bit integers added to the output array if the +** table has nCol columns and the query nPhrase phrases. +*/ +static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){ + int ret = -1; + switch( f ){ + case 'p': ret = 1; break; + case 'c': ret = 1; break; + case 'x': ret = 3 * nCol * nPhrase; break; + case 'y': ret = nCol * nPhrase; break; + case 'b': ret = ((nCol + 31) / 32) * nPhrase; break; + case 'n': ret = 1; break; + case 'a': ret = nCol; break; + case 'l': ret = nCol; break; + case 's': ret = nCol; break; + } + return ret; +} + +static int fts5MatchinfoIter( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + Fts5MatchinfoCtx *p, + int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*) +){ + int i; + int n = 0; + int rc = SQLITE_OK; + char f; + for(i=0; (f = p->zArg[i]); i++){ + rc = x(pApi, pFts, p, f, &p->aRet[n]); + if( rc!=SQLITE_OK ) break; + n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f); + } + return rc; +} + +static int fts5MatchinfoXCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pUserData +){ + Fts5PhraseIter iter; + int iCol, iOff; + u32 *aOut = (u32*)pUserData; + int iPrev = -1; + + for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); + iCol>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + aOut[iCol*3+1]++; + if( iCol!=iPrev ) aOut[iCol*3 + 2]++; + iPrev = iCol; + } + + return SQLITE_OK; +} + +static int fts5MatchinfoGlobalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int rc = SQLITE_OK; + switch( f ){ + case 'p': + aOut[0] = p->nPhrase; + break; + + case 'c': + aOut[0] = p->nCol; + break; + + case 'x': { + int i; + for(i=0; inPhrase && rc==SQLITE_OK; i++){ + void *pPtr = (void*)&aOut[i * p->nCol * 3]; + rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb); + } + break; + } + + case 'n': { + sqlite3_int64 nRow; + rc = pApi->xRowCount(pFts, &nRow); + aOut[0] = (u32)nRow; + break; + } + + case 'a': { + sqlite3_int64 nRow = 0; + rc = pApi->xRowCount(pFts, &nRow); + if( nRow==0 ){ + memset(aOut, 0, sizeof(u32) * p->nCol); + }else{ + int i; + for(i=0; rc==SQLITE_OK && inCol; i++){ + sqlite3_int64 nToken; + rc = pApi->xColumnTotalSize(pFts, i, &nToken); + if( rc==SQLITE_OK){ + aOut[i] = (u32)((2*nToken + nRow) / (2*nRow)); + } + } + } + break; + } + + } + return rc; +} + +static int fts5MatchinfoLocalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int i; + int rc = SQLITE_OK; + + switch( f ){ + case 'b': { + int iPhrase; + int nInt = ((p->nCol + 31) / 32) * p->nPhrase; + for(i=0; inPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iCol; + for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); + iCol>=0; + pApi->xPhraseNextColumn(pFts, &iter, &iCol) + ){ + aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); + } + } + + break; + } + + case 'x': + case 'y': { + int nMul = (f=='x' ? 3 : 1); + int iPhrase; + + for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; + + for(iPhrase=0; iPhrasenPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iOff, iCol; + for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); + iOff>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + aOut[nMul * (iCol + iPhrase * p->nCol)]++; + } + } + + break; + } + + case 'l': { + for(i=0; rc==SQLITE_OK && inCol; i++){ + int nToken; + rc = pApi->xColumnSize(pFts, i, &nToken); + aOut[i] = (u32)nToken; + } + break; + } + + case 's': { + int nInst; + + memset(aOut, 0, sizeof(u32) * p->nCol); + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &iPhrase, &iCol, &iOff); + iNextPhrase = iPhrase+1; + iNextOff = iOff+pApi->xPhraseSize(pFts, 0); + for(j=i+1; rc==SQLITE_OK && jxInst(pFts, j, &ip, &ic, &io); + if( ic!=iCol || io>iNextOff ) break; + if( ip==iNextPhrase && io==iNextOff ){ + nSeq++; + iNextPhrase = ip+1; + iNextOff = io + pApi->xPhraseSize(pFts, ip); + } + } + + if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq; + } + + break; + } + } + return rc; +} + +static Fts5MatchinfoCtx *fts5MatchinfoNew( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning error message */ + const char *zArg /* Matchinfo flag string */ +){ + Fts5MatchinfoCtx *p; + int nCol; + int nPhrase; + int i; + int nInt; + sqlite3_int64 nByte; + int rc; + + nCol = pApi->xColumnCount(pFts); + nPhrase = pApi->xPhraseCount(pFts); + + nInt = 0; + for(i=0; zArg[i]; i++){ + int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]); + if( n<0 ){ + char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return 0; + } + nInt += n; + } + + nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */ + + sizeof(u32) * nInt /* The p->aRet[] array */ + + (i+1); /* The p->zArg string */ + p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, nByte); + + p->nCol = nCol; + p->nPhrase = nPhrase; + p->aRet = (u32*)&p[1]; + p->nRet = nInt; + p->zArg = (char*)&p->aRet[nInt]; + memcpy(p->zArg, zArg, i); + + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(p); + p = 0; + } + + return p; +} + +static void fts5MatchinfoFunc( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + const char *zArg; + Fts5MatchinfoCtx *p; + int rc = SQLITE_OK; + + if( nVal>0 ){ + zArg = (const char*)sqlite3_value_text(apVal[0]); + }else{ + zArg = "pcx"; + } + + p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0); + if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){ + p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } + } + + if( rc==SQLITE_OK ){ + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + }else{ + /* No error has occurred, so return a copy of the array of integers. */ + int nByte = p->nRet * sizeof(u32); + sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT); + } +} + +int db_register_fts5(sqlite3 *db){ + int rc; /* Return code */ + fts5_api *pApi; /* FTS5 API functions */ + + /* Extract the FTS5 API pointer from the database handle. The + ** fts5_api_from_db() function above is copied verbatim from the + ** FTS5 documentation. Refer there for details. */ + rc = fts5_api_from_db(db, &pApi); + if( rc!=SQLITE_OK ) return rc; + + /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered + ** with this database handle, or an error (OOM perhaps?) has occurred. + ** + ** Also check that the fts5_api object is version 2 or newer. + */ + if( pApi==0 || pApi->iVersion<2 ){ + return SQLITE_ERROR; + } + + /* Register the implementation of matchinfo() */ + rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); + + return rc; +} -- cgit v1.3.1