summaryrefslogtreecommitdiff
path: root/src/xfer.c
diff options
context:
space:
mode:
authorcashmere <cashmere@autistici.org>2026-09-04 09:05:24 +0200
committercashmere <cashmere@autistici.org>2026-09-04 09:05:24 +0200
commit4f49b16079b27d7ce633619e4cddfdf2bffbc952 (patch)
treec296c1e7f3c3e2f6859c6d3a1b1b208779b89679 /src/xfer.c
downloadrelic-master.tar.gz
initial commitHEADmaster
Diffstat (limited to 'src/xfer.c')
-rw-r--r--src/xfer.c3083
1 files changed, 3083 insertions, 0 deletions
diff --git a/src/xfer.c b/src/xfer.c
new file mode 100644
index 0000000..190dfcf
--- /dev/null
+++ b/src/xfer.c
@@ -0,0 +1,3083 @@
+/*
+** Copyright (c) 2007 D. Richard Hipp
+**
+** This program is free software; you can redistribute it and/or
+** modify it under the terms of the Simplified BSD License (also
+** known as the "2-Clause License" or "FreeBSD License".)
+**
+** This program is distributed in the hope that it will be useful,
+** but without any warranty; without even the implied warranty of
+** merchantability or fitness for a particular purpose.
+**
+** Author contact information:
+** drh@hwaci.com
+** http://www.hwaci.com/drh/
+**
+*******************************************************************************
+**
+** This file contains code to implement the file transfer protocol.
+*/
+#include "config.h"
+#include "xfer.h"
+
+#include <time.h>
+
+/*
+** Maximum number of HTTP redirects that any http_exchange() call will
+** follow before throwing a fatal error. Most browsers use a limit of 20.
+*/
+#define MAX_REDIRECTS 20
+
+/*
+** This structure holds information about the current state of either
+** a client or a server that is participating in xfer.
+*/
+typedef struct Xfer Xfer;
+struct Xfer {
+ Blob *pIn; /* Input text from the other side */
+ Blob *pOut; /* Compose our reply here */
+ Blob line; /* The current line of input */
+ Blob aToken[6]; /* Tokenized version of line */
+ Blob err; /* Error message text */
+ int nToken; /* Number of tokens in line */
+ int nIGotSent; /* Number of "igot" cards sent */
+ int nPrivIGot; /* Number of private "igot" cards */
+ int nGimmeSent; /* Number of gimme cards sent */
+ int nFileSent; /* Number of files sent */
+ int nDeltaSent; /* Number of deltas sent */
+ int nFileRcvd; /* Number of files received */
+ int nDeltaRcvd; /* Number of deltas received */
+ int nDanglingFile; /* Number of dangling deltas received */
+ int mxSend; /* Stop sending "file" when pOut reaches this size */
+ int resync; /* Send igot cards for all holdings */
+ u8 syncPrivate; /* True to enable syncing private content */
+ u8 nextIsPrivate; /* If true, next "file" received is a private */
+ u32 remoteVersion; /* Version of fossil running on the other side */
+ u32 remoteDate; /* Date for specific client software edition */
+ u32 remoteTime; /* Time of date corresponding on remoteDate */
+ time_t maxTime; /* Time when this transfer should be finished */
+};
+
+
+/*
+** The input blob contains an artifact. Convert it into a record ID.
+** Create a phantom record if no prior record exists and
+** phantomize is true.
+**
+** Compare to uuid_to_rid(). This routine takes a blob argument
+** and does less error checking.
+*/
+static int rid_from_uuid(Blob *pUuid, int phantomize, int isPrivate){
+ static Stmt q;
+ int rid;
+
+ db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
+ db_bind_str(&q, ":uuid", pUuid);
+ if( db_step(&q)==SQLITE_ROW ){
+ rid = db_column_int(&q, 0);
+ }else{
+ rid = 0;
+ }
+ db_reset(&q);
+ if( rid==0 && phantomize ){
+ rid = content_new(blob_str(pUuid), isPrivate);
+ }
+ return rid;
+}
+
+/*
+** Remember that the other side of the connection already has a copy
+** of the file rid.
+*/
+static void remote_has(int rid){
+ if( rid ){
+ static Stmt q;
+ db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
+ db_bind_int(&q, ":r", rid);
+ db_step(&q);
+ db_reset(&q);
+ }
+}
+
+/*
+** Remember that the other side of the connection lacks a copy of
+** the artifact with the given hash.
+*/
+static void remote_unk(Blob *pHash){
+ static Stmt q;
+ db_static_prepare(&q, "INSERT OR IGNORE INTO unk VALUES(:h)");
+ db_bind_text(&q, ":h", blob_str(pHash));
+ db_step(&q);
+ db_reset(&q);
+}
+
+/*
+** The aToken[0..nToken-1] blob array is a parse of a "file" line
+** message. This routine finishes parsing that message and does
+** a record insert of the file.
+**
+** The file line is in one of the following two forms:
+**
+** file HASH SIZE \n CONTENT
+** file HASH DELTASRC SIZE \n CONTENT
+**
+** The content is SIZE bytes immediately following the newline.
+** If DELTASRC exists, then the CONTENT is a delta against the
+** content of DELTASRC.
+**
+** If any error occurs, write a message into pErr which has already
+** be initialized to an empty string.
+**
+** Any artifact successfully received by this routine is considered to
+** be public and is therefore removed from the "private" table.
+*/
+static void xfer_accept_file(
+ Xfer *pXfer,
+ int cloneFlag,
+ char **pzUuidList,
+ int *pnUuidList
+){
+ int n;
+ int rid;
+ int srcid = 0;
+ Blob content;
+ int isPriv;
+ Blob *pUuid;
+
+ isPriv = pXfer->nextIsPrivate;
+ pXfer->nextIsPrivate = 0;
+ if( pXfer->nToken<3
+ || pXfer->nToken>4
+ || !blob_is_hname(&pXfer->aToken[1])
+ || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &n)
+ || n<0
+ || (pXfer->nToken==4 && !blob_is_hname(&pXfer->aToken[2]))
+ ){
+ blob_appendf(&pXfer->err, "malformed file line");
+ return;
+ }
+ blob_zero(&content);
+ blob_extract(pXfer->pIn, n, &content);
+ pUuid = &pXfer->aToken[1];
+ if( !cloneFlag && uuid_is_shunned(blob_str(pUuid)) ){
+ /* Ignore files that have been shunned */
+ blob_reset(&content);
+ return;
+ }
+ if( isPriv && !g.perm.Private ){
+ /* Do not accept private files if not authorized */
+ blob_reset(&content);
+ return;
+ }
+ if( cloneFlag ){
+ if( pXfer->nToken==4 ){
+ srcid = rid_from_uuid(&pXfer->aToken[2], 1, isPriv);
+ pXfer->nDeltaRcvd++;
+ }else{
+ srcid = 0;
+ pXfer->nFileRcvd++;
+ }
+ rid = content_put_ex(&content, blob_str(pUuid), srcid,
+ 0, isPriv);
+ Th_AppendToList(pzUuidList, pnUuidList, blob_str(pUuid),
+ blob_size(pUuid));
+ remote_has(rid);
+ blob_reset(&content);
+ return;
+ }
+ if( pXfer->nToken==4 ){
+ Blob src, next;
+ srcid = rid_from_uuid(&pXfer->aToken[2], 1, isPriv);
+ if( content_get(srcid, &src)==0 ){
+ rid = content_put_ex(&content, blob_str(pUuid), srcid,
+ 0, isPriv);
+ Th_AppendToList(pzUuidList, pnUuidList, blob_str(pUuid),
+ blob_size(pUuid));
+ pXfer->nDanglingFile++;
+ db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
+ if( !isPriv ) content_make_public(rid);
+ blob_reset(&src);
+ blob_reset(&content);
+ return;
+ }
+ pXfer->nDeltaRcvd++;
+ blob_delta_apply(&src, &content, &next);
+ blob_reset(&src);
+ blob_reset(&content);
+ content = next;
+ }else{
+ pXfer->nFileRcvd++;
+ }
+ if( hname_verify_hash(&content, blob_buffer(pUuid), blob_size(pUuid))==0 ){
+ blob_appendf(&pXfer->err, "wrong hash on received artifact: %b", pUuid);
+ }
+ rid = content_put_ex(&content, blob_str(pUuid), 0, 0, isPriv);
+ Th_AppendToList(pzUuidList, pnUuidList, blob_str(pUuid), blob_size(pUuid));
+ if( rid==0 ){
+ blob_appendf(&pXfer->err, "%s", g.zErrMsg);
+ blob_reset(&content);
+ }else{
+ if( !isPriv ) content_make_public(rid);
+ manifest_crosslink(rid, &content, MC_NO_ERRORS);
+ }
+ assert( blob_is_reset(&content) );
+ remote_has(rid);
+}
+
+/*
+** The aToken[0..nToken-1] blob array is a parse of a "cfile" line
+** message. This routine finishes parsing that message and does
+** a record insert of the file. The difference between "file" and
+** "cfile" is that with "cfile" the content is already compressed.
+**
+** The file line is in one of the following two forms:
+**
+** cfile HASH USIZE CSIZE \n CONTENT
+** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
+**
+** The content is CSIZE bytes immediately following the newline.
+** If DELTASRC exists, then the CONTENT is a delta against the
+** content of DELTASRC.
+**
+** The original size of the HASH artifact is USIZE.
+**
+** If any error occurs, write a message into pErr which has already
+** be initialized to an empty string.
+**
+** Any artifact successfully received by this routine is considered to
+** be public and is therefore removed from the "private" table.
+*/
+static void xfer_accept_compressed_file(
+ Xfer *pXfer,
+ char **pzUuidList,
+ int *pnUuidList
+){
+ int szC; /* CSIZE */
+ int szU; /* USIZE */
+ int rid;
+ int srcid = 0;
+ Blob content;
+ int isPriv;
+
+ isPriv = pXfer->nextIsPrivate;
+ pXfer->nextIsPrivate = 0;
+ if( pXfer->nToken<4
+ || pXfer->nToken>5
+ || !blob_is_hname(&pXfer->aToken[1])
+ || !blob_is_int(&pXfer->aToken[pXfer->nToken-2], &szU)
+ || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &szC)
+ || szC<0 || szU<0
+ || (pXfer->nToken==5 && !blob_is_hname(&pXfer->aToken[2]))
+ ){
+ blob_appendf(&pXfer->err, "malformed cfile line");
+ return;
+ }
+ if( isPriv && !g.perm.Private ){
+ /* Do not accept private files if not authorized */
+ return;
+ }
+ blob_zero(&content);
+ blob_extract(pXfer->pIn, szC, &content);
+ if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
+ /* Ignore files that have been shunned */
+ blob_reset(&content);
+ return;
+ }
+ if( pXfer->nToken==5 ){
+ srcid = rid_from_uuid(&pXfer->aToken[2], 1, isPriv);
+ pXfer->nDeltaRcvd++;
+ }else{
+ srcid = 0;
+ pXfer->nFileRcvd++;
+ }
+ rid = content_put_ex(&content, blob_str(&pXfer->aToken[1]), srcid,
+ szC, isPriv);
+ Th_AppendToList(pzUuidList, pnUuidList, blob_str(&pXfer->aToken[1]),
+ blob_size(&pXfer->aToken[1]));
+ remote_has(rid);
+ blob_reset(&content);
+}
+
+/*
+** The aToken[0..nToken-1] blob array is a parse of a "uvfile" line
+** message. This routine finishes parsing that message and adds the
+** unversioned file to the "unversioned" table.
+**
+** The file line is in one of the following two forms:
+**
+** uvfile NAME MTIME HASH SIZE FLAGS
+** uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
+**
+** If the 0x0001 bit of FLAGS is set, that means the file has been
+** deleted, SIZE is zero, the HASH is "-", and the "\n CONTENT" is omitted.
+**
+** SIZE is the number of bytes of CONTENT. The CONTENT is uncompressed.
+** HASH is the artifact hash of CONTENT.
+**
+** If the 0x0004 bit of FLAGS is set, that means the CONTENT is omitted.
+** The sender might have omitted the content because it is too big to
+** transmit, or because it is unchanged and this record exists purely
+** to update the MTIME.
+*/
+static void xfer_accept_unversioned_file(Xfer *pXfer, int isWriter){
+ sqlite3_int64 mtime; /* The MTIME */
+ Blob *pHash; /* The HASH value */
+ int sz; /* The SIZE */
+ int flags; /* The FLAGS */
+ Blob content; /* The CONTENT */
+ Blob x; /* Compressed content */
+ Stmt q; /* SQL statements for comparison and insert */
+ int isDelete; /* HASH is "-" indicating this is a delete */
+ int nullContent; /* True of CONTENT is NULL */
+ int iStatus; /* Result from unversioned_status() */
+
+ pHash = &pXfer->aToken[3];
+ if( pXfer->nToken==5
+ || !blob_is_filename(&pXfer->aToken[1])
+ || !blob_is_int64(&pXfer->aToken[2], &mtime)
+ || (!blob_eq(pHash,"-") && !blob_is_hname(pHash))
+ || !blob_is_int(&pXfer->aToken[4], &sz)
+ || !blob_is_int(&pXfer->aToken[5], &flags)
+ || (mtime<0 || sz<0 || flags<0)
+ ){
+ blob_appendf(&pXfer->err, "malformed uvfile line");
+ return;
+ }
+ blob_init(&content, 0, 0);
+ blob_init(&x, 0, 0);
+ if( sz>0 && (flags & 0x0005)==0 ){
+ blob_extract(pXfer->pIn, sz, &content);
+ nullContent = 0;
+ if( hname_verify_hash(&content, blob_buffer(pHash), blob_size(pHash))==0 ){
+ blob_appendf(&pXfer->err, "in uvfile line, HASH does not match CONTENT");
+ goto end_accept_unversioned_file;
+ }
+ }else{
+ nullContent = 1;
+ }
+
+ /* The isWriter flag must be true in order to land the new file */
+ if( !isWriter ){
+ blob_appendf(&pXfer->err,"Write permissions for unversioned files missing");
+ goto end_accept_unversioned_file;
+ }
+
+ /* Make sure we have a valid g.rcvid marker */
+ content_rcvid_init(0);
+
+ /* Check to see if current content really should be overwritten. Ideally,
+ ** a uvfile card should never have been sent unless the overwrite should
+ ** occur. But do not trust the sender. Double-check.
+ */
+ iStatus = unversioned_status(blob_str(&pXfer->aToken[1]), mtime,
+ blob_str(pHash));
+ if( iStatus>=3 ) goto end_accept_unversioned_file;
+
+ /* Store the content */
+ isDelete = blob_eq(pHash, "-");
+ if( isDelete ){
+ db_prepare(&q,
+ "UPDATE unversioned"
+ " SET rcvid=:rcvid, mtime=:mtime, hash=NULL,"
+ " sz=0, encoding=0, content=NULL"
+ " WHERE name=:name"
+ );
+ db_bind_int(&q, ":rcvid", g.rcvid);
+ }else if( iStatus==2 ){
+ db_prepare(&q, "UPDATE unversioned SET mtime=:mtime WHERE name=:name");
+ }else{
+ db_prepare(&q,
+ "REPLACE INTO unversioned(name,rcvid,mtime,hash,sz,encoding,content)"
+ " VALUES(:name,:rcvid,:mtime,:hash,:sz,:encoding,:content)"
+ );
+ db_bind_int(&q, ":rcvid", g.rcvid);
+ db_bind_text(&q, ":hash", blob_str(pHash));
+ db_bind_int(&q, ":sz", blob_size(&content));
+ if( !nullContent ){
+ blob_compress(&content, &x);
+ if( blob_size(&x) < 0.8*blob_size(&content) ){
+ db_bind_blob(&q, ":content", &x);
+ db_bind_int(&q, ":encoding", 1);
+ }else{
+ db_bind_blob(&q, ":content", &content);
+ db_bind_int(&q, ":encoding", 0);
+ }
+ }else{
+ db_bind_int(&q, ":encoding", 0);
+ }
+ }
+ db_bind_text(&q, ":name", blob_str(&pXfer->aToken[1]));
+ db_bind_int64(&q, ":mtime", mtime);
+ db_step(&q);
+ db_finalize(&q);
+ db_unset("uv-hash", 0);
+
+end_accept_unversioned_file:
+ blob_reset(&x);
+ blob_reset(&content);
+}
+
+/*
+** Try to send a file as a delta against its parent.
+** If successful, return the number of bytes in the delta.
+** If we cannot generate an appropriate delta, then send
+** nothing and return zero.
+**
+** Never send a delta against a private artifact.
+*/
+static int send_delta_parent(
+ Xfer *pXfer, /* The transfer context */
+ int rid, /* record id of the file to send */
+ int isPrivate, /* True if rid is a private artifact */
+ Blob *pContent, /* The content of the file to send */
+ Blob *pUuid /* The HASH of the file to send */
+){
+ static const char *const azQuery[] = {
+ "SELECT pid FROM plink x"
+ " WHERE cid=%d"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)",
+
+ "SELECT pid, min(mtime) FROM mlink, event ON mlink.mid=event.objid"
+ " WHERE fid=%d"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
+ };
+ int i;
+ Blob src, delta;
+ int size = 0;
+ int srcId = 0;
+
+ for(i=0; srcId==0 && i<count(azQuery); i++){
+ srcId = db_int(0, azQuery[i] /*works-like:"%d"*/, rid);
+ }
+ if( srcId>0
+ && (pXfer->syncPrivate || !content_is_private(srcId))
+ && content_get(srcId, &src)
+ ){
+ char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", srcId);
+ blob_delta_create(&src, pContent, &delta);
+ size = blob_size(&delta);
+ if( size>=(int)blob_size(pContent)-50 ){
+ size = 0;
+ }else if( uuid_is_shunned(zUuid) ){
+ size = 0;
+ }else{
+ if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
+ blob_appendf(pXfer->pOut, "file %b %s %d\n", pUuid, zUuid, size);
+ blob_append(pXfer->pOut, blob_buffer(&delta), size);
+ }
+ blob_reset(&delta);
+ free(zUuid);
+ blob_reset(&src);
+ }
+ return size;
+}
+
+/*
+** Try to send a file as a native delta.
+** If successful, return the number of bytes in the delta.
+** If we cannot generate an appropriate delta, then send
+** nothing and return zero.
+**
+** Never send a delta against a private artifact.
+*/
+static int send_delta_native(
+ Xfer *pXfer, /* The transfer context */
+ int rid, /* record id of the file to send */
+ int isPrivate, /* True if rid is a private artifact */
+ Blob *pUuid /* The HASH of the file to send */
+){
+ Blob src, delta;
+ int size = 0;
+ int srcId;
+
+ srcId = db_int(0, "SELECT srcid FROM delta WHERE rid=%d", rid);
+ if( srcId>0
+ && (pXfer->syncPrivate || !content_is_private(srcId))
+ ){
+ blob_zero(&src);
+ db_blob(&src, "SELECT uuid FROM blob WHERE rid=%d", srcId);
+ if( uuid_is_shunned(blob_str(&src)) ){
+ blob_reset(&src);
+ return 0;
+ }
+ blob_zero(&delta);
+ db_blob(&delta, "SELECT content FROM blob WHERE rid=%d", rid);
+ blob_uncompress(&delta, &delta);
+ if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
+ blob_appendf(pXfer->pOut, "file %b %b %d\n",
+ pUuid, &src, blob_size(&delta));
+ blob_append(pXfer->pOut, blob_buffer(&delta), blob_size(&delta));
+ size = blob_size(&delta);
+ blob_reset(&delta);
+ blob_reset(&src);
+ }else{
+ size = 0;
+ }
+ return size;
+}
+
+/*
+** Push an error message to alert the older client that the repository
+** has SHA3 content and cannot be synced or cloned.
+*/
+static void xfer_cannot_send_sha3_error(Xfer *pXfer){
+ blob_appendf(pXfer->pOut,
+ "error Fossil\\sversion\\s2.0\\sor\\slater\\srequired.\n"
+ );
+}
+
+
+/*
+** Send the file identified by rid.
+**
+** The pUuid can be NULL in which case the correct hash is computed
+** from the rid.
+**
+** Try to send the file as a native delta if nativeDelta is true, or
+** as a parent delta if nativeDelta is false.
+**
+** It should never be the case that rid is a private artifact. But
+** as a precaution, this routine does check on rid and if it is private
+** this routine becomes a no-op.
+*/
+static void send_file(Xfer *pXfer, int rid, Blob *pUuid, int nativeDelta){
+ Blob content, uuid;
+ int size = 0;
+ int isPriv = content_is_private(rid);
+
+ if( isPriv && pXfer->syncPrivate==0 ){
+ if( pXfer->remoteDate>=20200413 && pUuid && blob_size(pUuid)>0 ){
+ /* If the artifact is private and we are not doing a private sync,
+ ** at least tell the other side that the artifact exists and is
+ ** known to be private. But only do this for newer clients since
+ ** older ones will throw an error if they get a private igot card
+ ** and private syncing is disallowed */
+ blob_appendf(pXfer->pOut, "igot %b 1\n", pUuid);
+ pXfer->nIGotSent++;
+ pXfer->nPrivIGot++;
+ }
+ return;
+ }
+ if( db_exists("SELECT 1 FROM onremote WHERE rid=%d", rid) ){
+ return;
+ }
+ blob_zero(&uuid);
+ db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
+ if( blob_size(&uuid)==0 ){
+ return;
+ }
+ if( blob_size(&uuid)>HNAME_LEN_SHA1 && pXfer->remoteVersion<20000 ){
+ xfer_cannot_send_sha3_error(pXfer);
+ return;
+ }
+ if( pUuid ){
+ if( blob_compare(pUuid, &uuid)!=0 ){
+ blob_reset(&uuid);
+ return;
+ }
+ }else{
+ pUuid = &uuid;
+ }
+ if( uuid_is_shunned(blob_str(pUuid)) ){
+ blob_reset(&uuid);
+ return;
+ }
+ if( (pXfer->maxTime != -1 && time(NULL) >= pXfer->maxTime) ||
+ pXfer->mxSend<=(int)blob_size(pXfer->pOut) ){
+ const char *zFormat = isPriv ? "igot %b 1\n" : "igot %b\n";
+ blob_appendf(pXfer->pOut, zFormat /*works-like:"%b"*/, pUuid);
+ pXfer->nIGotSent++;
+ blob_reset(&uuid);
+ return;
+ }
+ if( nativeDelta ){
+ size = send_delta_native(pXfer, rid, isPriv, pUuid);
+ if( size ){
+ pXfer->nDeltaSent++;
+ }
+ }
+ if( size==0 ){
+ content_get(rid, &content);
+
+ if( !nativeDelta && blob_size(&content)>100 ){
+ size = send_delta_parent(pXfer, rid, isPriv, &content, pUuid);
+ }
+ if( size==0 ){
+ int size = blob_size(&content);
+ if( isPriv ) blob_append(pXfer->pOut, "private\n", -1);
+ blob_appendf(pXfer->pOut, "file %b %d\n", pUuid, size);
+ blob_append(pXfer->pOut, blob_buffer(&content), size);
+ pXfer->nFileSent++;
+ }else{
+ pXfer->nDeltaSent++;
+ }
+ blob_reset(&content);
+ }
+ remote_has(rid);
+ blob_reset(&uuid);
+#if 0
+ if( blob_buffer(pXfer->pOut)[blob_size(pXfer->pOut)-1]!='\n' ){
+ blob_append(pXfer->pOut, "\n", 1);
+ }
+#endif
+}
+
+/*
+** Send the file identified by rid as a compressed artifact. Basically,
+** send the content exactly as it appears in the BLOB table using
+** a "cfile" card.
+*/
+static void send_compressed_file(Xfer *pXfer, int rid){
+ const char *zContent;
+ const char *zUuid;
+ const char *zDelta;
+ int szU;
+ int szC;
+ int rc;
+ int isPrivate;
+ int srcIsPrivate;
+ static Stmt q1;
+ Blob fullContent;
+
+ isPrivate = content_is_private(rid);
+ if( isPrivate && pXfer->syncPrivate==0 ) return;
+ db_static_prepare(&q1,
+ "SELECT uuid, size, content, delta.srcid IN private,"
+ " (SELECT uuid FROM blob WHERE rid=delta.srcid)"
+ " FROM blob LEFT JOIN delta ON (blob.rid=delta.rid)"
+ " WHERE blob.rid=:rid"
+ " AND blob.size>=0"
+ " AND NOT EXISTS(SELECT 1 FROM shun WHERE shun.uuid=blob.uuid)"
+ );
+ db_bind_int(&q1, ":rid", rid);
+ rc = db_step(&q1);
+ if( rc==SQLITE_ROW ){
+ zUuid = db_column_text(&q1, 0);
+ szU = db_column_int(&q1, 1);
+ szC = db_column_bytes(&q1, 2);
+ zContent = db_column_raw(&q1, 2);
+ srcIsPrivate = db_column_int(&q1, 3);
+ zDelta = db_column_text(&q1, 4);
+ if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
+ if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,0)!=HNAME_LEN_SHA1 ){
+ xfer_cannot_send_sha3_error(pXfer);
+ db_reset(&q1);
+ return;
+ }
+ blob_appendf(pXfer->pOut, "cfile %s ", zUuid);
+ if( !isPrivate && srcIsPrivate ){
+ content_get(rid, &fullContent);
+ szU = blob_size(&fullContent);
+ blob_compress(&fullContent, &fullContent);
+ szC = blob_size(&fullContent);
+ zContent = blob_buffer(&fullContent);
+ zDelta = 0;
+ }
+ if( zDelta ){
+ blob_appendf(pXfer->pOut, "%s ", zDelta);
+ pXfer->nDeltaSent++;
+ }else{
+ pXfer->nFileSent++;
+ }
+ blob_appendf(pXfer->pOut, "%d %d\n", szU, szC);
+ blob_append(pXfer->pOut, zContent, szC);
+ if( blob_buffer(pXfer->pOut)[blob_size(pXfer->pOut)-1]!='\n' ){
+ blob_append(pXfer->pOut, "\n", 1);
+ }
+ if( !isPrivate && srcIsPrivate ){
+ blob_reset(&fullContent);
+ }
+ }
+ db_reset(&q1);
+}
+
+/*
+** Send the unversioned file identified by zName by generating the
+** appropriate "uvfile" card.
+**
+** uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
+**
+** If the noContent flag is set, omit the CONTENT and set the 0x0004
+** flag in FLAGS.
+*/
+static void send_unversioned_file(
+ Xfer *pXfer, /* Transfer context */
+ const char *zName, /* Name of unversioned file to be sent */
+ int noContent /* True to omit the content */
+){
+ Stmt q1;
+
+ if( (int)blob_size(pXfer->pOut)>=pXfer->mxSend ) noContent = 1;
+ if( noContent ){
+ db_prepare(&q1,
+ "SELECT mtime, hash, encoding, sz FROM unversioned WHERE name=%Q",
+ zName
+ );
+ }else{
+ db_prepare(&q1,
+ "SELECT mtime, hash, encoding, sz, content FROM unversioned"
+ " WHERE name=%Q",
+ zName
+ );
+ }
+ if( db_step(&q1)==SQLITE_ROW ){
+ sqlite3_int64 mtime = db_column_int64(&q1, 0);
+ const char *zHash = db_column_text(&q1, 1);
+ if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,1)>HNAME_LEN_SHA1 ){
+ xfer_cannot_send_sha3_error(pXfer);
+ db_reset(&q1);
+ return;
+ }
+ if( (int)blob_size(pXfer->pOut)>=pXfer->mxSend ){
+ /* If we have already reached the send size limit, send a (short)
+ ** uvigot card rather than a uvfile card. This only happens on the
+ ** server side. The uvigot card will provoke the client to resend
+ ** another uvgimme on the next cycle. */
+ blob_appendf(pXfer->pOut, "uvigot %s %lld %s %d\n",
+ zName, mtime, zHash, db_column_int(&q1,3));
+ }else{
+ blob_appendf(pXfer->pOut, "uvfile %s %lld", zName, mtime);
+ if( zHash==0 ){
+ blob_append(pXfer->pOut, " - 0 1\n", -1);
+ }else if( noContent ){
+ blob_appendf(pXfer->pOut, " %s %d 4\n", zHash, db_column_int(&q1,3));
+ }else{
+ Blob content;
+ blob_init(&content, 0, 0);
+ db_column_blob(&q1, 4, &content);
+ if( db_column_int(&q1, 2) ){
+ blob_uncompress(&content, &content);
+ }
+ blob_appendf(pXfer->pOut, " %s %d 0\n", zHash, blob_size(&content));
+ blob_append(pXfer->pOut, blob_buffer(&content), blob_size(&content));
+ blob_reset(&content);
+ }
+ }
+ }
+ db_finalize(&q1);
+}
+
+/*
+** Send a gimme message for every phantom.
+**
+** Except: do not request shunned artifacts. And do not request
+** private artifacts if we are not doing a private transfer.
+*/
+static void request_phantoms(Xfer *pXfer, int maxReq){
+ Stmt q;
+ db_prepare(&q,
+ "SELECT uuid FROM phantom CROSS JOIN blob USING(rid) /*scan*/"
+ " WHERE NOT EXISTS(SELECT 1 FROM unk WHERE unk.uuid=blob.uuid)"
+ " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid) %s",
+ (pXfer->syncPrivate ? "" :
+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)")
+ );
+ while( db_step(&q)==SQLITE_ROW && maxReq-- > 0 ){
+ const char *zUuid = db_column_text(&q, 0);
+ blob_appendf(pXfer->pOut, "gimme %s\n", zUuid);
+ pXfer->nGimmeSent++;
+ }
+ db_finalize(&q);
+}
+
+/*
+** Compute an hash on the tail of pMsg. Verify that it matches the
+** the hash given in pHash. Return non-zero for an error and 0 on success.
+**
+** The type of hash computed (SHA1, SHA3-256) is determined by
+** the length of the input hash in pHash.
+*/
+static int check_tail_hash(Blob *pHash, Blob *pMsg){
+ Blob tail;
+ int rc;
+ blob_tail(pMsg, &tail);
+ rc = hname_verify_hash(&tail, blob_buffer(pHash), blob_size(pHash));
+ blob_reset(&tail);
+ return rc==HNAME_ERROR;
+}
+
+/*
+** Check the signature on an application/x-fossil payload received by
+** the HTTP server. The signature is a line of the following form:
+**
+** login LOGIN NONCE SIGNATURE
+**
+** The NONCE is the SHA1 hash of the remainder of the input.
+** SIGNATURE is the SHA1 checksum of the NONCE concatenated
+** with the sha1_shared_secret() encoding of the users password.
+**
+** SIGNATURE = sha1_sum( NONCE + sha1_shared_secret(PASSWORD) );
+**
+** The parameters to this routine are ephemeral blobs holding the
+** LOGIN, NONCE and SIGNATURE.
+**
+** This routine attempts to locate the user and verify the signature.
+** If everything checks out, the USER.CAP column for the USER table
+** is consulted to set privileges in the global g variable.
+**
+** If anything fails to check out, no changes are made to privileges.
+**
+** Signature generation on the client side is handled by the
+** http_exchange() routine.
+**
+** Return non-zero for a login failure and zero for success.
+*/
+static int check_login(Blob *pLogin, Blob *pNonce, Blob *pSig){
+ Stmt q;
+ int rc = -1;
+ char *zLogin = blob_terminate(pLogin);
+ defossilize(zLogin);
+
+ if( fossil_strcmp(zLogin, "nobody")==0
+ || fossil_strcmp(zLogin, "anonymous")==0
+ ){
+ return 0; /* Anybody is allowed to sync as "nobody" or "anonymous" */
+ }
+ if( fossil_strcmp(P("REMOTE_USER"), zLogin)==0
+ && db_get_boolean("remote_user_ok",0) ){
+ return 0; /* Accept Basic Authorization */
+ }
+ db_prepare(&q,
+ "SELECT pw, cap, uid FROM user"
+ " WHERE login=%Q"
+ " AND login NOT IN ('anonymous','nobody','developer','reader')"
+ " AND length(pw)>0",
+ zLogin
+ );
+ if( db_step(&q)==SQLITE_ROW ){
+ int szPw;
+ Blob pw, combined, hash;
+ blob_zero(&pw);
+ db_ephemeral_blob(&q, 0, &pw);
+ szPw = blob_size(&pw);
+ blob_zero(&combined);
+ blob_copy(&combined, pNonce);
+ blob_append(&combined, blob_buffer(&pw), szPw);
+ sha1sum_blob(&combined, &hash);
+ assert( blob_size(&hash)==40 );
+ rc = blob_constant_time_cmp(&hash, pSig);
+ blob_reset(&hash);
+ blob_reset(&combined);
+ if( rc!=0 && szPw!=40 ){
+ /* If this server stores cleartext passwords and the password did not
+ ** match, then perhaps the client is sending SHA1 passwords. Try
+ ** again with the SHA1 password.
+ */
+ const char *zPw = db_column_text(&q, 0);
+ char *zSecret = sha1_shared_secret(zPw, blob_str(pLogin), 0);
+ blob_zero(&combined);
+ blob_copy(&combined, pNonce);
+ blob_append(&combined, zSecret, -1);
+ fossil_free(zSecret);
+ sha1sum_blob(&combined, &hash);
+ rc = blob_constant_time_cmp(&hash, pSig);
+ blob_reset(&hash);
+ blob_reset(&combined);
+ }
+ if( rc==0 ){
+ const char *zCap;
+ zCap = db_column_text(&q, 1);
+ login_set_capabilities(zCap, 0);
+ g.userUid = db_column_int(&q, 2);
+ g.zLogin = mprintf("%b", pLogin);
+ g.zNonce = mprintf("%b", pNonce);
+ }
+ }
+ db_finalize(&q);
+ return rc;
+}
+
+/*
+** Send the content of all files in the unsent table.
+**
+** This is really just an optimization. If you clear the
+** unsent table, all the right files will still get transferred.
+** It just might require an extra round trip or two.
+*/
+static void send_unsent(Xfer *pXfer){
+ Stmt q;
+ db_prepare(&q, "SELECT rid FROM unsent EXCEPT SELECT rid FROM private");
+ while( db_step(&q)==SQLITE_ROW ){
+ int rid = db_column_int(&q, 0);
+ send_file(pXfer, rid, 0, 0);
+ }
+ db_finalize(&q);
+ db_multi_exec("DELETE FROM unsent");
+}
+
+/*
+** Check to see if the number of unclustered entries is greater than
+** 100 and if it is, form a new cluster. Unclustered phantoms do not
+** count toward the 100 total. And phantoms are never added to a new
+** cluster.
+*/
+void create_cluster(void){
+ Blob cluster, cksum;
+ Blob deleteWhere;
+ Stmt q;
+ int nUncl;
+ int nRow = 0;
+ int rid;
+
+#if 0
+ /* We should not ever get any private artifacts in the unclustered table.
+ ** But if we do (because of a bug) now is a good time to delete them. */
+ db_multi_exec(
+ "DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
+ );
+#endif
+
+ nUncl = db_int(0, "SELECT count(*) FROM unclustered /*scan*/"
+ " WHERE NOT EXISTS(SELECT 1 FROM phantom"
+ " WHERE rid=unclustered.rid)");
+ if( nUncl>=100 ){
+ blob_zero(&cluster);
+ blob_zero(&deleteWhere);
+ db_prepare(&q, "SELECT uuid FROM unclustered, blob"
+ " WHERE NOT EXISTS(SELECT 1 FROM phantom"
+ " WHERE rid=unclustered.rid)"
+ " AND unclustered.rid=blob.rid"
+ " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
+ " ORDER BY 1");
+ while( db_step(&q)==SQLITE_ROW ){
+ blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
+ nRow++;
+ if( nRow>=800 && nUncl>nRow+100 ){
+ md5sum_blob(&cluster, &cksum);
+ blob_appendf(&cluster, "Z %b\n", &cksum);
+ blob_reset(&cksum);
+ rid = content_put(&cluster);
+ manifest_crosslink(rid, &cluster, MC_NONE);
+ blob_reset(&cluster);
+ nUncl -= nRow;
+ nRow = 0;
+ blob_append_sql(&deleteWhere, ",%d", rid);
+ }
+ }
+ db_finalize(&q);
+ db_multi_exec(
+ "DELETE FROM unclustered WHERE rid NOT IN (0 %s)"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=unclustered.rid)",
+ blob_sql_text(&deleteWhere)
+ );
+ blob_reset(&deleteWhere);
+ if( nRow>0 ){
+ md5sum_blob(&cluster, &cksum);
+ blob_appendf(&cluster, "Z %b\n", &cksum);
+ blob_reset(&cksum);
+ rid = content_put(&cluster);
+ manifest_crosslink(rid, &cluster, MC_NONE);
+ blob_reset(&cluster);
+ }
+ }
+}
+
+/*
+** Send igot messages for every private artifact
+*/
+static int send_private(Xfer *pXfer){
+ int cnt = 0;
+ Stmt q;
+ if( pXfer->syncPrivate ){
+ db_prepare(&q, "SELECT uuid FROM private JOIN blob USING(rid)");
+ while( db_step(&q)==SQLITE_ROW ){
+ blob_appendf(pXfer->pOut, "igot %s 1\n", db_column_text(&q,0));
+ cnt++;
+ }
+ db_finalize(&q);
+ }
+ return cnt;
+}
+
+/*
+** Send an igot message for every entry in unclustered table.
+** Return the number of cards sent.
+**
+** Except:
+** * Do not send igot cards for shunned artifacts
+** * Do not send igot cards for phantoms
+** * Do not send igot cards for private artifacts
+** * Do not send igot cards for any artifact that is in the
+** ONREMOTE table, if that table exists.
+**
+** If the pXfer->resync flag is set, that means we are doing a "--verily"
+** sync and all artifacts that don't meet the restrictions above should
+** be sent.
+*/
+static int send_unclustered(Xfer *pXfer){
+ Stmt q;
+ int cnt = 0;
+ const char *zExtra;
+ if( db_table_exists("temp","onremote") ){
+ zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
+ }else{
+ zExtra = "";
+ }
+ if( pXfer->resync ){
+ db_prepare(&q,
+ "SELECT uuid, rid FROM blob"
+ " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s"
+ " AND blob.rid<=%d"
+ " ORDER BY blob.rid DESC",
+ zExtra /*safe-for-%s*/, pXfer->resync
+ );
+ }else{
+ db_prepare(&q,
+ "SELECT uuid FROM unclustered JOIN blob USING(rid) /*scan*/"
+ " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
+ zExtra /*safe-for-%s*/
+ );
+ }
+ while( db_step(&q)==SQLITE_ROW ){
+ blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
+ cnt++;
+ if( pXfer->resync && pXfer->mxSend<(int)blob_size(pXfer->pOut) ){
+ pXfer->resync = db_column_int(&q, 1)-1;
+ }
+ }
+ db_finalize(&q);
+ if( cnt==0 ) pXfer->resync = 0;
+ return cnt;
+}
+
+/*
+** Send an igot message for every cluster artifact that is not a phantom,
+** is not shunned, is not private, and that is not in the UNCLUSTERED table.
+** Return the number of cards sent.
+*/
+static int send_all_clusters(Xfer *pXfer){
+ Stmt q;
+ int cnt = 0;
+ const char *zExtra;
+ if( db_table_exists("temp","onremote") ){
+ zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
+ }else{
+ zExtra = "";
+ }
+ db_prepare(&q,
+ "SELECT uuid"
+ " FROM tagxref JOIN blob ON tagxref.rid=blob.rid AND tagxref.tagid=%d"
+ " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
+ " AND NOT EXISTS(SELECT 1 FROM unclustered WHERE rid=blob.rid)"
+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
+ TAG_CLUSTER, zExtra /*safe-for-%s*/
+ );
+ while( db_step(&q)==SQLITE_ROW ){
+ if( cnt==0 ) blob_appendf(pXfer->pOut, "# sending-clusters\n");
+ blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
+ cnt++;
+ }
+ db_finalize(&q);
+ if( cnt ) blob_appendf(pXfer->pOut, "# end-of-clusters\n");
+ return cnt;
+}
+
+/*
+** Send an igot message for every artifact.
+*/
+static void send_all(Xfer *pXfer){
+ Stmt q;
+ db_prepare(&q,
+ "SELECT uuid FROM blob "
+ " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
+ );
+ while( db_step(&q)==SQLITE_ROW ){
+ blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
+ }
+ db_finalize(&q);
+}
+
+/*
+** pXfer is a "pragma uv-hash HASH" card.
+**
+** If HASH is different from the unversioned content hash on this server,
+** then send a bunch of uvigot cards, one for each entry unversioned file
+** on this server.
+*/
+static void send_unversioned_catalog(Xfer *pXfer){
+ Stmt uvq;
+ unversioned_schema();
+ db_prepare(&uvq,
+ "SELECT name, mtime, hash, sz FROM unversioned"
+ );
+ while( db_step(&uvq)==SQLITE_ROW ){
+ const char *zName = db_column_text(&uvq,0);
+ sqlite3_int64 mtime = db_column_int64(&uvq,1);
+ const char *zHash = db_column_text(&uvq,2);
+ int sz = db_column_int(&uvq,3);
+ if( zHash==0 ){ sz = 0; zHash = "-"; }
+ blob_appendf(pXfer->pOut, "uvigot %s %lld %s %d\n",
+ zName, mtime, zHash, sz);
+ }
+ db_finalize(&uvq);
+}
+
+/*
+** Return a string that contains supplemental information about a
+** "not authorized" error. The string might be empty if no additional
+** information is available.
+*/
+static char *whyNotAuth(void){
+ if( g.useLocalauth && db_get_boolean("localauth",0)!=0 ){
+ return "\\sbecause\\sthe\\s'localauth'\\ssetting\\sis\\senabled";
+ }
+ return "";
+}
+
+/*
+** Called when there is an attempt to transfer private content to and
+** from a server without authorization.
+*/
+static void server_private_xfer_not_authorized(void){
+ @ error not\sauthorized\sto\ssync\sprivate\scontent%s(whyNotAuth())
+}
+
+/*
+** Return the common TH1 code to evaluate prior to evaluating any other
+** TH1 transfer notification scripts.
+*/
+const char *xfer_common_code(void){
+ return db_get("xfer-common-script", 0);
+}
+
+/*
+** Return the TH1 code to evaluate when a push is processed.
+*/
+const char *xfer_push_code(void){
+ return db_get("xfer-push-script", 0);
+}
+
+/*
+** Return the TH1 code to evaluate when a commit is processed.
+*/
+const char *xfer_commit_code(void){
+ return db_get("xfer-commit-script", 0);
+}
+
+/*
+** Return the TH1 code to evaluate when a ticket change is processed.
+*/
+const char *xfer_ticket_code(void){
+ return db_get("xfer-ticket-script", 0);
+}
+
+/*
+** Reset the CGI content, roll back any pending db transaction, and
+** emit an "error" xfer message. The message text gets fossil-encoded
+** by this function. This is only intended for use with
+** fail-fast/fatal errors, not ones which can be skipped over.
+*/
+static void xfer_fatal_error(const char *zMsg){
+ cgi_reset_content();
+ if( db_transaction_nesting_depth()>0 ){
+ db_rollback_transaction();
+ }
+ @ error %F(zMsg)
+}
+
+/*
+** Run the specified TH1 script, if any, and returns 1 on error.
+*/
+int xfer_run_script(
+ const char *zScript,
+ const char *zUuidOrList,
+ int bIsList
+){
+ int rc = TH_OK;
+ if( !zScript ) return rc;
+ Th_FossilInit(TH_INIT_DEFAULT);
+ Th_Store(bIsList ? "uuids" : "uuid", zUuidOrList ? zUuidOrList : "");
+ rc = Th_Eval(g.interp, 0, zScript, -1);
+ if( rc!=TH_OK ){
+ fossil_error(1, "%s", Th_GetResult(g.interp, 0));
+ }
+ return rc;
+}
+
+/*
+** Runs the pre-transfer TH1 script, if any, and returns its return code.
+** This script may be run multiple times. If the script performs actions
+** that cannot be redone, it should use an internal [if] guard similar to
+** the following:
+**
+** if {![info exists common_done]} {
+** # ... code here
+** set common_done 1
+** }
+*/
+int xfer_run_common_script(void){
+ return xfer_run_script(xfer_common_code(), 0, 0);
+}
+
+/*
+** This routine makes a "syncwith:URL" entry in the CONFIG table to
+** indicate that a sync is occurring with zUrl.
+**
+** Add a "syncfrom:URL" entry instead of "syncwith:URL" if bSyncFrom is true.
+*/
+static void xfer_syncwith(const char *zUrl, int bSyncFrom){
+ UrlData x;
+ memset(&x, 0, sizeof(x));
+ url_parse_local(zUrl, URL_OMIT_USER, &x);
+ if( x.protocol && strncmp(x.protocol,"http",4)==0
+ && x.name && sqlite3_strlike("%localhost%", x.name, 0)!=0
+ ){
+ db_unprotect(PROTECT_CONFIG);
+ db_multi_exec("REPLACE INTO config(name,value,mtime)"
+ "VALUES('sync%q:%q','{}',now())",
+ bSyncFrom ? "from" : "with", x.canonical);
+ db_protect_pop();
+ }
+ url_unparse(&x);
+}
+
+/*
+** If this variable is set, disable login checks. Used for debugging
+** only.
+*/
+static int disableLogin = 0;
+
+/*
+** Must be passed the version info from pragmas
+** client-version/server-version cards. If the version info is "new
+** enough" then the loginCardMode is ORd into the X-Fossil-Xfer-Login
+** card flag, else this is a no-op.
+*/
+static void xfer_xflc_check(int iRemoteVersion, int iDate, int iTime,
+ int fLoginCardMode){
+ if( iRemoteVersion>=22700
+ && (iDate > 20250727
+ || (iDate == 20250727 && iTime >= 110500)) ){
+ g.syncInfo.fLoginCardMode |= fLoginCardMode;
+ }
+}
+
+/*
+** The CGI/HTTP preprocessor always redirects requests with a content-type
+** of application/x-fossil or application/x-fossil-debug to this page,
+** regardless of what path was specified in the HTTP header. This allows
+** clone clients to specify a URL that omits default pathnames, such
+** as "http://fossil-scm.org/" instead of "http://fossil-scm.org/index.cgi".
+**
+** WEBPAGE: xfer raw-content loadavg-exempt
+**
+** This is the transfer handler on the server side. The transfer
+** message has been uncompressed and placed in the g.cgiIn blob.
+** Process this message and form an appropriate reply.
+*/
+void page_xfer(void){
+ int isPull = 0;
+ int isPush = 0;
+ int nErr = 0;
+ Xfer xfer;
+ int deltaFlag = 0;
+ int isClone = 0;
+ int nGimme = 0;
+ int size;
+ char *zNow;
+ int rc;
+ const char *zScript = 0;
+ char *zUuidList = 0;
+ int nUuidList = 0;
+ char **pzUuidList = 0;
+ int *pnUuidList = 0;
+ int uvCatalogSent = 0;
+ int bSendLinks = 0;
+ int nLogin = 0;
+
+ if( fossil_strcmp(PD("REQUEST_METHOD","POST"),"POST") ){
+ fossil_redirect_home();
+ }
+ g.zLogin = "anonymous";
+ login_set_anon_nobody_capabilities();
+ login_check_credentials();
+ if( g.perm.Clone ) g.perm.Read = 1;
+ cgi_check_for_malice();
+ memset(&xfer, 0, sizeof(xfer));
+ blobarray_zero(xfer.aToken, count(xfer.aToken));
+ cgi_set_content_type(g.zContentType);
+ cgi_reset_content();
+ if( db_schema_is_outofdate() ){
+ @ error database\sschema\sis\sout-of-date\son\sthe\sserver.
+ return;
+ }
+ blob_zero(&xfer.err);
+ xfer.pIn = &g.cgiIn;
+ xfer.pOut = cgi_output_blob();
+ xfer.mxSend = db_get_int("max-download", 5000000);
+ xfer.maxTime = db_get_int("max-download-time", 30);
+ if( xfer.maxTime<1 ) xfer.maxTime = 1;
+ xfer.maxTime += time(NULL);
+ g.xferPanic = 1;
+
+ db_begin_write();
+ db_multi_exec(
+ "CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
+ "CREATE TEMP TABLE unk(uuid TEXT PRIMARY KEY) WITHOUT ROWID;"
+ );
+ manifest_crosslink_begin();
+ rc = xfer_run_common_script();
+ if( rc==TH_ERROR ){
+ cgi_reset_content();
+ @ error common\sscript\sfailed:\s%F(g.zErrMsg)
+ nErr++;
+ }
+ zScript = xfer_push_code();
+ if( zScript ){ /* NOTE: Are TH1 transfer hooks enabled? */
+ pzUuidList = &zUuidList;
+ pnUuidList = &nUuidList;
+ }
+ if( g.syncInfo.zLoginCard ){
+ /* Login card received via HTTP Cookie header */
+ blob_zero(&xfer.line);
+ blob_append(&xfer.line, g.syncInfo.zLoginCard, -1);
+ xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken,
+ count(xfer.aToken));
+ fossil_free( g.syncInfo.zLoginCard );
+ g.syncInfo.zLoginCard = 0;
+ if( xfer.nToken==4
+ && blob_eq(&xfer.aToken[0], "login") ){
+ goto handle_login_card;
+ }
+ }
+ while( blob_line(xfer.pIn, &xfer.line) ){
+ if( blob_buffer(&xfer.line)[0]=='#' ) continue;
+ if( blob_size(&xfer.line)==0 ) continue;
+ xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
+
+ /* file HASH SIZE \n CONTENT
+ ** file HASH DELTASRC SIZE \n CONTENT
+ **
+ ** Server accepts a file from the client.
+ */
+ if( blob_eq(&xfer.aToken[0], "file") ){
+ if( !isPush ){
+ cgi_reset_content();
+ @ error not\sauthorized\sto\swrite%s(whyNotAuth())
+ nErr++;
+ break;
+ }
+ xfer_accept_file(&xfer, 0, pzUuidList, pnUuidList);
+ if( blob_size(&xfer.err) ){
+ cgi_reset_content();
+ @ error %T(blob_str(&xfer.err))
+ nErr++;
+ break;
+ }
+ }else
+
+ /* cfile HASH USIZE CSIZE \n CONTENT
+ ** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
+ **
+ ** Server accepts a compressed file from the client.
+ */
+ if( blob_eq(&xfer.aToken[0], "cfile") ){
+ if( !isPush ){
+ cgi_reset_content();
+ @ error not\sauthorized\sto\swrite%s(whyNotAuth())
+ nErr++;
+ break;
+ }
+ xfer_accept_compressed_file(&xfer, pzUuidList, pnUuidList);
+ if( blob_size(&xfer.err) ){
+ cgi_reset_content();
+ @ error %T(blob_str(&xfer.err))
+ nErr++;
+ break;
+ }
+ }else
+
+ /* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
+ **
+ ** Server accepts an unversioned file from the client.
+ */
+ if( blob_eq(&xfer.aToken[0], "uvfile") ){
+ xfer_accept_unversioned_file(&xfer, g.perm.WrUnver);
+ if( blob_size(&xfer.err) ){
+ cgi_reset_content();
+ @ error %T(blob_str(&xfer.err))
+ fossil_print("%%%%%%%% xfer.err: '%s'\n", blob_str(&xfer.err));
+ nErr++;
+ break;
+ }
+ }else
+
+ /* gimme HASH
+ **
+ ** Client is requesting a file from the server. Send it.
+ */
+ if( blob_eq(&xfer.aToken[0], "gimme")
+ && xfer.nToken==2
+ && blob_is_hname(&xfer.aToken[1])
+ ){
+ nGimme++;
+ remote_unk(&xfer.aToken[1]);
+ if( isPull ){
+ int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
+ if( rid ){
+ send_file(&xfer, rid, &xfer.aToken[1], deltaFlag);
+ }
+ }
+ }else
+
+ /* uvgimme NAME
+ **
+ ** Client is requesting an unversioned file from the server. Send it.
+ */
+ if( blob_eq(&xfer.aToken[0], "uvgimme")
+ && xfer.nToken==2
+ && blob_is_filename(&xfer.aToken[1])
+ ){
+ send_unversioned_file(&xfer, blob_str(&xfer.aToken[1]), 0);
+ }else
+
+ /* igot HASH ?ISPRIVATE?
+ **
+ ** Client announces that it has a particular file. If the ISPRIVATE
+ ** argument exists and is "1", then the file is a private file.
+ */
+ if( xfer.nToken>=2
+ && blob_eq(&xfer.aToken[0], "igot")
+ && blob_is_hname(&xfer.aToken[1])
+ ){
+ if( isPush ){
+ int rid = 0;
+ int isPriv = 0;
+ if( xfer.nToken==2 || blob_eq(&xfer.aToken[2],"1")==0 ){
+ /* Client says the artifact is public */
+ rid = rid_from_uuid(&xfer.aToken[1], 1, 0);
+ }else if( g.perm.Private ){
+ /* Client says the artifact is private and the client has
+ ** permission to push private content. Create a new phantom
+ ** artifact that is marked private. */
+ rid = rid_from_uuid(&xfer.aToken[1], 1, 1);
+ isPriv = 1;
+ }else{
+ /* Client says the artifact is private and the client is unable
+ ** or unwilling to send us the artifact. If we already hold the
+ ** artifact here on the server as a phantom, make sure that
+ ** phantom is marked as private so that we don't keep asking about
+ ** it in subsequent sync requests. */
+ rid = rid_from_uuid(&xfer.aToken[1], 0, 1);
+ isPriv = 1;
+ }
+ if( rid ){
+ remote_has(rid);
+ if( isPriv ){
+ content_make_private(rid);
+ }else{
+ content_make_public(rid);
+ }
+ }
+ }
+ }else
+
+
+ /* pull SERVERCODE PROJECTCODE
+ ** push SERVERCODE PROJECTCODE
+ **
+ ** The client wants either send or receive. The server should
+ ** verify that the project code matches. The server code is ignored.
+ */
+ if( xfer.nToken==3
+ && (blob_eq(&xfer.aToken[0], "pull") || blob_eq(&xfer.aToken[0], "push"))
+ && blob_is_hname(&xfer.aToken[2])
+ ){
+ const char *zPCode;
+ zPCode = db_get("project-code", 0);
+ if( zPCode==0 ){
+ fossil_fatal("missing project code");
+ }
+ if( !blob_eq_str(&xfer.aToken[2], zPCode, -1) ){
+ cgi_reset_content();
+ @ error wrong\sproject
+ nErr++;
+ break;
+ }
+ login_check_credentials();
+ if( blob_eq(&xfer.aToken[0], "pull") ){
+ if( !g.perm.Read ){
+ cgi_reset_content();
+ @ error not\sauthorized\sto\sread%s(whyNotAuth())
+ nErr++;
+ break;
+ }
+ isPull = 1;
+ }else{
+ if( !g.perm.Write ){
+ if( !isPull ){
+ cgi_reset_content();
+ @ error not\sauthorized\sto\swrite%s(whyNotAuth())
+ nErr++;
+ }else{
+ @ message pull\sonly\s-\snot\sauthorized\sto\spush%s(whyNotAuth())
+ }
+ }else{
+ isPush = 1;
+ }
+ }
+ }else
+
+ /* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
+ **
+ ** The client knows nothing. Tell all.
+ */
+ if( blob_eq(&xfer.aToken[0], "clone") ){
+ int iVers;
+ login_check_credentials();
+ if( !g.perm.Clone ){
+ cgi_reset_content();
+ @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
+ @ error not\sauthorized\sto\sclone%s(whyNotAuth())
+ nErr++;
+ break;
+ }
+ if( db_get_boolean("uv-sync",0) && !uvCatalogSent ){
+ @ pragma uv-pull-only
+ send_unversioned_catalog(&xfer);
+ uvCatalogSent = 1;
+ }
+ if( xfer.nToken==3
+ && blob_is_int(&xfer.aToken[1], &iVers)
+ && iVers>=2
+ ){
+ int seqno, max;
+ if( iVers>=3 ){
+ cgi_set_content_type("application/x-fossil-uncompressed");
+ }
+ blob_is_int(&xfer.aToken[2], &seqno);
+ if( seqno<=0 ){
+ xfer_fatal_error("invalid clone sequence number");
+ db_rollback_transaction();
+ return;
+ }
+ max = db_int(0, "SELECT max(rid) FROM blob");
+ while( xfer.mxSend>(int)blob_size(xfer.pOut) && seqno<=max){
+ if( time(NULL) >= xfer.maxTime ) break;
+ if( iVers>=3 ){
+ send_compressed_file(&xfer, seqno);
+ }else{
+ send_file(&xfer, seqno, 0, 1);
+ }
+ seqno++;
+ }
+ if( seqno>max ) seqno = 0;
+ @ clone_seqno %d(seqno)
+ }else{
+ isClone = 1;
+ isPull = 1;
+ deltaFlag = 1;
+ }
+ @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
+ }else
+
+ /* login USER NONCE SIGNATURE
+ **
+ ** The client has sent login credentials to the server.
+ ** Validate the login. This has to happen before anything else.
+ **
+ ** For many years, Fossil would accept multiple login cards with
+ ** cumulative permissions. But that feature was never used. Hence
+ ** it is now prohibited. Any login card after the first generates
+ ** a fatal error.
+ */
+ if( blob_eq(&xfer.aToken[0], "login")
+ && xfer.nToken==4
+ ){
+ handle_login_card:
+ nLogin++;
+ if( disableLogin ){
+ g.perm.Read = g.perm.Write = g.perm.Private = g.perm.Admin = 1;
+ }else if( nLogin > 1 ){
+ cgi_reset_content();
+ @ error multiple\slogin\cards
+ nErr++;
+ break;
+ }else{
+ if( check_tail_hash(&xfer.aToken[2], xfer.pIn)
+ || check_login(&xfer.aToken[1], &xfer.aToken[2], &xfer.aToken[3])
+ ){
+ cgi_reset_content();
+ @ error login\sfailed
+ nErr++;
+ break;
+ }
+ }
+ }else
+
+ /* reqconfig NAME
+ **
+ ** Client is requesting a configuration value from the server
+ */
+ if( blob_eq(&xfer.aToken[0], "reqconfig")
+ && xfer.nToken==2
+ ){
+ if( g.perm.Read ){
+ char *zName = blob_str(&xfer.aToken[1]);
+ if( zName[0]=='/' ){
+ /* New style configuration transfer */
+ int groupMask = configure_name_to_mask(&zName[1], 0);
+ if( !g.perm.Admin ) groupMask &= ~(CONFIGSET_USER|CONFIGSET_SCRIBER);
+ if( !g.perm.RdAddr ) groupMask &= ~CONFIGSET_ADDR;
+ configure_send_group(xfer.pOut, groupMask, 0);
+ }
+ }
+ }else
+
+ /* config NAME SIZE \n CONTENT
+ **
+ ** Client has sent a configuration value to the server.
+ ** This is only permitted for high-privilege users.
+ */
+ if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
+ && blob_is_int(&xfer.aToken[2], &size) ){
+ const char *zName = blob_str(&xfer.aToken[1]);
+ Blob content;
+ if( size<0 ){
+ xfer_fatal_error("invalid config record");
+ db_rollback_transaction();
+ return;
+ }
+ blob_zero(&content);
+ blob_extract(xfer.pIn, size, &content);
+ if( !g.perm.Admin ){
+ cgi_reset_content();
+ @ error not\sauthorized\sto\spush\sconfiguration%s(whyNotAuth())
+ nErr++;
+ break;
+ }
+ configure_receive(zName, &content, CONFIGSET_ALL);
+ blob_reset(&content);
+ blob_seek(xfer.pIn, 1, BLOB_SEEK_CUR);
+ }else
+
+
+ /* cookie TEXT
+ **
+ ** A cookie contains an arbitrary-length argument that is server-defined.
+ ** The argument must be encoded so as not to contain any whitespace.
+ ** The server can optionally send a cookie to the client. The client
+ ** might then return the same cookie back to the server on its next
+ ** communication. The cookie might record information that helps
+ ** the server optimize a push or pull.
+ **
+ ** The client is not required to return a cookie. So the server
+ ** must not depend on the cookie. The cookie should be an optimization
+ ** only. The client might also send a cookie that came from a different
+ ** server. So the server must be prepared to distinguish its own cookie
+ ** from cookies originating from other servers. The client might send
+ ** back several different cookies to the server. The server should be
+ ** prepared to sift through the cookies and pick the one that it wants.
+ */
+ if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
+ /* Process the cookie */
+ }else
+
+
+ /* private
+ **
+ ** The client card indicates that the next "file" or "cfile" will contain
+ ** private content.
+ */
+ if( blob_eq(&xfer.aToken[0], "private") ){
+ if( !g.perm.Private ){
+ server_private_xfer_not_authorized();
+ }else{
+ xfer.nextIsPrivate = 1;
+ }
+ }else
+
+ /* pragma NAME VALUE...
+ **
+ ** The client issues pragmas to try to influence the behavior of the
+ ** server. These are requests only. Unknown pragmas are silently
+ ** ignored.
+ */
+ if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
+
+ /* pragma send-private
+ **
+ ** The client is requesting private artifacts.
+ **
+ ** If the user has the "x" privilege (which must be set explicitly -
+ ** it is not automatic with "a" or "s") then this pragma causes
+ ** private information to be pulled in addition to public records.
+ */
+ if( blob_eq(&xfer.aToken[1], "send-private") ){
+ login_check_credentials();
+ if( !g.perm.Private ){
+ server_private_xfer_not_authorized();
+ }else{
+ xfer.syncPrivate = 1;
+ }
+ }else
+
+ /* pragma send-catalog
+ **
+ ** The client wants to see igot cards for all known artifacts.
+ ** This is used as part of "sync --verily" to help ensure that
+ ** no artifacts have been missed on prior syncs.
+ */
+ if( blob_eq(&xfer.aToken[1], "send-catalog") ){
+ xfer.resync = 0x7fffffff;
+ }else
+
+ /* pragma client-version VERSION ?DATE? ?TIME?
+ **
+ ** The client announces to the server what version of Fossil it
+ ** is running. The DATE and TIME are a pure numeric ISO8601 time
+ ** for the specific check-in of the client.
+ */
+ if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
+ xfer.remoteVersion = g.syncInfo.remoteVersion =
+ atoi(blob_str(&xfer.aToken[2]));
+ if( xfer.nToken>=5 ){
+ xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
+ xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
+ @ pragma server-version %d(RELEASE_VERSION_NUMBER) \
+ @ %d(MANIFEST_NUMERIC_DATE) %d(MANIFEST_NUMERIC_TIME)
+ }
+ xfer_xflc_check( xfer.remoteVersion, xfer.remoteDate,
+ xfer.remoteTime, 0x04 );
+ }else
+
+ /* pragma uv-hash HASH
+ **
+ ** The client wants to make sure that unversioned files are all synced.
+ ** If the HASH does not match, send a complete catalog of
+ ** "uvigot" cards.
+ */
+ if( blob_eq(&xfer.aToken[1], "uv-hash")
+ && blob_is_hname(&xfer.aToken[2])
+ ){
+ if( !uvCatalogSent
+ && g.perm.Read
+ && !blob_eq_str(&xfer.aToken[2], unversioned_content_hash(0),-1)
+ ){
+ if( g.perm.WrUnver ){
+ @ pragma uv-push-ok
+ }else if( g.perm.Read ){
+ @ pragma uv-pull-only
+ }
+ send_unversioned_catalog(&xfer);
+ }
+ uvCatalogSent = 1;
+ }else
+
+ /* pragma ci-lock CHECKIN-HASH CLIENT-ID
+ **
+ ** The client wants to make non-branch commit against the check-in
+ ** identified by CHECKIN-HASH. The server will remember this and
+ ** subsequent ci-lock requests from different clients will generate
+ ** a ci-lock-fail pragma in the reply.
+ */
+ if( blob_eq(&xfer.aToken[1], "ci-lock")
+ && xfer.nToken==4
+ && blob_is_hname(&xfer.aToken[2])
+ ){
+ Stmt q;
+ sqlite3_int64 iNow = time(0);
+ sqlite3_int64 maxAge = db_get_int("lock-timeout",60);
+ int seenFault = 0;
+ db_prepare(&q,
+ "SELECT value->>'login',"
+ " mtime,"
+ " value->>'clientid',"
+ " (SELECT rid FROM blob WHERE uuid=substr(name,9)),"
+ " name"
+ " FROM config"
+ " WHERE name GLOB 'ci-lock-*'"
+ " AND json_valid(value)"
+ );
+ while( db_step(&q)==SQLITE_ROW ){
+ int x = db_column_int(&q,3);
+ const char *zName = db_column_text(&q,4);
+ if( db_column_int64(&q,1)<=iNow-maxAge || !is_a_leaf(x) ){
+ /* check-in locks expire after maxAge seconds, or when the
+ ** check-in is no longer a leaf */
+ db_unprotect(PROTECT_CONFIG);
+ db_multi_exec("DELETE FROM config WHERE name=%Q", zName);
+ db_protect_pop();
+ continue;
+ }
+ if( fossil_strcmp(zName+8, blob_str(&xfer.aToken[2]))==0 ){
+ const char *zClientId = db_column_text(&q, 2);
+ const char *zLogin = db_column_text(&q,0);
+ sqlite3_int64 mtime = db_column_int64(&q, 1);
+ if( fossil_strcmp(zClientId, blob_str(&xfer.aToken[3]))!=0 ){
+ @ pragma ci-lock-fail %F(zLogin) %lld(mtime)
+ }
+ seenFault = 1;
+ }
+ }
+ db_finalize(&q);
+ if( !seenFault ){
+ db_unprotect(PROTECT_CONFIG);
+ db_multi_exec(
+ "REPLACE INTO config(name,value,mtime)"
+ "VALUES('ci-lock-%q',json_object('login',%Q,'clientid',%Q),now())",
+ blob_str(&xfer.aToken[2]), g.zLogin,
+ blob_str(&xfer.aToken[3])
+ );
+ db_protect_pop();
+ }
+ if( db_get_boolean("forbid-delta-manifests",0) ){
+ @ pragma avoid-delta-manifests
+ }
+ }else
+
+ /* pragma ci-unlock CLIENT-ID
+ **
+ ** Remove any locks previously held by CLIENT-ID. Clients send this
+ ** pragma with their own ID whenever they know that they no longer
+ ** have any commits pending.
+ */
+ if( blob_eq(&xfer.aToken[1], "ci-unlock")
+ && xfer.nToken==3
+ && blob_is_hname(&xfer.aToken[2])
+ ){
+ db_unprotect(PROTECT_CONFIG);
+ db_multi_exec(
+ "DELETE FROM config"
+ " WHERE name GLOB 'ci-lock-*'"
+ " AND (NOT json_valid(value) OR value->>'clientid'==%Q)",
+ blob_str(&xfer.aToken[2])
+ );
+ db_protect_pop();
+ }else
+
+ /* pragma client-url URL
+ **
+ ** This pragma is an informational notification to the server that
+ ** their relationship could, in theory, be inverted by having the
+ ** server call the client at URL.
+ */
+ if( blob_eq(&xfer.aToken[1], "client-url")
+ && xfer.nToken==3
+ && g.perm.Write
+ ){
+ xfer_syncwith(blob_str(&xfer.aToken[2]), 1);
+ }else
+
+ /* pragma req-links
+ **
+ ** The client sends this message to the server to ask the server
+ ** to tell it about alternative repositories in the reply.
+ */
+ if( blob_eq(&xfer.aToken[1], "req-links") ){
+ bSendLinks = 1;
+ }else
+
+ /* pragma req-clusters
+ **
+ ** This pragma requests that the server send igot cards for every
+ ** cluster artifact that it knows about.
+ */
+ if( blob_eq(&xfer.aToken[1], "req-clusters") ){
+ send_all_clusters(&xfer);
+ }
+
+ }else
+
+ /* Unknown message
+ */
+ {
+ cgi_reset_content();
+ @ error bad\scommand:\s%F(blob_str(&xfer.line))
+ }
+ blobarray_reset(xfer.aToken, xfer.nToken);
+ blob_reset(&xfer.line);
+ }
+ if( isPush ){
+ if( rc==TH_OK ){
+ rc = xfer_run_script(zScript, zUuidList, 1);
+ if( rc==TH_ERROR ){
+ cgi_reset_content();
+ @ error push\sscript\sfailed:\s%F(g.zErrMsg)
+ nErr++;
+ }
+ }
+ request_phantoms(&xfer, 500);
+ }
+ if( zUuidList ){
+ Th_Free(g.interp, zUuidList);
+ }
+ if( isClone && nGimme==0 ){
+ /* The initial "clone" message from client to server contains no
+ ** "gimme" cards. On that initial message, send the client an "igot"
+ ** card for every artifact currently in the repository. This will
+ ** cause the client to create phantoms for all artifacts, which will
+ ** in turn make sure that the entire repository is sent efficiently
+ ** and expeditiously.
+ */
+ send_all(&xfer);
+ if( xfer.syncPrivate ) send_private(&xfer);
+ }else if( isPull ){
+ create_cluster();
+ send_unclustered(&xfer);
+ if( xfer.syncPrivate ) send_private(&xfer);
+ }
+ hook_expecting_more_artifacts(xfer.nGimmeSent?60:0);
+ db_multi_exec("DROP TABLE onremote; DROP TABLE unk;");
+ manifest_crosslink_end(MC_PERMIT_HOOKS);
+
+ /* Send URLs for alternative repositories for the same project,
+ ** if requested by the client. */
+ if( bSendLinks && g.zBaseURL ){
+ Stmt q;
+ db_prepare(&q,
+ "WITH remote(mtime, url, arg) AS (\n"
+ " SELECT mtime, substr(name,10), '{}' FROM config\n"
+ " WHERE name GLOB 'syncwith:http*'\n"
+ " UNION ALL\n"
+ " SELECT mtime, substr(name,10), '{}' FROM config\n"
+ " WHERE name GLOB 'syncfrom:http*'\n"
+ " UNION ALL\n"
+ " SELECT mtime, substr(name,9), '{\"type\":\"git\"}' FROM config\n"
+ " WHERE name GLOB 'gitpush:*'\n"
+ ")\n"
+ "SELECT url, json_insert(arg,'$.src',%Q), max(mtime)\n"
+ " FROM remote WHERE mtime>unixepoch('now','-1 month')\n"
+ " GROUP BY url;",
+ g.zBaseURL
+ );
+ while( db_step(&q)==SQLITE_ROW ){
+ UrlData x;
+ const char *zUrl = db_column_text(&q, 0);
+ const char *zArg = db_column_text(&q, 1);
+ i64 iMtime = db_column_int64(&q, 2);
+ memset(&x, 0, sizeof(x));
+ url_parse_local(zUrl, URL_OMIT_USER, &x);
+ if( x.name!=0 && sqlite3_strlike("%localhost%", x.name, 0)!=0 ){
+ @ pragma link %F(x.canonical) %F(zArg) %lld(iMtime)
+ }
+ url_unparse(&x);
+ }
+ db_finalize(&q);
+ }
+
+ /* Send the server timestamp last, in case prior processing happened
+ ** to use up a significant fraction of our time window.
+ */
+ zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
+ @ # timestamp %s(zNow) errors %d(nErr)
+ fossil_free(zNow);
+
+ db_commit_transaction();
+ configure_rebuild();
+}
+
+/*
+** COMMAND: test-xfer
+**
+** Usage: %fossil test-xfer ?OPTIONS? XFERFILE
+**
+** Pass the sync-protocol input file XFERFILE into the server-side sync
+** protocol handler. Generate a reply on standard output.
+**
+** This command was original created to help debug the server side of
+** sync messages. The XFERFILE is the uncompressed content of an
+** "xfer" HTTP request from client to server. This command interprets
+** that message and generates the content of an HTTP reply (without any
+** encoding and without the HTTP reply headers) and writes that reply
+** on standard output.
+**
+** One possible usages scenario is to capture some XFERFILE examples
+** using a command like:
+**
+** fossil push http://bogus/ --httptrace
+**
+** The complete HTTP requests are stored in files named "http-request-N.txt".
+** Find one of those requests, remove the HTTP header, and make other edits
+** as necessary to generate an appropriate XFERFILE test case. Then run:
+**
+** fossil test-xfer xferfile.txt
+**
+** Options:
+** --host HOSTNAME Supply a server hostname used to populate
+** g.zBaseURL and similar.
+*/
+void cmd_test_xfer(void){
+ const char *zHost;
+ db_find_and_open_repository(0,0);
+ zHost = find_option("host",0,1);
+ verify_all_options();
+ if( g.argc!=2 && g.argc!=3 ){
+ usage("?MESSAGEFILE?");
+ }
+ if( zHost==0 ) zHost = "localhost:8080";
+ g.zBaseURL = mprintf("http://%s", zHost);
+ g.zHttpsURL = mprintf("https://%s", zHost);
+ g.zTop = mprintf("");
+ blob_zero(&g.cgiIn);
+ blob_read_from_file(&g.cgiIn, g.argc==2 ? "-" : g.argv[2], ExtFILE);
+ disableLogin = 1;
+ page_xfer();
+ fossil_print("%s", cgi_extract_content());
+}
+
+/*
+** Format strings for progress reporting.
+*/
+static const char zLabelFormat[] = "%-10s %10s %10s %10s %10s\n";
+static const char zValueFormat[] = "\r%-10s %10d %10d %10d %10d\n";
+static const char zBriefFormat[] =
+ "Round-trips: %d Artifacts sent: %d received: %d\r";
+
+#if INTERFACE
+/*
+** Flag options for controlling client_sync()
+*/
+#define SYNC_PUSH 0x00001 /* push content client to server */
+#define SYNC_PULL 0x00002 /* pull content server to client */
+#define SYNC_CLONE 0x00004 /* clone the repository */
+#define SYNC_PRIVATE 0x00008 /* Also transfer private content */
+#define SYNC_VERBOSE 0x00010 /* Extra diagnostics */
+#define SYNC_RESYNC 0x00020 /* --verily */
+#define SYNC_FROMPARENT 0x00040 /* Pull from the parent project */
+#define SYNC_UNVERSIONED 0x00100 /* Sync unversioned content */
+#define SYNC_UV_REVERT 0x00200 /* Copy server unversioned to client */
+#define SYNC_UV_TRACE 0x00400 /* Describe UV activities */
+#define SYNC_UV_DRYRUN 0x00800 /* Do not actually exchange files */
+#define SYNC_IFABLE 0x01000 /* Inability to sync is not fatal */
+#define SYNC_CKIN_LOCK 0x02000 /* Lock the current check-in */
+#define SYNC_NOHTTPCOMPRESS 0x04000 /* Do not compression HTTP messages */
+#define SYNC_ALLURL 0x08000 /* The --all flag - sync to all URLs */
+#define SYNC_SHARE_LINKS 0x10000 /* Request alternate repo links */
+#define SYNC_XVERBOSE 0x20000 /* Extra verbose. Network traffic */
+#define SYNC_PING 0x40000 /* Verify server is alive */
+#define SYNC_QUIET 0x80000 /* No output */
+#endif
+
+/*
+** Floating-point absolute value
+*/
+static double fossil_fabs(double x){
+ return x>0.0 ? x : -x;
+}
+
+/*
+** Sync to the host identified in g.url.name and g.url.path. This
+** routine is called by the client.
+**
+** Records are pushed to the server if pushFlag is true. Records
+** are pulled if pullFlag is true. A full sync occurs if both are
+** true.
+*/
+int client_sync(
+ unsigned syncFlags, /* Mask of SYNC_* flags */
+ unsigned configRcvMask, /* Receive these configuration items */
+ unsigned configSendMask, /* Send these configuration items */
+ const char *zAltPCode, /* Alternative project code (usually NULL) */
+ int *pnRcvd /* Set to # received artifacts, if not NULL */
+){
+ int go = 1; /* Loop until zero */
+ int nCardSent = 0; /* Number of cards sent */
+ int nCardRcvd = 0; /* Number of cards received */
+ int nCycle = 0; /* Number of round trips to the server */
+ int size; /* Size of a config value or uvfile */
+ int origConfigRcvMask; /* Original value of configRcvMask */
+ int nFileRecv = 0; /* Number of files received */
+ int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
+ const char *zCookie; /* Server cookie */
+ i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
+ i64 nSent, nRcvd; /* Bytes sent and received (after compression) */
+ int cloneSeqno = 1; /* Sequence number for clones */
+ Blob send; /* Text we are sending to the server */
+ Blob recv; /* Reply we got back from the server */
+ Xfer xfer; /* Transfer data */
+ int pctDone; /* Percentage done with a message */
+ int lastPctDone = -1; /* Last displayed pctDone */
+ double rArrivalTime; /* Time at which a message arrived */
+ const char *zSCode = db_get("server-code", "x");
+ const char *zPCode = db_get("project-code", 0);
+ int nErr = 0; /* Number of errors */
+ int nRoundtrip= 0; /* Number of HTTP requests */
+ int nArtifactSent = 0; /* Total artifacts sent */
+ int nArtifactRcvd = 0; /* Total artifacts received */
+ int nPriorArtifact = 0; /* Artifacts received on prior round-trips */
+ const char *zOpType = 0;/* Push, Pull, Sync, Clone */
+ double rSkew = 0.0; /* Maximum time skew */
+ int uvHashSent = 0; /* The "pragma uv-hash" message has been sent */
+ int uvDoPush = 0; /* Generate uvfile messages to send to server */
+ int uvPullOnly = 0; /* 1: pull-only. 2: pull-only warning issued */
+ int nUvGimmeSent = 0; /* Number of uvgimme cards sent on this cycle */
+ int nUvFileRcvd = 0; /* Number of uvfile cards received on this cycle */
+ int nGimmeRcvd = 0; /* Number of gimme cards recevied on the prev cycle */
+ sqlite3_int64 mtime; /* Modification time on a UV file */
+ int autopushFailed = 0; /* Autopush following commit failed if true */
+ const char *zCkinLock; /* Name of check-in to lock. NULL for none */
+ const char *zClientId; /* A unique identifier for this check-out */
+ unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */
+ const int bOutIsTty = fossil_isatty(fossil_fileno(stdout));
+
+ if( pnRcvd ) *pnRcvd = 0;
+ if( db_get_boolean("dont-push", 0) ) syncFlags &= ~SYNC_PUSH;
+ if( (syncFlags & (SYNC_PUSH|SYNC_PULL|SYNC_CLONE|SYNC_UNVERSIONED|SYNC_PING))
+ ==0
+ && configRcvMask==0
+ && configSendMask==0
+ ){
+ return 0; /* Nothing to do */
+ }
+
+ /* Compute an appropriate project code. zPCode is the project code
+ ** for the local repository. zAltPCode will usually be NULL, but might
+ ** also be an alternative project code to expect on the server. When
+ ** zAltPCode is not NULL, that means we are doing a cross-project import -
+ ** in other words, reading content from one project into a different
+ ** project.
+ */
+ if( syncFlags & SYNC_FROMPARENT ){
+ const char *zPX;
+ configRcvMask = 0;
+ configSendMask = 0;
+ syncFlags &= ~(SYNC_PUSH);
+ zPX = db_get("parent-project-code", 0);
+ if( zPX==0 || db_get("parent-project-name",0)==0 ){
+ fossil_fatal("there is no parent project: set the 'parent-project-code'"
+ " and 'parent-project-name' config parameters in order"
+ " to pull from a parent project");
+ }
+ if( zPX ){
+ zAltPCode = zPX;
+ }
+ }
+ if( zAltPCode!=0 && zPCode!=0 && sqlite3_stricmp(zPCode, zAltPCode)==0 ){
+ zAltPCode = 0;
+ }
+
+ transport_stats(0, 0, 1);
+ socket_global_init();
+ memset(&xfer, 0, sizeof(xfer));
+ xfer.pIn = &recv;
+ xfer.pOut = &send;
+ xfer.mxSend = db_get_int("max-upload", 250000);
+ xfer.maxTime = -1;
+ xfer.remoteVersion = RELEASE_VERSION_NUMBER;
+ if( syncFlags & SYNC_PRIVATE ){
+ g.perm.Private = 1;
+ xfer.syncPrivate = 1;
+ }
+
+ blobarray_zero(xfer.aToken, count(xfer.aToken));
+ blob_zero(&send);
+ blob_zero(&recv);
+ blob_zero(&xfer.err);
+ blob_zero(&xfer.line);
+ origConfigRcvMask = 0;
+ nUncSent = nUncRcvd = 0;
+
+ /* Send the send-private pragma if we are trying to sync private data */
+ if( syncFlags & SYNC_PRIVATE ){
+ blob_append(&send, "pragma send-private\n", -1);
+ }
+
+ /* Figure out which check-in to lock */
+ if( syncFlags & SYNC_CKIN_LOCK ){
+ int vid = db_lget_int("checkout",0);
+ zCkinLock = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", vid);
+ }else{
+ zCkinLock = 0;
+ }
+ zClientId = g.localOpen ? db_lget("client-id", 0) : 0;
+
+ /* When syncing unversioned files, create a TEMP table in which to store
+ ** the names of files that need to be sent from client to server.
+ **
+ ** The initial assumption is that all unversioned files need to be sent
+ ** to the other side. But "uvigot" cards received back from the remote
+ ** side will normally cause many of these entries to be removed since they
+ ** do not really need to be sent.
+ */
+ if( (syncFlags & (SYNC_UNVERSIONED|SYNC_CLONE))!=0 ){
+ unversioned_schema();
+ db_multi_exec(
+ "CREATE TEMP TABLE IF NOT EXISTS uv_tosend("
+ " name TEXT PRIMARY KEY," /* Name of file to send client->server */
+ " mtimeOnly BOOLEAN" /* True to only send mtime, not content */
+ ") WITHOUT ROWID;"
+ "REPLACE INTO uv_tosend(name,mtimeOnly)"
+ " SELECT name, 0 FROM unversioned WHERE hash IS NOT NULL;"
+ );
+ }
+
+ /*
+ ** The request from the client always begin with a clone, pull,
+ ** or push message.
+ */
+ blob_appendf(&send, "pragma client-version %d %d %d\n",
+ RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
+ MANIFEST_NUMERIC_TIME);
+ if( syncFlags & SYNC_CLONE ){
+ blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
+ syncFlags &= ~(SYNC_PUSH|SYNC_PULL);
+ nCardSent++;
+ /* TBD: Request all transferable configuration values */
+ content_enable_dephantomize(0);
+ zOpType = "Clone";
+ }else if( syncFlags & SYNC_PULL ){
+ blob_appendf(&send, "pull %s %s\n", zSCode,
+ zAltPCode ? zAltPCode : zPCode);
+ nCardSent++;
+ zOpType = (syncFlags & SYNC_PUSH)?"Sync":"Pull";
+ if( (syncFlags & SYNC_RESYNC)!=0 && nCycle<2 ){
+ blob_appendf(&send, "pragma send-catalog\n");
+ nCardSent++;
+ }
+ }
+ if( syncFlags & SYNC_PUSH ){
+ blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
+ nCardSent++;
+ if( (syncFlags & SYNC_PULL)==0 ) zOpType = "Push";
+ if( (syncFlags & SYNC_RESYNC)!=0 ) xfer.resync = 0x7fffffff;
+ }
+ if( syncFlags & SYNC_VERBOSE ){
+ fossil_print(zLabelFormat /*works-like:"%s%s%s%s%d"*/,
+ "", "Bytes", "Cards", "Artifacts", "Deltas");
+ }
+
+ /* Send the client-url pragma on the first cycle if the client has
+ ** a known public url.
+ */
+ if( zAltPCode==0 ){
+ const char *zSelfUrl = public_url();
+ if( zSelfUrl ){
+ blob_appendf(&send, "pragma client-url %s\n", zSelfUrl);
+ }
+ }
+
+ /* Request URLs of alternative repositories
+ */
+ if( zAltPCode==0 && (syncFlags & SYNC_SHARE_LINKS)!=0 ){
+ blob_appendf(&send, "pragma req-links\n");
+ }
+
+ while( go ){
+ int newPhantom = 0;
+ char *zRandomness;
+ db_begin_transaction();
+ db_record_repository_filename(0);
+ db_multi_exec(
+ "CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
+ "CREATE TEMP TABLE unk(uuid TEXT PRIMARY KEY) WITHOUT ROWID;"
+ );
+ manifest_crosslink_begin();
+
+
+ /* Client sends the most recently received cookie back to the server.
+ ** Let the server figure out if this is a cookie that it cares about.
+ */
+ zCookie = db_get("cookie", 0);
+ if( zCookie ){
+ blob_appendf(&send, "cookie %s\n", zCookie);
+ }
+
+ /* Client sends gimme cards for phantoms
+ */
+ if( (syncFlags & SYNC_PULL)!=0
+ || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
+ ){
+ request_phantoms(&xfer, mxPhantomReq);
+ if( xfer.nGimmeSent>0 && nCycle==2 && (syncFlags & SYNC_PULL)!=0 ){
+ blob_appendf(&send, "pragma req-clusters\n");
+ }
+ }
+ if( syncFlags & SYNC_PUSH ){
+ send_unsent(&xfer);
+ nCardSent += send_unclustered(&xfer);
+ if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
+ if( nGimmeRcvd>0 && nCycle==2 ){
+ send_all_clusters(&xfer);
+ }
+ }
+
+ /* Client sends configuration parameter requests. On a clone, delay sending
+ ** this until the second cycle since the login card might fail on
+ ** the first cycle.
+ */
+ if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 || nCycle>0) ){
+ const char *zName;
+ if( zOpType==0 ) zOpType = "Pull";
+ zName = configure_first_name(configRcvMask);
+ while( zName ){
+ blob_appendf(&send, "reqconfig %s\n", zName);
+ zName = configure_next_name(configRcvMask);
+ nCardSent++;
+ }
+ origConfigRcvMask = configRcvMask;
+ configRcvMask = 0;
+ }
+
+ /* Client sends a request to sync unversioned files.
+ ** On a clone, delay sending this until the second cycle since
+ ** the login card might fail on the first cycle.
+ */
+ if( (syncFlags & SYNC_UNVERSIONED)!=0
+ && ((syncFlags & SYNC_CLONE)==0 || nCycle>0)
+ && !uvHashSent
+ ){
+ blob_appendf(&send, "pragma uv-hash %s\n", unversioned_content_hash(0));
+ nCardSent++;
+ uvHashSent = 1;
+ }
+
+ /* On a "fossil config push", the client send configuration parameters
+ ** being pushed up to the server */
+ if( configSendMask ){
+ if( zOpType==0 ) zOpType = "Push";
+ nCardSent += configure_send_group(xfer.pOut, configSendMask, 0);
+ configSendMask = 0;
+ }
+
+ /* Send unversioned files present here on the client but missing or
+ ** obsolete on the server.
+ **
+ ** Or, if the SYNC_UV_REVERT flag is set, delete the local unversioned
+ ** files that do not exist on the server.
+ **
+ ** This happens on the second exchange, since we do not know what files
+ ** need to be sent until after the uvigot cards from the first exchange
+ ** have been processed.
+ */
+ if( uvDoPush ){
+ assert( (syncFlags & SYNC_UNVERSIONED)!=0 );
+ if( syncFlags & SYNC_UV_DRYRUN ){
+ uvDoPush = 0;
+ }else if( syncFlags & SYNC_UV_REVERT ){
+ db_multi_exec(
+ "DELETE FROM unversioned"
+ " WHERE name IN (SELECT name FROM uv_tosend);"
+ "DELETE FROM uv_tosend;"
+ );
+ uvDoPush = 0;
+ }else{
+ Stmt uvq;
+ int rc = SQLITE_OK;
+ db_prepare(&uvq, "SELECT name, mtimeOnly FROM uv_tosend");
+ while( (rc = db_step(&uvq))==SQLITE_ROW ){
+ const char *zName = db_column_text(&uvq, 0);
+ send_unversioned_file(&xfer, zName, db_column_int(&uvq,1));
+ nCardSent++;
+ nArtifactSent++;
+ db_multi_exec("DELETE FROM uv_tosend WHERE name=%Q", zName);
+ if( syncFlags & SYNC_VERBOSE ){
+ fossil_print("\rUnversioned-file sent: %s\n", zName);
+ }
+ if( (int)blob_size(xfer.pOut)>xfer.mxSend ) break;
+ }
+ db_finalize(&uvq);
+ if( rc==SQLITE_DONE ) uvDoPush = 0;
+ }
+ }
+
+ /* Lock the current check-out */
+ if( zCkinLock ){
+ if( zClientId==0 ){
+ zClientId = db_text(0, "SELECT lower(hex(randomblob(20)))");
+ db_lset("client-id", zClientId);
+ }
+ blob_appendf(&send, "pragma ci-lock %s %s\n", zCkinLock, zClientId);
+ zCkinLock = 0;
+ }else if( zClientId ){
+ blob_appendf(&send, "pragma ci-unlock %s\n", zClientId);
+ }
+ /* Append randomness to the end of the uplink message. This makes all
+ ** messages unique so that the login-card nonce will always
+ ** be unique.
+ */
+ zRandomness = db_text(0, "SELECT hex(randomblob(20))");
+ blob_appendf(&send, "# %s\n", zRandomness);
+ fossil_free(zRandomness);
+
+ if( (syncFlags & SYNC_VERBOSE)!=0
+ && (syncFlags & SYNC_XVERBOSE)==0
+ ){
+ fossil_print("waiting for server...");
+ }
+ fflush(stdout);
+ /* Exchange messages with the server */
+ if( (syncFlags & SYNC_CLONE)!=0 && nCycle==0 ){
+ /* Do not send a login card on the first round-trip of a clone */
+ mHttpFlags = 0;
+ }else{
+ mHttpFlags = HTTP_USE_LOGIN;
+ }
+ if( syncFlags & SYNC_NOHTTPCOMPRESS ){
+ mHttpFlags |= HTTP_NOCOMPRESS;
+ }
+ if( syncFlags & SYNC_XVERBOSE ){
+ mHttpFlags |= HTTP_VERBOSE;
+ }
+ if( syncFlags & SYNC_QUIET ){
+ mHttpFlags |= HTTP_QUIET;
+ }
+
+ /* Do the round-trip to the server */
+ if( http_exchange(&send, &recv, mHttpFlags, MAX_REDIRECTS, 0) ){
+ nErr++;
+ go = 2;
+ break;
+ }
+
+ /* Remember the URL of the sync target in the config file on the
+ ** first successful round-trip */
+ if( nCycle==0 && db_is_writeable("repository") ){
+ xfer_syncwith(g.url.canonical, 0);
+ }
+
+ /* Output current stats */
+ nRoundtrip++;
+ nArtifactSent += xfer.nFileSent + xfer.nDeltaSent;
+ if( syncFlags & SYNC_VERBOSE ){
+ fossil_print(zValueFormat /*works-like:"%s%d%d%d%d"*/, "Sent:",
+ blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
+ xfer.nFileSent, xfer.nDeltaSent);
+ }else if( syncFlags & SYNC_QUIET ){
+ /* No-op */
+ }else{
+ if( bOutIsTty!=0 ){
+ fossil_print(zBriefFormat /*works-like:"%d%d%d"*/,
+ nRoundtrip, nArtifactSent, nArtifactRcvd);
+ }
+ }
+ nCardSent = 0;
+ nCardRcvd = 0;
+ xfer.nFileSent = 0;
+ xfer.nDeltaSent = 0;
+ xfer.nGimmeSent = 0;
+ xfer.nIGotSent = 0;
+ xfer.nPrivIGot = 0;
+
+ lastPctDone = -1;
+ nUncSent += blob_size(&send);
+ blob_reset(&send);
+ blob_appendf(&send, "pragma client-version %d %d %d\n",
+ RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
+ MANIFEST_NUMERIC_TIME);
+ rArrivalTime = db_double(0.0, "SELECT julianday('now')");
+
+ /* Send the send-private pragma if we are trying to sync private data */
+ if( syncFlags & SYNC_PRIVATE ){
+ blob_append(&send, "pragma send-private\n", -1);
+ }
+
+ /* Begin constructing the next message (which might never be
+ ** sent) by beginning with the pull or push cards
+ */
+ if( syncFlags & SYNC_PULL ){
+ blob_appendf(&send, "pull %s %s\n", zSCode,
+ zAltPCode ? zAltPCode : zPCode);
+ nCardSent++;
+ }
+ if( syncFlags & SYNC_PUSH ){
+ blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
+ nCardSent++;
+ }
+ go = 0;
+ nUvGimmeSent = 0;
+ nUvFileRcvd = 0;
+ nGimmeRcvd = 0;
+ nPriorArtifact = nArtifactRcvd;
+
+ /* Process the reply that came back from the server */
+ while( blob_line(&recv, &xfer.line) ){
+ if( blob_buffer(&xfer.line)[0]=='#' ){
+ const char *zLine = blob_buffer(&xfer.line);
+ if( memcmp(zLine, "# timestamp ", 12)==0 ){
+ char zTime[20];
+ double rDiff;
+ sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
+ rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
+ zTime, rArrivalTime);
+ if( rDiff>9e98 || rDiff<-9e98 ) rDiff = 0.0;
+ if( rDiff*24.0*3600.0 >= -(blob_size(&recv)/5000.0 + 20) ){
+ rDiff = 0.0;
+ }
+ if( fossil_fabs(rDiff)>fossil_fabs(rSkew) ) rSkew = rDiff;
+ }
+ nCardRcvd++;
+ continue;
+ }
+ xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
+ nCardRcvd++;
+ if( (syncFlags & SYNC_VERBOSE)!=0 && recv.nUsed>0 ){
+ pctDone = (recv.iCursor*100)/recv.nUsed;
+ if( pctDone!=lastPctDone ){
+ fossil_print("\rprocessed: %d%% ", pctDone);
+ lastPctDone = pctDone;
+ fflush(stdout);
+ }
+ }
+
+ /* file HASH SIZE \n CONTENT
+ ** file HASH DELTASRC SIZE \n CONTENT
+ **
+ ** Client receives a file transmitted from the server.
+ */
+ if( blob_eq(&xfer.aToken[0],"file") ){
+ xfer_accept_file(&xfer, (syncFlags & SYNC_CLONE)!=0, 0, 0);
+ nArtifactRcvd++;
+ }else
+
+ /* cfile HASH USIZE CSIZE \n CONTENT
+ ** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
+ **
+ ** Client receives a compressed file transmitted from the server.
+ */
+ if( blob_eq(&xfer.aToken[0],"cfile") ){
+ xfer_accept_compressed_file(&xfer, 0, 0);
+ nArtifactRcvd++;
+ }else
+
+ /* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
+ **
+ ** Client accepts an unversioned file from the server.
+ */
+ if( blob_eq(&xfer.aToken[0], "uvfile") ){
+ xfer_accept_unversioned_file(&xfer, 1);
+ nArtifactRcvd++;
+ nUvFileRcvd++;
+ if( syncFlags & SYNC_VERBOSE ){
+ fossil_print("\rUnversioned-file received: %s\n",
+ blob_str(&xfer.aToken[1]));
+ }
+ }else
+
+ /* gimme HASH
+ **
+ ** Client receives an artifact request from the server.
+ ** If the file is a manifest, assume that the server will also want
+ ** to know all of the content artifacts associated with the manifest
+ ** and send those too.
+ */
+ if( blob_eq(&xfer.aToken[0], "gimme")
+ && xfer.nToken==2
+ && blob_is_hname(&xfer.aToken[1])
+ ){
+ remote_unk(&xfer.aToken[1]);
+ if( syncFlags & SYNC_PUSH ){
+ int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
+ if( rid ){
+ send_file(&xfer, rid, &xfer.aToken[1], 0);
+ nGimmeRcvd++;
+ }
+ }
+ }else
+
+ /* igot HASH ?PRIVATEFLAG?
+ **
+ ** Server announces that it has a particular file. If this is
+ ** not a file that we have and we are pulling, then create a
+ ** phantom to cause this file to be requested on the next cycle.
+ ** Always remember that the server has this file so that we do
+ ** not transmit it by accident.
+ **
+ ** If the PRIVATE argument exists and is 1, then the file is
+ ** private. Pretend it does not exists if we are not pulling
+ ** private files.
+ */
+ if( xfer.nToken>=2
+ && blob_eq(&xfer.aToken[0], "igot")
+ && blob_is_hname(&xfer.aToken[1])
+ ){
+ int rid;
+ int isPriv = xfer.nToken>=3 && blob_eq(&xfer.aToken[2],"1");
+ rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
+ if( rid>0 ){
+ if( isPriv ){
+ content_make_private(rid);
+ }else{
+ content_make_public(rid);
+ }
+ }else if( isPriv && !g.perm.Private ){
+ /* ignore private files */
+ }else if( (syncFlags & (SYNC_PULL|SYNC_CLONE))!=0 ){
+ rid = content_new(blob_str(&xfer.aToken[1]), isPriv);
+ if( rid ) newPhantom = 1;
+ }
+ remote_has(rid);
+ }else
+
+ /* uvigot NAME MTIME HASH SIZE
+ **
+ ** Server announces that it has a particular unversioned file. The
+ ** server will only send this card if the client had previously sent
+ ** a "pragma uv-hash" card with a hash that does not match.
+ **
+ ** If the identified file needs to be transferred, then setup for the
+ ** transfer. Generate a "uvgimme" card in the reply if the server
+ ** version is newer than the client. Generate a "uvfile" card if
+ ** the client version is newer than the server. If HASH is "-"
+ ** (indicating that the file has been deleted) and MTIME is newer,
+ ** then do the deletion.
+ */
+ if( xfer.nToken==5
+ && blob_eq(&xfer.aToken[0], "uvigot")
+ && blob_is_filename(&xfer.aToken[1])
+ && blob_is_int64(&xfer.aToken[2], &mtime)
+ && blob_is_int(&xfer.aToken[4], &size)
+ && (blob_eq(&xfer.aToken[3],"-") || blob_is_hname(&xfer.aToken[3]))
+ ){
+ const char *zName = blob_str(&xfer.aToken[1]);
+ const char *zHash = blob_str(&xfer.aToken[3]);
+ int iStatus;
+ if( mtime<0 || size<0 ){
+ xfer_fatal_error("invalid uvigot");
+ return ++nErr;
+ }
+ iStatus = unversioned_status(zName, mtime, zHash);
+ if( (syncFlags & SYNC_UV_REVERT)!=0 ){
+ if( iStatus==4 ) iStatus = 2;
+ if( iStatus==5 ) iStatus = 1;
+ }
+ if( syncFlags & (SYNC_UV_TRACE|SYNC_UV_DRYRUN) ){
+ const char *zMsg = 0;
+ switch( iStatus ){
+ case 0:
+ case 1: zMsg = "UV-PULL"; break;
+ case 2: zMsg = "UV-PULL-MTIME-ONLY"; break;
+ case 4: zMsg = "UV-PUSH-MTIME-ONLY"; break;
+ case 5: zMsg = "UV-PUSH"; break;
+ }
+ if( zMsg ) fossil_print("\r%s: %s\n", zMsg, zName);
+ if( syncFlags & SYNC_UV_DRYRUN ){
+ iStatus = 99; /* Prevent any changes or reply messages */
+ }
+ }
+ if( iStatus<=1 ){
+ if( zHash[0]!='-' ){
+ blob_appendf(xfer.pOut, "uvgimme %s\n", zName);
+ nCardSent++;
+ nUvGimmeSent++;
+ db_multi_exec("DELETE FROM unversioned WHERE name=%Q", zName);
+ }else if( iStatus==1 ){
+ db_multi_exec(
+ "UPDATE unversioned"
+ " SET mtime=%lld, hash=NULL, sz=0, encoding=0, content=NULL"
+ " WHERE name=%Q", mtime, zName
+ );
+ db_unset("uv-hash", 0);
+ }
+ }else if( iStatus==2 ){
+ db_multi_exec(
+ "UPDATE unversioned SET mtime=%lld WHERE name=%Q", mtime, zName
+ );
+ db_unset("uv-hash", 0);
+ }
+ if( iStatus>=4 && uvPullOnly==1 ){
+ fossil_warning(
+ "Warning: uv-pull-only \n"
+ " Unable to push unversioned content because you lack\n"
+ " sufficient permission on the server\n"
+ );
+ uvPullOnly = 2;
+ }
+ if( iStatus<=3 || uvPullOnly ){
+ db_multi_exec("DELETE FROM uv_tosend WHERE name=%Q", zName);
+ }else if( iStatus==4 ){
+ db_multi_exec("UPDATE uv_tosend SET mtimeOnly=1 WHERE name=%Q",zName);
+ }else if( iStatus==5 ){
+ db_multi_exec("REPLACE INTO uv_tosend(name,mtimeOnly) VALUES(%Q,0)",
+ zName);
+ }
+ }else
+
+ /* push SERVERCODE PRODUCTCODE
+ **
+ ** Should only happen in response to a clone. This message tells
+ ** the client what product code to use for the new database.
+ */
+ if( blob_eq(&xfer.aToken[0],"push")
+ && xfer.nToken==3
+ && (syncFlags & SYNC_CLONE)!=0
+ && blob_is_hname(&xfer.aToken[2])
+ ){
+ if( zPCode==0 ){
+ zPCode = mprintf("%b", &xfer.aToken[2]);
+ db_set("project-code", zPCode, 0);
+ }
+ if( cloneSeqno>0 ) blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
+ nCardSent++;
+ }else
+
+ /* config NAME SIZE \n CONTENT
+ **
+ ** Client receive a configuration value from the server.
+ **
+ ** The received configuration setting is silently ignored if it was
+ ** not requested by a prior "reqconfig" sent from client to server.
+ */
+ if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
+ && blob_is_int(&xfer.aToken[2], &size) ){
+ const char *zName = blob_str(&xfer.aToken[1]);
+ Blob content;
+ if( size<0 ){
+ xfer_fatal_error("invalid config record");
+ return ++nErr;
+ }
+ blob_zero(&content);
+ blob_extract(xfer.pIn, size, &content);
+ g.perm.Admin = g.perm.RdAddr = 1;
+ configure_receive(zName, &content, origConfigRcvMask);
+ nCardRcvd++;
+ nArtifactRcvd++;
+ blob_reset(&content);
+ blob_seek(xfer.pIn, 1, BLOB_SEEK_CUR);
+ }else
+
+
+ /* cookie TEXT
+ **
+ ** The client reserves a cookie from the server. The client
+ ** should remember this cookie and send it back to the server
+ ** in its next query.
+ **
+ ** Each cookie received overwrites the prior cookie from the
+ ** same server.
+ */
+ if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
+ db_set("cookie", blob_str(&xfer.aToken[1]), 0);
+ }else
+
+
+ /* private
+ **
+ ** The server tells the client that the next "file" or "cfile" will
+ ** contain private content.
+ */
+ if( blob_eq(&xfer.aToken[0], "private") ){
+ xfer.nextIsPrivate = 1;
+ }else
+
+
+ /* clone_seqno N
+ **
+ ** When doing a clone, the server tries to send all of its artifacts
+ ** in sequence. This card indicates the sequence number of the next
+ ** blob that needs to be sent. If N<=0 that indicates that all blobs
+ ** have been sent.
+ */
+ if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
+ blob_is_int(&xfer.aToken[1], &cloneSeqno);
+ if( cloneSeqno<0 ){
+ xfer_fatal_error("invalid clone_seqno");
+ return ++nErr;
+ }
+ }else
+
+ /* message MESSAGE
+ **
+ ** A message is received from the server. Print it.
+ ** Similar to "error" but does not stop processing.
+ */
+ if( blob_eq(&xfer.aToken[0],"message") && xfer.nToken==2 ){
+ char *zMsg = blob_terminate(&xfer.aToken[1]);
+ defossilize(zMsg);
+ if( (syncFlags & SYNC_PUSH) && zMsg
+ && sqlite3_strglob("pull only *", zMsg)==0 ){
+ syncFlags &= ~SYNC_PUSH;
+ zMsg = 0;
+ }
+ if( zMsg && zMsg[0] ){
+ fossil_force_newline();
+ fossil_print("Server says: %s\n", zMsg);
+ }
+ }else
+
+ /* pragma NAME VALUE...
+ **
+ ** The server can send pragmas to try to convey meta-information to
+ ** the client. These are informational only. Unknown pragmas are
+ ** silently ignored.
+ */
+ if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
+ /* pragma server-version VERSION ?DATE? ?TIME?
+ **
+ ** The server announces to the server what version of Fossil it
+ ** is running. The DATE and TIME are a pure numeric ISO8601 time
+ ** for the specific check-in of the client.
+ */
+ if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "server-version") ){
+ xfer.remoteVersion = g.syncInfo.remoteVersion =
+ atoi(blob_str(&xfer.aToken[2]));
+ if( xfer.nToken>=5 ){
+ xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
+ xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
+ }
+ xfer_xflc_check( xfer.remoteVersion, xfer.remoteDate,
+ xfer.remoteTime, 0x08 );
+ }
+
+ /* pragma uv-pull-only
+ ** pragma uv-push-ok
+ **
+ ** If the server is unwilling to accept new unversioned content (because
+ ** this client lacks the necessary permissions) then it sends a
+ ** "uv-pull-only" pragma so that the client will know not to waste
+ ** bandwidth trying to upload unversioned content. If the server
+ ** does accept new unversioned content, it sends "uv-push-ok".
+ */
+ else if( syncFlags & SYNC_UNVERSIONED ){
+ if( blob_eq(&xfer.aToken[1], "uv-pull-only") ){
+ uvPullOnly = 1;
+ if( syncFlags & SYNC_UV_REVERT ) uvDoPush = 1;
+ }else if( blob_eq(&xfer.aToken[1], "uv-push-ok") ){
+ uvDoPush = 1;
+ }
+ }
+
+ /* pragma ci-lock-fail USER-HOLDING-LOCK LOCK-TIME
+ **
+ ** The server generates this message when a "pragma ci-lock"
+ ** is attempted on a check-in for which there is an existing
+ ** lock. USER-HOLDING-LOCK is the name of the user who originated
+ ** the lock, and LOCK-TIME is the timestamp (seconds since 1970)
+ ** when the lock was taken.
+ */
+ else if( blob_eq(&xfer.aToken[1], "ci-lock-fail") && xfer.nToken==4 ){
+ char *zUser = blob_terminate(&xfer.aToken[2]);
+ sqlite3_int64 mtime, iNow;
+ defossilize(zUser);
+ iNow = time(NULL);
+ if( blob_is_int64(&xfer.aToken[3], &mtime) && iNow>mtime ){
+ if( mtime<0 ){
+ xfer_fatal_error("invalid ci-lock-fail time");
+ return ++nErr;
+ }
+ iNow = time(NULL);
+ fossil_print("\nParent check-in locked by %s %s ago\n",
+ zUser, human_readable_age((iNow+1-mtime)/86400.0));
+ }else{
+ fossil_print("\nParent check-in locked by %s\n", zUser);
+ }
+ g.ckinLockFail = fossil_strdup(zUser);
+ }
+
+ /* pragma avoid-delta-manifests
+ **
+ ** Discourage the use of delta manifests. The remote side sends
+ ** this pragma when its forbid-delta-manifests setting is true.
+ */
+ else if( blob_eq(&xfer.aToken[1], "avoid-delta-manifests") ){
+ g.bAvoidDeltaManifests = 1;
+ }
+
+ /* pragma link URL ARG MTIME
+ **
+ ** The server has sent the URL for a link to another repository.
+ ** Record this as a link:URL entry in the config table.
+ */
+ else if( blob_eq(&xfer.aToken[1], "link")
+ && xfer.nToken==5
+ && (syncFlags & SYNC_SHARE_LINKS)!=0
+ ){
+ UrlData x;
+ char *zUrl = blob_str(&xfer.aToken[2]);
+ char *zArg = blob_str(&xfer.aToken[3]);
+ i64 iTime = strtoll(blob_str(&xfer.aToken[4]),0,0);
+ memset(&x, 0, sizeof(x));
+ defossilize(zUrl);
+ defossilize(zArg);
+ url_parse_local(zUrl, URL_OMIT_USER, &x);
+ if( x.protocol
+ && strncmp(x.protocol,"http",4)==0
+ && iTime>0
+ ){
+ db_unprotect(PROTECT_CONFIG);
+ db_multi_exec(
+ "INSERT INTO config(name,value,mtime)\n"
+ " VALUES('link:%q',%Q,%lld)\n"
+ " ON CONFLICT DO UPDATE\n"
+ " SET value=excluded.value, mtime=excluded.mtime\n"
+ " WHERE mtime<excluded.mtime;",
+ zUrl, zArg, iTime
+ );
+ db_protect_pop();
+ }
+ url_unparse(&x);
+ }
+
+ }else
+
+ /* error MESSAGE
+ **
+ ** The server is reporting an error. The client will abandon
+ ** the sync session.
+ **
+ ** Except, when cloning we will sometimes get an error on the
+ ** first message exchange because the project-code is unknown
+ ** and so the login card on the request was invalid. The project-code
+ ** is returned in the reply before the error card, so second and
+ ** subsequent messages should be OK. Nevertheless, we need to ignore
+ ** the error card on the first message of a clone.
+ **
+ ** Also ignore "not authorized to write" errors if this is an
+ ** autopush following a commit.
+ */
+ if( blob_eq(&xfer.aToken[0],"error") && xfer.nToken==2 ){
+ char *zMsg = blob_terminate(&xfer.aToken[1]);
+ defossilize(zMsg);
+ if( (syncFlags & SYNC_IFABLE)!=0
+ && sqlite3_strlike("%not authorized to write%",zMsg,0)==0 ){
+ autopushFailed = 1;
+ nErr++;
+ }else if( (syncFlags & SYNC_CLONE)==0 || nCycle>0 ){
+ fossil_force_newline();
+ fossil_print("Error: %s\n", zMsg);
+ blob_appendf(&xfer.err, "server says: %s\n", zMsg);
+ nErr++;
+ break;
+ }
+ }else
+
+ /* Unknown message */
+ if( xfer.nToken>0 ){
+ if( blob_str(&xfer.aToken[0])[0]=='<' ){
+ fossil_warning(
+ "server replies with HTML instead of fossil sync protocol:\n%b",
+ &recv
+ );
+ nErr++;
+ break;
+ }
+ blob_appendf(&xfer.err, "unknown command: [%b]\n", &xfer.line);
+ }
+
+ if( blob_size(&xfer.err) ){
+ fossil_force_newline();
+ fossil_warning("%b", &xfer.err);
+ nErr++;
+ break;
+ }
+ blobarray_reset(xfer.aToken, xfer.nToken);
+ blob_reset(&xfer.line);
+ }
+ origConfigRcvMask = 0;
+ if( nCardRcvd>0 && (syncFlags & SYNC_VERBOSE) ){
+ fossil_print(zValueFormat /*works-like:"%s%d%d%d%d"*/, "Received:",
+ blob_size(&recv), nCardRcvd,
+ xfer.nFileRcvd, xfer.nDeltaRcvd + xfer.nDanglingFile);
+ }else if( syncFlags & SYNC_QUIET ){
+ /* No-op */
+ }else{
+ if( bOutIsTty!=0 ){
+ fossil_print(zBriefFormat /*works-like:"%d%d%d"*/,
+ nRoundtrip, nArtifactSent, nArtifactRcvd);
+ }
+ }
+ nUncRcvd += blob_size(&recv);
+ blob_reset(&recv);
+ nCycle++;
+
+ /* Set go to 1 if we need to continue the sync/push/pull/clone for
+ ** another round. Set go to 0 if it is time to quit. */
+ nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
+ if( (nFileRecv>0 || newPhantom) && db_exists("SELECT 1 FROM phantom") ){
+ go = 1;
+ mxPhantomReq = nFileRecv*2;
+ if( mxPhantomReq<200 ) mxPhantomReq = 200;
+ }else if( xfer.nFileSent+xfer.nDeltaSent>0 || uvDoPush ){
+ /* Go another round if files are queued to send */
+ go = 1;
+ }else if( xfer.nPrivIGot>0 && nCycle==1 ){
+ go = 1;
+ }else if( nUvGimmeSent>0 && (nUvFileRcvd>0 || nCycle<3) ){
+ /* Continue looping as long as new uvfile cards are being received
+ ** and uvgimme cards are being sent. */
+ go = 1;
+ }else if( (syncFlags & SYNC_CLONE)!=0 ){
+ if( nCycle==1 ){
+ go = 1; /* go at least two rounds on a clone */
+ }else if( nFileRecv>0 ){
+ go = 1;
+ }else if( cloneSeqno>0 && nArtifactRcvd>nPriorArtifact ){
+ /* Continue the clone until we see the clone_seqno 0" card or
+ ** until we stop receiving artifacts */
+ go = 1;
+ }
+ }
+
+ nCardRcvd = 0;
+ xfer.nFileRcvd = 0;
+ xfer.nDeltaRcvd = 0;
+ xfer.nDanglingFile = 0;
+ db_multi_exec("DROP TABLE onremote; DROP TABLE unk;");
+ if( go ){
+ manifest_crosslink_end(MC_PERMIT_HOOKS);
+ }else{
+ manifest_crosslink_end(MC_PERMIT_HOOKS);
+ content_enable_dephantomize(1);
+ }
+ db_end_transaction(0);
+ }; /* while(go) */
+ transport_stats(&nSent, &nRcvd, 1);
+ if( pnRcvd ) *pnRcvd = nArtifactRcvd;
+ if( (rSkew*24.0*3600.0) > 10.0 ){
+ fossil_warning("*** time skew *** server is fast by %s",
+ db_timespan_name(rSkew));
+ g.clockSkewSeen = 1;
+ }else if( rSkew*24.0*3600.0 < -10.0 ){
+ fossil_warning("*** time skew *** server is slow by %s",
+ db_timespan_name(-rSkew));
+ g.clockSkewSeen = 1;
+ }
+ if( bOutIsTty==0 && (syncFlags & SYNC_QUIET)==0 ){
+ fossil_print(zBriefFormat /*works-like:"%d%d%d"*/,
+ nRoundtrip, nArtifactSent, nArtifactRcvd);
+ fossil_force_newline();
+ }
+ fossil_force_newline();
+ if( g.zHttpCmd==0 ){
+ if( syncFlags & SYNC_QUIET ){
+ /* no-op */
+ }else if( syncFlags & SYNC_VERBOSE ){
+ fossil_print(
+ "%s done, wire bytes sent: %lld received: %lld remote: %s%s\n",
+ zOpType, nSent, nRcvd,
+ (g.url.name && g.url.name[0]!='\0') ? g.url.name : "",
+ (g.zIpAddr && g.zIpAddr[0]!='\0'
+ && fossil_strcmp(g.zIpAddr, g.url.name))
+ ? mprintf(" (%s)", g.zIpAddr) : "");
+ }else{
+ fossil_print(
+ "%s done, wire bytes sent: %lld received: %lld remote: %s\n",
+ zOpType, nSent, nRcvd, g.zIpAddr);
+ }
+ }
+ if( syncFlags & SYNC_VERBOSE ){
+ fossil_print(
+ "Uncompressed payload sent: %lld received: %lld\n", nUncSent, nUncRcvd);
+ }
+ blob_reset(&send);
+ blob_reset(&recv);
+ transport_close(&g.url);
+ transport_global_shutdown(&g.url);
+ if( nErr && go==2 ){
+ db_multi_exec("DROP TABLE onremote; DROP TABLE unk;");
+ manifest_crosslink_end(MC_PERMIT_HOOKS);
+ content_enable_dephantomize(1);
+ db_end_transaction(0);
+ }
+ if( nErr && autopushFailed ){
+ fossil_warning(
+ "Warning: The check-in was successful and is saved locally but you\n"
+ " are not authorized to push the changes back to the server\n"
+ " at %s",
+ g.url.canonical
+ );
+ nErr--;
+ }
+ if( (syncFlags & SYNC_CLONE)==0 && g.rcvid && fossil_any_has_fork(g.rcvid) ){
+ fossil_warning("***** WARNING: a fork has occurred *****\n"
+ "use \"fossil leaves -multiple\" for more details.");
+ }
+ return nErr;
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-17 05:41:12 +0000