000001 /*
000002 ** 2001 September 15
000003 **
000004 ** The author disclaims copyright to this source code. In place of
000005 ** a legal notice, here is a blessing:
000006 **
000007 ** May you do good and not evil.
000008 ** May you find forgiveness for yourself and forgive others.
000009 ** May you share freely, never taking more than you give.
000010 **
000011 *************************************************************************
000012 ** This module contains C code that generates VDBE code used to process
000013 ** the WHERE clause of SQL statements. This module is responsible for
000014 ** generating the code that loops through a table looking for applicable
000015 ** rows. Indices are selected and used to speed the search when doing
000016 ** so is applicable. Because this module is responsible for selecting
000017 ** indices, you might also think of this module as the "query optimizer".
000018 */
000019 #include "sqliteInt.h"
000020 #include "whereInt.h"
000021
000022 /*
000023 ** Extra information appended to the end of sqlite3_index_info but not
000024 ** visible to the xBestIndex function, at least not directly. The
000025 ** sqlite3_vtab_collation() interface knows how to reach it, however.
000026 **
000027 ** This object is not an API and can be changed from one release to the
000028 ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation()
000029 ** agree on the structure, all will be well.
000030 */
000031 typedef struct HiddenIndexInfo HiddenIndexInfo;
000032 struct HiddenIndexInfo {
000033 WhereClause *pWC; /* The Where clause being analyzed */
000034 Parse *pParse; /* The parsing context */
000035 int eDistinct; /* Value to return from sqlite3_vtab_distinct() */
000036 u32 mIn; /* Mask of terms that are <col> IN (...) */
000037 u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */
000038 sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST
000039 ** because extra space is allocated to hold up
000040 ** to nTerm such values */
000041 };
000042
000043 /* Forward declaration of methods */
000044 static int whereLoopResize(sqlite3*, WhereLoop*, int);
000045
000046 /*
000047 ** Return the estimated number of output rows from a WHERE clause
000048 */
000049 LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
000050 return pWInfo->nRowOut;
000051 }
000052
000053 /*
000054 ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
000055 ** WHERE clause returns outputs for DISTINCT processing.
000056 */
000057 int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
000058 return pWInfo->eDistinct;
000059 }
000060
000061 /*
000062 ** Return the number of ORDER BY terms that are satisfied by the
000063 ** WHERE clause. A return of 0 means that the output must be
000064 ** completely sorted. A return equal to the number of ORDER BY
000065 ** terms means that no sorting is needed at all. A return that
000066 ** is positive but less than the number of ORDER BY terms means that
000067 ** block sorting is required.
000068 */
000069 int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
000070 return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat;
000071 }
000072
000073 /*
000074 ** In the ORDER BY LIMIT optimization, if the inner-most loop is known
000075 ** to emit rows in increasing order, and if the last row emitted by the
000076 ** inner-most loop did not fit within the sorter, then we can skip all
000077 ** subsequent rows for the current iteration of the inner loop (because they
000078 ** will not fit in the sorter either) and continue with the second inner
000079 ** loop - the loop immediately outside the inner-most.
000080 **
000081 ** When a row does not fit in the sorter (because the sorter already
000082 ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
000083 ** label returned by this function.
000084 **
000085 ** If the ORDER BY LIMIT optimization applies, the jump destination should
000086 ** be the continuation for the second-inner-most loop. If the ORDER BY
000087 ** LIMIT optimization does not apply, then the jump destination should
000088 ** be the continuation for the inner-most loop.
000089 **
000090 ** It is always safe for this routine to return the continuation of the
000091 ** inner-most loop, in the sense that a correct answer will result.
000092 ** Returning the continuation the second inner loop is an optimization
000093 ** that might make the code run a little faster, but should not change
000094 ** the final answer.
000095 */
000096 int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
000097 WhereLevel *pInner;
000098 if( !pWInfo->bOrderedInnerLoop ){
000099 /* The ORDER BY LIMIT optimization does not apply. Jump to the
000100 ** continuation of the inner-most loop. */
000101 return pWInfo->iContinue;
000102 }
000103 pInner = &pWInfo->a[pWInfo->nLevel-1];
000104 assert( pInner->addrNxt!=0 );
000105 return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt;
000106 }
000107
000108 /*
000109 ** While generating code for the min/max optimization, after handling
000110 ** the aggregate-step call to min() or max(), check to see if any
000111 ** additional looping is required. If the output order is such that
000112 ** we are certain that the correct answer has already been found, then
000113 ** code an OP_Goto to by pass subsequent processing.
000114 **
000115 ** Any extra OP_Goto that is coded here is an optimization. The
000116 ** correct answer should be obtained regardless. This OP_Goto just
000117 ** makes the answer appear faster.
000118 */
000119 void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){
000120 WhereLevel *pInner;
000121 int i;
000122 if( !pWInfo->bOrderedInnerLoop ) return;
000123 if( pWInfo->nOBSat==0 ) return;
000124 for(i=pWInfo->nLevel-1; i>=0; i--){
000125 pInner = &pWInfo->a[i];
000126 if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){
000127 sqlite3VdbeGoto(v, pInner->addrNxt);
000128 return;
000129 }
000130 }
000131 sqlite3VdbeGoto(v, pWInfo->iBreak);
000132 }
000133
000134 /*
000135 ** Return the VDBE address or label to jump to in order to continue
000136 ** immediately with the next row of a WHERE clause.
000137 */
000138 int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
000139 assert( pWInfo->iContinue!=0 );
000140 return pWInfo->iContinue;
000141 }
000142
000143 /*
000144 ** Return the VDBE address or label to jump to in order to break
000145 ** out of a WHERE loop.
000146 */
000147 int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
000148 return pWInfo->iBreak;
000149 }
000150
000151 /*
000152 ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
000153 ** operate directly on the rowids returned by a WHERE clause. Return
000154 ** ONEPASS_SINGLE (1) if the statement can operation directly because only
000155 ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
000156 ** optimization can be used on multiple
000157 **
000158 ** If the ONEPASS optimization is used (if this routine returns true)
000159 ** then also write the indices of open cursors used by ONEPASS
000160 ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
000161 ** table and iaCur[1] gets the cursor used by an auxiliary index.
000162 ** Either value may be -1, indicating that cursor is not used.
000163 ** Any cursors returned will have been opened for writing.
000164 **
000165 ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
000166 ** unable to use the ONEPASS optimization.
000167 */
000168 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
000169 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
000170 #ifdef WHERETRACE_ENABLED
000171 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){
000172 sqlite3DebugPrintf("%s cursors: %d %d\n",
000173 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
000174 aiCur[0], aiCur[1]);
000175 }
000176 #endif
000177 return pWInfo->eOnePass;
000178 }
000179
000180 /*
000181 ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move
000182 ** the data cursor to the row selected by the index cursor.
000183 */
000184 int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){
000185 return pWInfo->bDeferredSeek;
000186 }
000187
000188 /*
000189 ** Move the content of pSrc into pDest
000190 */
000191 static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
000192 pDest->n = pSrc->n;
000193 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
000194 }
000195
000196 /*
000197 ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
000198 **
000199 ** The new entry might overwrite an existing entry, or it might be
000200 ** appended, or it might be discarded. Do whatever is the right thing
000201 ** so that pSet keeps the N_OR_COST best entries seen so far.
000202 */
000203 static int whereOrInsert(
000204 WhereOrSet *pSet, /* The WhereOrSet to be updated */
000205 Bitmask prereq, /* Prerequisites of the new entry */
000206 LogEst rRun, /* Run-cost of the new entry */
000207 LogEst nOut /* Number of outputs for the new entry */
000208 ){
000209 u16 i;
000210 WhereOrCost *p;
000211 for(i=pSet->n, p=pSet->a; i>0; i--, p++){
000212 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
000213 goto whereOrInsert_done;
000214 }
000215 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
000216 return 0;
000217 }
000218 }
000219 if( pSet->n<N_OR_COST ){
000220 p = &pSet->a[pSet->n++];
000221 p->nOut = nOut;
000222 }else{
000223 p = pSet->a;
000224 for(i=1; i<pSet->n; i++){
000225 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
000226 }
000227 if( p->rRun<=rRun ) return 0;
000228 }
000229 whereOrInsert_done:
000230 p->prereq = prereq;
000231 p->rRun = rRun;
000232 if( p->nOut>nOut ) p->nOut = nOut;
000233 return 1;
000234 }
000235
000236 /*
000237 ** Return the bitmask for the given cursor number. Return 0 if
000238 ** iCursor is not in the set.
000239 */
000240 Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){
000241 int i;
000242 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
000243 assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 );
000244 assert( iCursor>=-1 );
000245 if( pMaskSet->ix[0]==iCursor ){
000246 return 1;
000247 }
000248 for(i=1; i<pMaskSet->n; i++){
000249 if( pMaskSet->ix[i]==iCursor ){
000250 return MASKBIT(i);
000251 }
000252 }
000253 return 0;
000254 }
000255
000256 /* Allocate memory that is automatically freed when pWInfo is freed.
000257 */
000258 void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){
000259 WhereMemBlock *pBlock;
000260 pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock));
000261 if( pBlock ){
000262 pBlock->pNext = pWInfo->pMemToFree;
000263 pBlock->sz = nByte;
000264 pWInfo->pMemToFree = pBlock;
000265 pBlock++;
000266 }
000267 return (void*)pBlock;
000268 }
000269 void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){
000270 void *pNew = sqlite3WhereMalloc(pWInfo, nByte);
000271 if( pNew && pOld ){
000272 WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld;
000273 pOldBlk--;
000274 assert( pOldBlk->sz<nByte );
000275 memcpy(pNew, pOld, pOldBlk->sz);
000276 }
000277 return pNew;
000278 }
000279
000280 /*
000281 ** Create a new mask for cursor iCursor.
000282 **
000283 ** There is one cursor per table in the FROM clause. The number of
000284 ** tables in the FROM clause is limited by a test early in the
000285 ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
000286 ** array will never overflow.
000287 */
000288 static void createMask(WhereMaskSet *pMaskSet, int iCursor){
000289 assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
000290 pMaskSet->ix[pMaskSet->n++] = iCursor;
000291 }
000292
000293 /*
000294 ** If the right-hand branch of the expression is a TK_COLUMN, then return
000295 ** a pointer to the right-hand branch. Otherwise, return NULL.
000296 */
000297 static Expr *whereRightSubexprIsColumn(Expr *p){
000298 p = sqlite3ExprSkipCollateAndLikely(p->pRight);
000299 if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){
000300 return p;
000301 }
000302 return 0;
000303 }
000304
000305 /*
000306 ** Term pTerm is guaranteed to be a WO_IN term. It may be a component term
000307 ** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)".
000308 ** This function checks to see if the term is compatible with an index
000309 ** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so,
000310 ** it returns a pointer to the name of the collation sequence (e.g. "BINARY"
000311 ** or "NOCASE") used by the comparison in pTerm. If it is not compatible
000312 ** with affinity idxaff, NULL is returned.
000313 */
000314 static SQLITE_NOINLINE const char *indexInAffinityOk(
000315 Parse *pParse,
000316 WhereTerm *pTerm,
000317 u8 idxaff
000318 ){
000319 Expr *pX = pTerm->pExpr;
000320 Expr inexpr;
000321
000322 assert( pTerm->eOperator & WO_IN );
000323
000324 if( sqlite3ExprIsVector(pX->pLeft) ){
000325 int iField = pTerm->u.x.iField - 1;
000326 inexpr.flags = 0;
000327 inexpr.op = TK_EQ;
000328 inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr;
000329 assert( ExprUseXSelect(pX) );
000330 inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr;
000331 pX = &inexpr;
000332 }
000333
000334 if( sqlite3IndexAffinityOk(pX, idxaff) ){
000335 CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX);
000336 return pRet ? pRet->zName : sqlite3StrBINARY;
000337 }
000338 return 0;
000339 }
000340
000341 /*
000342 ** Advance to the next WhereTerm that matches according to the criteria
000343 ** established when the pScan object was initialized by whereScanInit().
000344 ** Return NULL if there are no more matching WhereTerms.
000345 */
000346 static WhereTerm *whereScanNext(WhereScan *pScan){
000347 int iCur; /* The cursor on the LHS of the term */
000348 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */
000349 Expr *pX; /* An expression being tested */
000350 WhereClause *pWC; /* Shorthand for pScan->pWC */
000351 WhereTerm *pTerm; /* The term being tested */
000352 int k = pScan->k; /* Where to start scanning */
000353
000354 assert( pScan->iEquiv<=pScan->nEquiv );
000355 pWC = pScan->pWC;
000356 while(1){
000357 iColumn = pScan->aiColumn[pScan->iEquiv-1];
000358 iCur = pScan->aiCur[pScan->iEquiv-1];
000359 assert( pWC!=0 );
000360 assert( iCur>=0 );
000361 do{
000362 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
000363 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 );
000364 if( pTerm->leftCursor==iCur
000365 && pTerm->u.x.leftColumn==iColumn
000366 && (iColumn!=XN_EXPR
000367 || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
000368 pScan->pIdxExpr,iCur)==0)
000369 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON))
000370 ){
000371 if( (pTerm->eOperator & WO_EQUIV)!=0
000372 && pScan->nEquiv<ArraySize(pScan->aiCur)
000373 && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0
000374 ){
000375 int j;
000376 for(j=0; j<pScan->nEquiv; j++){
000377 if( pScan->aiCur[j]==pX->iTable
000378 && pScan->aiColumn[j]==pX->iColumn ){
000379 break;
000380 }
000381 }
000382 if( j==pScan->nEquiv ){
000383 pScan->aiCur[j] = pX->iTable;
000384 pScan->aiColumn[j] = pX->iColumn;
000385 pScan->nEquiv++;
000386 }
000387 }
000388 if( (pTerm->eOperator & pScan->opMask)!=0 ){
000389 /* Verify the affinity and collating sequence match */
000390 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
000391 const char *zCollName;
000392 Parse *pParse = pWC->pWInfo->pParse;
000393 pX = pTerm->pExpr;
000394
000395 if( (pTerm->eOperator & WO_IN) ){
000396 zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff);
000397 if( !zCollName ) continue;
000398 }else{
000399 CollSeq *pColl;
000400 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
000401 continue;
000402 }
000403 assert(pX->pLeft);
000404 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
000405 zCollName = pColl ? pColl->zName : sqlite3StrBINARY;
000406 }
000407
000408 if( sqlite3StrICmp(zCollName, pScan->zCollName) ){
000409 continue;
000410 }
000411 }
000412 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0
000413 && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0))
000414 && pX->op==TK_COLUMN
000415 && pX->iTable==pScan->aiCur[0]
000416 && pX->iColumn==pScan->aiColumn[0]
000417 ){
000418 testcase( pTerm->eOperator & WO_IS );
000419 continue;
000420 }
000421 pScan->pWC = pWC;
000422 pScan->k = k+1;
000423 #ifdef WHERETRACE_ENABLED
000424 if( sqlite3WhereTrace & 0x20000 ){
000425 int ii;
000426 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
000427 pTerm, pScan->nEquiv);
000428 for(ii=0; ii<pScan->nEquiv; ii++){
000429 sqlite3DebugPrintf(" {%d:%d}",
000430 pScan->aiCur[ii], pScan->aiColumn[ii]);
000431 }
000432 sqlite3DebugPrintf("\n");
000433 }
000434 #endif
000435 return pTerm;
000436 }
000437 }
000438 }
000439 pWC = pWC->pOuter;
000440 k = 0;
000441 }while( pWC!=0 );
000442 if( pScan->iEquiv>=pScan->nEquiv ) break;
000443 pWC = pScan->pOrigWC;
000444 k = 0;
000445 pScan->iEquiv++;
000446 }
000447 return 0;
000448 }
000449
000450 /*
000451 ** This is whereScanInit() for the case of an index on an expression.
000452 ** It is factored out into a separate tail-recursion subroutine so that
000453 ** the normal whereScanInit() routine, which is a high-runner, does not
000454 ** need to push registers onto the stack as part of its prologue.
000455 */
000456 static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){
000457 pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr);
000458 return whereScanNext(pScan);
000459 }
000460
000461 /*
000462 ** Initialize a WHERE clause scanner object. Return a pointer to the
000463 ** first match. Return NULL if there are no matches.
000464 **
000465 ** The scanner will be searching the WHERE clause pWC. It will look
000466 ** for terms of the form "X <op> <expr>" where X is column iColumn of table
000467 ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
000468 ** must be one of the indexes of table iCur.
000469 **
000470 ** The <op> must be one of the operators described by opMask.
000471 **
000472 ** If the search is for X and the WHERE clause contains terms of the
000473 ** form X=Y then this routine might also return terms of the form
000474 ** "Y <op> <expr>". The number of levels of transitivity is limited,
000475 ** but is enough to handle most commonly occurring SQL statements.
000476 **
000477 ** If X is not the INTEGER PRIMARY KEY then X must be compatible with
000478 ** index pIdx.
000479 */
000480 static WhereTerm *whereScanInit(
000481 WhereScan *pScan, /* The WhereScan object being initialized */
000482 WhereClause *pWC, /* The WHERE clause to be scanned */
000483 int iCur, /* Cursor to scan for */
000484 int iColumn, /* Column to scan for */
000485 u32 opMask, /* Operator(s) to scan for */
000486 Index *pIdx /* Must be compatible with this index */
000487 ){
000488 pScan->pOrigWC = pWC;
000489 pScan->pWC = pWC;
000490 pScan->pIdxExpr = 0;
000491 pScan->idxaff = 0;
000492 pScan->zCollName = 0;
000493 pScan->opMask = opMask;
000494 pScan->k = 0;
000495 pScan->aiCur[0] = iCur;
000496 pScan->nEquiv = 1;
000497 pScan->iEquiv = 1;
000498 if( pIdx ){
000499 int j = iColumn;
000500 iColumn = pIdx->aiColumn[j];
000501 if( iColumn==pIdx->pTable->iPKey ){
000502 iColumn = XN_ROWID;
000503 }else if( iColumn>=0 ){
000504 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
000505 pScan->zCollName = pIdx->azColl[j];
000506 }else if( iColumn==XN_EXPR ){
000507 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
000508 pScan->zCollName = pIdx->azColl[j];
000509 pScan->aiColumn[0] = XN_EXPR;
000510 return whereScanInitIndexExpr(pScan);
000511 }
000512 }else if( iColumn==XN_EXPR ){
000513 return 0;
000514 }
000515 pScan->aiColumn[0] = iColumn;
000516 return whereScanNext(pScan);
000517 }
000518
000519 /*
000520 ** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
000521 ** where X is a reference to the iColumn of table iCur or of index pIdx
000522 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
000523 ** the op parameter. Return a pointer to the term. Return 0 if not found.
000524 **
000525 ** If pIdx!=0 then it must be one of the indexes of table iCur.
000526 ** Search for terms matching the iColumn-th column of pIdx
000527 ** rather than the iColumn-th column of table iCur.
000528 **
000529 ** The term returned might by Y=<expr> if there is another constraint in
000530 ** the WHERE clause that specifies that X=Y. Any such constraints will be
000531 ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
000532 ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
000533 ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
000534 ** other equivalent values. Hence a search for X will return <expr> if X=A1
000535 ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
000536 **
000537 ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
000538 ** then try for the one with no dependencies on <expr> - in other words where
000539 ** <expr> is a constant expression of some kind. Only return entries of
000540 ** the form "X <op> Y" where Y is a column in another table if no terms of
000541 ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
000542 ** exist, try to return a term that does not use WO_EQUIV.
000543 */
000544 WhereTerm *sqlite3WhereFindTerm(
000545 WhereClause *pWC, /* The WHERE clause to be searched */
000546 int iCur, /* Cursor number of LHS */
000547 int iColumn, /* Column number of LHS */
000548 Bitmask notReady, /* RHS must not overlap with this mask */
000549 u32 op, /* Mask of WO_xx values describing operator */
000550 Index *pIdx /* Must be compatible with this index, if not NULL */
000551 ){
000552 WhereTerm *pResult = 0;
000553 WhereTerm *p;
000554 WhereScan scan;
000555
000556 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
000557 op &= WO_EQ|WO_IS;
000558 while( p ){
000559 if( (p->prereqRight & notReady)==0 ){
000560 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){
000561 testcase( p->eOperator & WO_IS );
000562 return p;
000563 }
000564 if( pResult==0 ) pResult = p;
000565 }
000566 p = whereScanNext(&scan);
000567 }
000568 return pResult;
000569 }
000570
000571 /*
000572 ** This function searches pList for an entry that matches the iCol-th column
000573 ** of index pIdx.
000574 **
000575 ** If such an expression is found, its index in pList->a[] is returned. If
000576 ** no expression is found, -1 is returned.
000577 */
000578 static int findIndexCol(
000579 Parse *pParse, /* Parse context */
000580 ExprList *pList, /* Expression list to search */
000581 int iBase, /* Cursor for table associated with pIdx */
000582 Index *pIdx, /* Index to match column of */
000583 int iCol /* Column of index to match */
000584 ){
000585 int i;
000586 const char *zColl = pIdx->azColl[iCol];
000587
000588 for(i=0; i<pList->nExpr; i++){
000589 Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
000590 if( ALWAYS(p!=0)
000591 && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
000592 && p->iColumn==pIdx->aiColumn[iCol]
000593 && p->iTable==iBase
000594 ){
000595 CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr);
000596 if( 0==sqlite3StrICmp(pColl->zName, zColl) ){
000597 return i;
000598 }
000599 }
000600 }
000601
000602 return -1;
000603 }
000604
000605 /*
000606 ** Return TRUE if the iCol-th column of index pIdx is NOT NULL
000607 */
000608 static int indexColumnNotNull(Index *pIdx, int iCol){
000609 int j;
000610 assert( pIdx!=0 );
000611 assert( iCol>=0 && iCol<pIdx->nColumn );
000612 j = pIdx->aiColumn[iCol];
000613 if( j>=0 ){
000614 return pIdx->pTable->aCol[j].notNull;
000615 }else if( j==(-1) ){
000616 return 1;
000617 }else{
000618 assert( j==(-2) );
000619 return 0; /* Assume an indexed expression can always yield a NULL */
000620
000621 }
000622 }
000623
000624 /*
000625 ** Return true if the DISTINCT expression-list passed as the third argument
000626 ** is redundant.
000627 **
000628 ** A DISTINCT list is redundant if any subset of the columns in the
000629 ** DISTINCT list are collectively unique and individually non-null.
000630 */
000631 static int isDistinctRedundant(
000632 Parse *pParse, /* Parsing context */
000633 SrcList *pTabList, /* The FROM clause */
000634 WhereClause *pWC, /* The WHERE clause */
000635 ExprList *pDistinct /* The result set that needs to be DISTINCT */
000636 ){
000637 Table *pTab;
000638 Index *pIdx;
000639 int i;
000640 int iBase;
000641
000642 /* If there is more than one table or sub-select in the FROM clause of
000643 ** this query, then it will not be possible to show that the DISTINCT
000644 ** clause is redundant. */
000645 if( pTabList->nSrc!=1 ) return 0;
000646 iBase = pTabList->a[0].iCursor;
000647 pTab = pTabList->a[0].pTab;
000648
000649 /* If any of the expressions is an IPK column on table iBase, then return
000650 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
000651 ** current SELECT is a correlated sub-query.
000652 */
000653 for(i=0; i<pDistinct->nExpr; i++){
000654 Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
000655 if( NEVER(p==0) ) continue;
000656 if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
000657 if( p->iTable==iBase && p->iColumn<0 ) return 1;
000658 }
000659
000660 /* Loop through all indices on the table, checking each to see if it makes
000661 ** the DISTINCT qualifier redundant. It does so if:
000662 **
000663 ** 1. The index is itself UNIQUE, and
000664 **
000665 ** 2. All of the columns in the index are either part of the pDistinct
000666 ** list, or else the WHERE clause contains a term of the form "col=X",
000667 ** where X is a constant value. The collation sequences of the
000668 ** comparison and select-list expressions must match those of the index.
000669 **
000670 ** 3. All of those index columns for which the WHERE clause does not
000671 ** contain a "col=X" term are subject to a NOT NULL constraint.
000672 */
000673 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
000674 if( !IsUniqueIndex(pIdx) ) continue;
000675 if( pIdx->pPartIdxWhere ) continue;
000676 for(i=0; i<pIdx->nKeyCol; i++){
000677 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
000678 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
000679 if( indexColumnNotNull(pIdx, i)==0 ) break;
000680 }
000681 }
000682 if( i==pIdx->nKeyCol ){
000683 /* This index implies that the DISTINCT qualifier is redundant. */
000684 return 1;
000685 }
000686 }
000687
000688 return 0;
000689 }
000690
000691
000692 /*
000693 ** Estimate the logarithm of the input value to base 2.
000694 */
000695 static LogEst estLog(LogEst N){
000696 return N<=10 ? 0 : sqlite3LogEst(N) - 33;
000697 }
000698
000699 /*
000700 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
000701 **
000702 ** This routine runs over generated VDBE code and translates OP_Column
000703 ** opcodes into OP_Copy when the table is being accessed via co-routine
000704 ** instead of via table lookup.
000705 **
000706 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
000707 ** cursor iTabCur are transformed into OP_Sequence opcode for the
000708 ** iAutoidxCur cursor, in order to generate unique rowids for the
000709 ** automatic index being generated.
000710 */
000711 static void translateColumnToCopy(
000712 Parse *pParse, /* Parsing context */
000713 int iStart, /* Translate from this opcode to the end */
000714 int iTabCur, /* OP_Column/OP_Rowid references to this table */
000715 int iRegister, /* The first column is in this register */
000716 int iAutoidxCur /* If non-zero, cursor of autoindex being generated */
000717 ){
000718 Vdbe *v = pParse->pVdbe;
000719 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
000720 int iEnd = sqlite3VdbeCurrentAddr(v);
000721 if( pParse->db->mallocFailed ) return;
000722 for(; iStart<iEnd; iStart++, pOp++){
000723 if( pOp->p1!=iTabCur ) continue;
000724 if( pOp->opcode==OP_Column ){
000725 #ifdef SQLITE_DEBUG
000726 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000727 printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart);
000728 }
000729 #endif
000730 pOp->opcode = OP_Copy;
000731 pOp->p1 = pOp->p2 + iRegister;
000732 pOp->p2 = pOp->p3;
000733 pOp->p3 = 0;
000734 pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */
000735 }else if( pOp->opcode==OP_Rowid ){
000736 #ifdef SQLITE_DEBUG
000737 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000738 printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart);
000739 }
000740 #endif
000741 pOp->opcode = OP_Sequence;
000742 pOp->p1 = iAutoidxCur;
000743 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW
000744 if( iAutoidxCur==0 ){
000745 pOp->opcode = OP_Null;
000746 pOp->p3 = 0;
000747 }
000748 #endif
000749 }
000750 }
000751 }
000752
000753 /*
000754 ** Two routines for printing the content of an sqlite3_index_info
000755 ** structure. Used for testing and debugging only. If neither
000756 ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
000757 ** are no-ops.
000758 */
000759 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
000760 static void whereTraceIndexInfoInputs(
000761 sqlite3_index_info *p, /* The IndexInfo object */
000762 Table *pTab /* The TABLE that is the virtual table */
000763 ){
000764 int i;
000765 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000766 sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName);
000767 for(i=0; i<p->nConstraint; i++){
000768 sqlite3DebugPrintf(
000769 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
000770 i,
000771 p->aConstraint[i].iColumn,
000772 p->aConstraint[i].iTermOffset,
000773 p->aConstraint[i].op,
000774 p->aConstraint[i].usable,
000775 sqlite3_vtab_collation(p,i));
000776 }
000777 for(i=0; i<p->nOrderBy; i++){
000778 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
000779 i,
000780 p->aOrderBy[i].iColumn,
000781 p->aOrderBy[i].desc);
000782 }
000783 }
000784 static void whereTraceIndexInfoOutputs(
000785 sqlite3_index_info *p, /* The IndexInfo object */
000786 Table *pTab /* The TABLE that is the virtual table */
000787 ){
000788 int i;
000789 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000790 sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName);
000791 for(i=0; i<p->nConstraint; i++){
000792 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
000793 i,
000794 p->aConstraintUsage[i].argvIndex,
000795 p->aConstraintUsage[i].omit);
000796 }
000797 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum);
000798 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr);
000799 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed);
000800 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost);
000801 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows);
000802 }
000803 #else
000804 #define whereTraceIndexInfoInputs(A,B)
000805 #define whereTraceIndexInfoOutputs(A,B)
000806 #endif
000807
000808 /*
000809 ** We know that pSrc is an operand of an outer join. Return true if
000810 ** pTerm is a constraint that is compatible with that join.
000811 **
000812 ** pTerm must be EP_OuterON if pSrc is the right operand of an
000813 ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
000814 ** is the left operand of a RIGHT join.
000815 **
000816 ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
000817 ** for an example of a WHERE clause constraints that may not be used on
000818 ** the right table of a RIGHT JOIN because the constraint implies a
000819 ** not-NULL condition on the left table of the RIGHT JOIN.
000820 */
000821 static int constraintCompatibleWithOuterJoin(
000822 const WhereTerm *pTerm, /* WHERE clause term to check */
000823 const SrcItem *pSrc /* Table we are trying to access */
000824 ){
000825 assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */
000826 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
000827 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
000828 testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
000829 testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
000830 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
000831 || pTerm->pExpr->w.iJoin != pSrc->iCursor
000832 ){
000833 return 0;
000834 }
000835 if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
000836 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
000837 ){
000838 return 0;
000839 }
000840 return 1;
000841 }
000842
000843
000844
000845 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000846 /*
000847 ** Return TRUE if the WHERE clause term pTerm is of a form where it
000848 ** could be used with an index to access pSrc, assuming an appropriate
000849 ** index existed.
000850 */
000851 static int termCanDriveIndex(
000852 const WhereTerm *pTerm, /* WHERE clause term to check */
000853 const SrcItem *pSrc, /* Table we are trying to access */
000854 const Bitmask notReady /* Tables in outer loops of the join */
000855 ){
000856 char aff;
000857 if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
000858 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
000859 assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
000860 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
000861 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
000862 ){
000863 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
000864 }
000865 if( (pTerm->prereqRight & notReady)!=0 ) return 0;
000866 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000867 if( pTerm->u.x.leftColumn<0 ) return 0;
000868 aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity;
000869 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
000870 testcase( pTerm->pExpr->op==TK_IS );
000871 return 1;
000872 }
000873 #endif
000874
000875
000876 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000877
000878 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000879 /*
000880 ** Argument pIdx represents an automatic index that the current statement
000881 ** will create and populate. Add an OP_Explain with text of the form:
000882 **
000883 ** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>]
000884 **
000885 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to
000886 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
000887 ** values with. In order to avoid breaking legacy code and test cases,
000888 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
000889 */
000890 static void explainAutomaticIndex(
000891 Parse *pParse,
000892 Index *pIdx, /* Automatic index to explain */
000893 int bPartial, /* True if pIdx is a partial index */
000894 int *pAddrExplain /* OUT: Address of OP_Explain */
000895 ){
000896 if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){
000897 Table *pTab = pIdx->pTable;
000898 const char *zSep = "";
000899 char *zText = 0;
000900 int ii = 0;
000901 sqlite3_str *pStr = sqlite3_str_new(pParse->db);
000902 sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName);
000903 assert( pIdx->nColumn>1 );
000904 assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID );
000905 for(ii=0; ii<(pIdx->nColumn-1); ii++){
000906 const char *zName = 0;
000907 int iCol = pIdx->aiColumn[ii];
000908
000909 zName = pTab->aCol[iCol].zCnName;
000910 sqlite3_str_appendf(pStr, "%s%s", zSep, zName);
000911 zSep = ", ";
000912 }
000913 zText = sqlite3_str_finish(pStr);
000914 if( zText==0 ){
000915 sqlite3OomFault(pParse->db);
000916 }else{
000917 *pAddrExplain = sqlite3VdbeExplain(
000918 pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "")
000919 );
000920 sqlite3_free(zText);
000921 }
000922 }
000923 }
000924 #else
000925 # define explainAutomaticIndex(a,b,c,d)
000926 #endif
000927
000928 /*
000929 ** Generate code to construct the Index object for an automatic index
000930 ** and to set up the WhereLevel object pLevel so that the code generator
000931 ** makes use of the automatic index.
000932 */
000933 static SQLITE_NOINLINE void constructAutomaticIndex(
000934 Parse *pParse, /* The parsing context */
000935 WhereClause *pWC, /* The WHERE clause */
000936 const Bitmask notReady, /* Mask of cursors that are not available */
000937 WhereLevel *pLevel /* Write new index here */
000938 ){
000939 int nKeyCol; /* Number of columns in the constructed index */
000940 WhereTerm *pTerm; /* A single term of the WHERE clause */
000941 WhereTerm *pWCEnd; /* End of pWC->a[] */
000942 Index *pIdx; /* Object describing the transient index */
000943 Vdbe *v; /* Prepared statement under construction */
000944 int addrInit; /* Address of the initialization bypass jump */
000945 Table *pTable; /* The table being indexed */
000946 int addrTop; /* Top of the index fill loop */
000947 int regRecord; /* Register holding an index record */
000948 int n; /* Column counter */
000949 int i; /* Loop counter */
000950 int mxBitCol; /* Maximum column in pSrc->colUsed */
000951 CollSeq *pColl; /* Collating sequence to on a column */
000952 WhereLoop *pLoop; /* The Loop object */
000953 char *zNotUsed; /* Extra space on the end of pIdx */
000954 Bitmask idxCols; /* Bitmap of columns used for indexing */
000955 Bitmask extraCols; /* Bitmap of additional columns */
000956 u8 sentWarning = 0; /* True if a warning has been issued */
000957 u8 useBloomFilter = 0; /* True to also add a Bloom filter */
000958 Expr *pPartial = 0; /* Partial Index Expression */
000959 int iContinue = 0; /* Jump here to skip excluded rows */
000960 SrcList *pTabList; /* The complete FROM clause */
000961 SrcItem *pSrc; /* The FROM clause term to get the next index */
000962 int addrCounter = 0; /* Address where integer counter is initialized */
000963 int regBase; /* Array of registers where record is assembled */
000964 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000965 int addrExp = 0; /* Address of OP_Explain */
000966 #endif
000967
000968 /* Generate code to skip over the creation and initialization of the
000969 ** transient index on 2nd and subsequent iterations of the loop. */
000970 v = pParse->pVdbe;
000971 assert( v!=0 );
000972 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
000973
000974 /* Count the number of columns that will be added to the index
000975 ** and used to match WHERE clause constraints */
000976 nKeyCol = 0;
000977 pTabList = pWC->pWInfo->pTabList;
000978 pSrc = &pTabList->a[pLevel->iFrom];
000979 pTable = pSrc->pTab;
000980 pWCEnd = &pWC->a[pWC->nTerm];
000981 pLoop = pLevel->pWLoop;
000982 idxCols = 0;
000983 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
000984 Expr *pExpr = pTerm->pExpr;
000985 /* Make the automatic index a partial index if there are terms in the
000986 ** WHERE clause (or the ON clause of a LEFT join) that constrain which
000987 ** rows of the target table (pSrc) that can be used. */
000988 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
000989 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0)
000990 ){
000991 pPartial = sqlite3ExprAnd(pParse, pPartial,
000992 sqlite3ExprDup(pParse->db, pExpr, 0));
000993 }
000994 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
000995 int iCol;
000996 Bitmask cMask;
000997 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000998 iCol = pTerm->u.x.leftColumn;
000999 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
001000 testcase( iCol==BMS );
001001 testcase( iCol==BMS-1 );
001002 if( !sentWarning ){
001003 sqlite3_log(SQLITE_WARNING_AUTOINDEX,
001004 "automatic index on %s(%s)", pTable->zName,
001005 pTable->aCol[iCol].zCnName);
001006 sentWarning = 1;
001007 }
001008 if( (idxCols & cMask)==0 ){
001009 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
001010 goto end_auto_index_create;
001011 }
001012 pLoop->aLTerm[nKeyCol++] = pTerm;
001013 idxCols |= cMask;
001014 }
001015 }
001016 }
001017 assert( nKeyCol>0 || pParse->db->mallocFailed );
001018 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
001019 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
001020 | WHERE_AUTO_INDEX;
001021
001022 /* Count the number of additional columns needed to create a
001023 ** covering index. A "covering index" is an index that contains all
001024 ** columns that are needed by the query. With a covering index, the
001025 ** original table never needs to be accessed. Automatic indices must
001026 ** be a covering index because the index will not be updated if the
001027 ** original table changes and the index and table cannot both be used
001028 ** if they go out of sync.
001029 */
001030 if( IsView(pTable) ){
001031 extraCols = ALLBITS & ~idxCols;
001032 }else{
001033 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
001034 }
001035 mxBitCol = MIN(BMS-1,pTable->nCol);
001036 testcase( pTable->nCol==BMS-1 );
001037 testcase( pTable->nCol==BMS-2 );
001038 for(i=0; i<mxBitCol; i++){
001039 if( extraCols & MASKBIT(i) ) nKeyCol++;
001040 }
001041 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001042 nKeyCol += pTable->nCol - BMS + 1;
001043 }
001044
001045 /* Construct the Index object to describe this index */
001046 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
001047 if( pIdx==0 ) goto end_auto_index_create;
001048 pLoop->u.btree.pIndex = pIdx;
001049 pIdx->zName = "auto-index";
001050 pIdx->pTable = pTable;
001051 n = 0;
001052 idxCols = 0;
001053 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
001054 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
001055 int iCol;
001056 Bitmask cMask;
001057 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001058 iCol = pTerm->u.x.leftColumn;
001059 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
001060 testcase( iCol==BMS-1 );
001061 testcase( iCol==BMS );
001062 if( (idxCols & cMask)==0 ){
001063 Expr *pX = pTerm->pExpr;
001064 idxCols |= cMask;
001065 pIdx->aiColumn[n] = pTerm->u.x.leftColumn;
001066 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
001067 assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
001068 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
001069 n++;
001070 if( ALWAYS(pX->pLeft!=0)
001071 && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT
001072 ){
001073 /* TUNING: only use a Bloom filter on an automatic index
001074 ** if one or more key columns has the ability to hold numeric
001075 ** values, since strings all have the same hash in the Bloom
001076 ** filter implementation and hence a Bloom filter on a text column
001077 ** is not usually helpful. */
001078 useBloomFilter = 1;
001079 }
001080 }
001081 }
001082 }
001083 assert( (u32)n==pLoop->u.btree.nEq );
001084
001085 /* Add additional columns needed to make the automatic index into
001086 ** a covering index */
001087 for(i=0; i<mxBitCol; i++){
001088 if( extraCols & MASKBIT(i) ){
001089 pIdx->aiColumn[n] = i;
001090 pIdx->azColl[n] = sqlite3StrBINARY;
001091 n++;
001092 }
001093 }
001094 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001095 for(i=BMS-1; i<pTable->nCol; i++){
001096 pIdx->aiColumn[n] = i;
001097 pIdx->azColl[n] = sqlite3StrBINARY;
001098 n++;
001099 }
001100 }
001101 assert( n==nKeyCol );
001102 pIdx->aiColumn[n] = XN_ROWID;
001103 pIdx->azColl[n] = sqlite3StrBINARY;
001104
001105 /* Create the automatic index */
001106 explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp);
001107 assert( pLevel->iIdxCur>=0 );
001108 pLevel->iIdxCur = pParse->nTab++;
001109 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
001110 sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
001111 VdbeComment((v, "for %s", pTable->zName));
001112 if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){
001113 sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel);
001114 pLevel->regFilter = ++pParse->nMem;
001115 sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
001116 }
001117
001118 /* Fill the automatic index with content */
001119 assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] );
001120 if( pSrc->fg.viaCoroutine ){
001121 int regYield = pSrc->regReturn;
001122 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
001123 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub);
001124 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield);
001125 VdbeCoverage(v);
001126 VdbeComment((v, "next row of %s", pSrc->pTab->zName));
001127 }else{
001128 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
001129 }
001130 if( pPartial ){
001131 iContinue = sqlite3VdbeMakeLabel(pParse);
001132 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
001133 pLoop->wsFlags |= WHERE_PARTIALIDX;
001134 }
001135 regRecord = sqlite3GetTempReg(pParse);
001136 regBase = sqlite3GenerateIndexKey(
001137 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
001138 );
001139 if( pLevel->regFilter ){
001140 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
001141 regBase, pLoop->u.btree.nEq);
001142 }
001143 sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v));
001144 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
001145 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
001146 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
001147 if( pSrc->fg.viaCoroutine ){
001148 sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
001149 testcase( pParse->db->mallocFailed );
001150 assert( pLevel->iIdxCur>0 );
001151 translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
001152 pSrc->regResult, pLevel->iIdxCur);
001153 sqlite3VdbeGoto(v, addrTop);
001154 pSrc->fg.viaCoroutine = 0;
001155 }else{
001156 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
001157 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
001158 }
001159 sqlite3VdbeJumpHere(v, addrTop);
001160 sqlite3ReleaseTempReg(pParse, regRecord);
001161
001162 /* Jump here when skipping the initialization */
001163 sqlite3VdbeJumpHere(v, addrInit);
001164 sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1);
001165
001166 end_auto_index_create:
001167 sqlite3ExprDelete(pParse->db, pPartial);
001168 }
001169 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
001170
001171 /*
001172 ** Generate bytecode that will initialize a Bloom filter that is appropriate
001173 ** for pLevel.
001174 **
001175 ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
001176 ** flag set, initialize a Bloomfilter for them as well. Except don't do
001177 ** this recursive initialization if the SQLITE_BloomPulldown optimization has
001178 ** been turned off.
001179 **
001180 ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
001181 ** from the loop, but the regFilter value is set to a register that implements
001182 ** the Bloom filter. When regFilter is positive, the
001183 ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
001184 ** and skip the subsequence B-Tree seek if the Bloom filter indicates that
001185 ** no matching rows exist.
001186 **
001187 ** This routine may only be called if it has previously been determined that
001188 ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
001189 ** is set.
001190 */
001191 static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
001192 WhereInfo *pWInfo, /* The WHERE clause */
001193 int iLevel, /* Index in pWInfo->a[] that is pLevel */
001194 WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
001195 Bitmask notReady /* Loops that are not ready */
001196 ){
001197 int addrOnce; /* Address of opening OP_Once */
001198 int addrTop; /* Address of OP_Rewind */
001199 int addrCont; /* Jump here to skip a row */
001200 const WhereTerm *pTerm; /* For looping over WHERE clause terms */
001201 const WhereTerm *pWCEnd; /* Last WHERE clause term */
001202 Parse *pParse = pWInfo->pParse; /* Parsing context */
001203 Vdbe *v = pParse->pVdbe; /* VDBE under construction */
001204 WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
001205 int iCur; /* Cursor for table getting the filter */
001206 IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */
001207 IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */
001208
001209 saved_pIdxEpr = pParse->pIdxEpr;
001210 saved_pIdxPartExpr = pParse->pIdxPartExpr;
001211 pParse->pIdxEpr = 0;
001212 pParse->pIdxPartExpr = 0;
001213
001214 assert( pLoop!=0 );
001215 assert( v!=0 );
001216 assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
001217 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 );
001218
001219 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
001220 do{
001221 const SrcList *pTabList;
001222 const SrcItem *pItem;
001223 const Table *pTab;
001224 u64 sz;
001225 int iSrc;
001226 sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
001227 addrCont = sqlite3VdbeMakeLabel(pParse);
001228 iCur = pLevel->iTabCur;
001229 pLevel->regFilter = ++pParse->nMem;
001230
001231 /* The Bloom filter is a Blob held in a register. Initialize it
001232 ** to zero-filled blob of at least 80K bits, but maybe more if the
001233 ** estimated size of the table is larger. We could actually
001234 ** measure the size of the table at run-time using OP_Count with
001235 ** P3==1 and use that value to initialize the blob. But that makes
001236 ** testing complicated. By basing the blob size on the value in the
001237 ** sqlite_stat1 table, testing is much easier.
001238 */
001239 pTabList = pWInfo->pTabList;
001240 iSrc = pLevel->iFrom;
001241 pItem = &pTabList->a[iSrc];
001242 assert( pItem!=0 );
001243 pTab = pItem->pTab;
001244 assert( pTab!=0 );
001245 sz = sqlite3LogEstToInt(pTab->nRowLogEst);
001246 if( sz<10000 ){
001247 sz = 10000;
001248 }else if( sz>10000000 ){
001249 sz = 10000000;
001250 }
001251 sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
001252
001253 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
001254 pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
001255 for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
001256 Expr *pExpr = pTerm->pExpr;
001257 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
001258 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0)
001259 ){
001260 sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
001261 }
001262 }
001263 if( pLoop->wsFlags & WHERE_IPK ){
001264 int r1 = sqlite3GetTempReg(pParse);
001265 sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
001266 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
001267 sqlite3ReleaseTempReg(pParse, r1);
001268 }else{
001269 Index *pIdx = pLoop->u.btree.pIndex;
001270 int n = pLoop->u.btree.nEq;
001271 int r1 = sqlite3GetTempRange(pParse, n);
001272 int jj;
001273 for(jj=0; jj<n; jj++){
001274 assert( pIdx->pTable==pItem->pTab );
001275 sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj);
001276 }
001277 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
001278 sqlite3ReleaseTempRange(pParse, r1, n);
001279 }
001280 sqlite3VdbeResolveLabel(v, addrCont);
001281 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
001282 VdbeCoverage(v);
001283 sqlite3VdbeJumpHere(v, addrTop);
001284 pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
001285 if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
001286 while( ++iLevel < pWInfo->nLevel ){
001287 const SrcItem *pTabItem;
001288 pLevel = &pWInfo->a[iLevel];
001289 pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
001290 if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue;
001291 pLoop = pLevel->pWLoop;
001292 if( NEVER(pLoop==0) ) continue;
001293 if( pLoop->prereq & notReady ) continue;
001294 if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN))
001295 ==WHERE_BLOOMFILTER
001296 ){
001297 /* This is a candidate for bloom-filter pull-down (early evaluation).
001298 ** The test that WHERE_COLUMN_IN is omitted is important, as we are
001299 ** not able to do early evaluation of bloom filters that make use of
001300 ** the IN operator */
001301 break;
001302 }
001303 }
001304 }while( iLevel < pWInfo->nLevel );
001305 sqlite3VdbeJumpHere(v, addrOnce);
001306 pParse->pIdxEpr = saved_pIdxEpr;
001307 pParse->pIdxPartExpr = saved_pIdxPartExpr;
001308 }
001309
001310
001311 #ifndef SQLITE_OMIT_VIRTUALTABLE
001312 /*
001313 ** Allocate and populate an sqlite3_index_info structure. It is the
001314 ** responsibility of the caller to eventually release the structure
001315 ** by passing the pointer returned by this function to freeIndexInfo().
001316 */
001317 static sqlite3_index_info *allocateIndexInfo(
001318 WhereInfo *pWInfo, /* The WHERE clause */
001319 WhereClause *pWC, /* The WHERE clause being analyzed */
001320 Bitmask mUnusable, /* Ignore terms with these prereqs */
001321 SrcItem *pSrc, /* The FROM clause term that is the vtab */
001322 u16 *pmNoOmit /* Mask of terms not to omit */
001323 ){
001324 int i, j;
001325 int nTerm;
001326 Parse *pParse = pWInfo->pParse;
001327 struct sqlite3_index_constraint *pIdxCons;
001328 struct sqlite3_index_orderby *pIdxOrderBy;
001329 struct sqlite3_index_constraint_usage *pUsage;
001330 struct HiddenIndexInfo *pHidden;
001331 WhereTerm *pTerm;
001332 int nOrderBy;
001333 sqlite3_index_info *pIdxInfo;
001334 u16 mNoOmit = 0;
001335 const Table *pTab;
001336 int eDistinct = 0;
001337 ExprList *pOrderBy = pWInfo->pOrderBy;
001338
001339 assert( pSrc!=0 );
001340 pTab = pSrc->pTab;
001341 assert( pTab!=0 );
001342 assert( IsVirtual(pTab) );
001343
001344 /* Find all WHERE clause constraints referring to this virtual table.
001345 ** Mark each term with the TERM_OK flag. Set nTerm to the number of
001346 ** terms found.
001347 */
001348 for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
001349 pTerm->wtFlags &= ~TERM_OK;
001350 if( pTerm->leftCursor != pSrc->iCursor ) continue;
001351 if( pTerm->prereqRight & mUnusable ) continue;
001352 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
001353 testcase( pTerm->eOperator & WO_IN );
001354 testcase( pTerm->eOperator & WO_ISNULL );
001355 testcase( pTerm->eOperator & WO_IS );
001356 testcase( pTerm->eOperator & WO_ALL );
001357 if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
001358 if( pTerm->wtFlags & TERM_VNULL ) continue;
001359
001360 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001361 assert( pTerm->u.x.leftColumn>=XN_ROWID );
001362 assert( pTerm->u.x.leftColumn<pTab->nCol );
001363 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
001364 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
001365 ){
001366 continue;
001367 }
001368 nTerm++;
001369 pTerm->wtFlags |= TERM_OK;
001370 }
001371
001372 /* If the ORDER BY clause contains only columns in the current
001373 ** virtual table then allocate space for the aOrderBy part of
001374 ** the sqlite3_index_info structure.
001375 */
001376 nOrderBy = 0;
001377 if( pOrderBy ){
001378 int n = pOrderBy->nExpr;
001379 for(i=0; i<n; i++){
001380 Expr *pExpr = pOrderBy->a[i].pExpr;
001381 Expr *pE2;
001382
001383 /* Skip over constant terms in the ORDER BY clause */
001384 if( sqlite3ExprIsConstant(0, pExpr) ){
001385 continue;
001386 }
001387
001388 /* Virtual tables are unable to deal with NULLS FIRST */
001389 if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break;
001390
001391 /* First case - a direct column references without a COLLATE operator */
001392 if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){
001393 assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol );
001394 continue;
001395 }
001396
001397 /* 2nd case - a column reference with a COLLATE operator. Only match
001398 ** of the COLLATE operator matches the collation of the column. */
001399 if( pExpr->op==TK_COLLATE
001400 && (pE2 = pExpr->pLeft)->op==TK_COLUMN
001401 && pE2->iTable==pSrc->iCursor
001402 ){
001403 const char *zColl; /* The collating sequence name */
001404 assert( !ExprHasProperty(pExpr, EP_IntValue) );
001405 assert( pExpr->u.zToken!=0 );
001406 assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol );
001407 pExpr->iColumn = pE2->iColumn;
001408 if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */
001409 zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]);
001410 if( zColl==0 ) zColl = sqlite3StrBINARY;
001411 if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue;
001412 }
001413
001414 /* No matches cause a break out of the loop */
001415 break;
001416 }
001417 if( i==n ){
001418 nOrderBy = n;
001419 if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){
001420 eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
001421 }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
001422 eDistinct = 1;
001423 }
001424 }
001425 }
001426
001427 /* Allocate the sqlite3_index_info structure
001428 */
001429 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
001430 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
001431 + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden)
001432 + sizeof(sqlite3_value*)*nTerm );
001433 if( pIdxInfo==0 ){
001434 sqlite3ErrorMsg(pParse, "out of memory");
001435 return 0;
001436 }
001437 pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
001438 pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm];
001439 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
001440 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
001441 pIdxInfo->aConstraint = pIdxCons;
001442 pIdxInfo->aOrderBy = pIdxOrderBy;
001443 pIdxInfo->aConstraintUsage = pUsage;
001444 pHidden->pWC = pWC;
001445 pHidden->pParse = pParse;
001446 pHidden->eDistinct = eDistinct;
001447 pHidden->mIn = 0;
001448 for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
001449 u16 op;
001450 if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
001451 pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
001452 pIdxCons[j].iTermOffset = i;
001453 op = pTerm->eOperator & WO_ALL;
001454 if( op==WO_IN ){
001455 if( (pTerm->wtFlags & TERM_SLICE)==0 ){
001456 pHidden->mIn |= SMASKBIT32(j);
001457 }
001458 op = WO_EQ;
001459 }
001460 if( op==WO_AUX ){
001461 pIdxCons[j].op = pTerm->eMatchOp;
001462 }else if( op & (WO_ISNULL|WO_IS) ){
001463 if( op==WO_ISNULL ){
001464 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
001465 }else{
001466 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
001467 }
001468 }else{
001469 pIdxCons[j].op = (u8)op;
001470 /* The direct assignment in the previous line is possible only because
001471 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
001472 ** following asserts verify this fact. */
001473 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
001474 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
001475 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
001476 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
001477 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
001478 assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
001479
001480 if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
001481 && sqlite3ExprIsVector(pTerm->pExpr->pRight)
001482 ){
001483 testcase( j!=i );
001484 if( j<16 ) mNoOmit |= (1 << j);
001485 if( op==WO_LT ) pIdxCons[j].op = WO_LE;
001486 if( op==WO_GT ) pIdxCons[j].op = WO_GE;
001487 }
001488 }
001489
001490 j++;
001491 }
001492 assert( j==nTerm );
001493 pIdxInfo->nConstraint = j;
001494 for(i=j=0; i<nOrderBy; i++){
001495 Expr *pExpr = pOrderBy->a[i].pExpr;
001496 if( sqlite3ExprIsConstant(0, pExpr) ) continue;
001497 assert( pExpr->op==TK_COLUMN
001498 || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN
001499 && pExpr->iColumn==pExpr->pLeft->iColumn) );
001500 pIdxOrderBy[j].iColumn = pExpr->iColumn;
001501 pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC;
001502 j++;
001503 }
001504 pIdxInfo->nOrderBy = j;
001505
001506 *pmNoOmit = mNoOmit;
001507 return pIdxInfo;
001508 }
001509
001510 /*
001511 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
001512 ** and possibly modified by xBestIndex methods.
001513 */
001514 static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
001515 HiddenIndexInfo *pHidden;
001516 int i;
001517 assert( pIdxInfo!=0 );
001518 pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
001519 assert( pHidden->pParse!=0 );
001520 assert( pHidden->pParse->db==db );
001521 for(i=0; i<pIdxInfo->nConstraint; i++){
001522 sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */
001523 pHidden->aRhs[i] = 0;
001524 }
001525 sqlite3DbFree(db, pIdxInfo);
001526 }
001527
001528 /*
001529 ** The table object reference passed as the second argument to this function
001530 ** must represent a virtual table. This function invokes the xBestIndex()
001531 ** method of the virtual table with the sqlite3_index_info object that
001532 ** comes in as the 3rd argument to this function.
001533 **
001534 ** If an error occurs, pParse is populated with an error message and an
001535 ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from
001536 ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that
001537 ** the current configuration of "unusable" flags in sqlite3_index_info can
001538 ** not result in a valid plan.
001539 **
001540 ** Whether or not an error is returned, it is the responsibility of the
001541 ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
001542 ** that this is required.
001543 */
001544 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
001545 sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
001546 int rc;
001547
001548 whereTraceIndexInfoInputs(p, pTab);
001549 pParse->db->nSchemaLock++;
001550 rc = pVtab->pModule->xBestIndex(pVtab, p);
001551 pParse->db->nSchemaLock--;
001552 whereTraceIndexInfoOutputs(p, pTab);
001553
001554 if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
001555 if( rc==SQLITE_NOMEM ){
001556 sqlite3OomFault(pParse->db);
001557 }else if( !pVtab->zErrMsg ){
001558 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
001559 }else{
001560 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
001561 }
001562 }
001563 if( pTab->u.vtab.p->bAllSchemas ){
001564 sqlite3VtabUsesAllSchemas(pParse);
001565 }
001566 sqlite3_free(pVtab->zErrMsg);
001567 pVtab->zErrMsg = 0;
001568 return rc;
001569 }
001570 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
001571
001572 #ifdef SQLITE_ENABLE_STAT4
001573 /*
001574 ** Estimate the location of a particular key among all keys in an
001575 ** index. Store the results in aStat as follows:
001576 **
001577 ** aStat[0] Est. number of rows less than pRec
001578 ** aStat[1] Est. number of rows equal to pRec
001579 **
001580 ** Return the index of the sample that is the smallest sample that
001581 ** is greater than or equal to pRec. Note that this index is not an index
001582 ** into the aSample[] array - it is an index into a virtual set of samples
001583 ** based on the contents of aSample[] and the number of fields in record
001584 ** pRec.
001585 */
001586 static int whereKeyStats(
001587 Parse *pParse, /* Database connection */
001588 Index *pIdx, /* Index to consider domain of */
001589 UnpackedRecord *pRec, /* Vector of values to consider */
001590 int roundUp, /* Round up if true. Round down if false */
001591 tRowcnt *aStat /* OUT: stats written here */
001592 ){
001593 IndexSample *aSample = pIdx->aSample;
001594 int iCol; /* Index of required stats in anEq[] etc. */
001595 int i; /* Index of first sample >= pRec */
001596 int iSample; /* Smallest sample larger than or equal to pRec */
001597 int iMin = 0; /* Smallest sample not yet tested */
001598 int iTest; /* Next sample to test */
001599 int res; /* Result of comparison operation */
001600 int nField; /* Number of fields in pRec */
001601 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */
001602
001603 #ifndef SQLITE_DEBUG
001604 UNUSED_PARAMETER( pParse );
001605 #endif
001606 assert( pRec!=0 );
001607 assert( pIdx->nSample>0 );
001608 assert( pRec->nField>0 );
001609
001610
001611 /* Do a binary search to find the first sample greater than or equal
001612 ** to pRec. If pRec contains a single field, the set of samples to search
001613 ** is simply the aSample[] array. If the samples in aSample[] contain more
001614 ** than one fields, all fields following the first are ignored.
001615 **
001616 ** If pRec contains N fields, where N is more than one, then as well as the
001617 ** samples in aSample[] (truncated to N fields), the search also has to
001618 ** consider prefixes of those samples. For example, if the set of samples
001619 ** in aSample is:
001620 **
001621 ** aSample[0] = (a, 5)
001622 ** aSample[1] = (a, 10)
001623 ** aSample[2] = (b, 5)
001624 ** aSample[3] = (c, 100)
001625 ** aSample[4] = (c, 105)
001626 **
001627 ** Then the search space should ideally be the samples above and the
001628 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
001629 ** the code actually searches this set:
001630 **
001631 ** 0: (a)
001632 ** 1: (a, 5)
001633 ** 2: (a, 10)
001634 ** 3: (a, 10)
001635 ** 4: (b)
001636 ** 5: (b, 5)
001637 ** 6: (c)
001638 ** 7: (c, 100)
001639 ** 8: (c, 105)
001640 ** 9: (c, 105)
001641 **
001642 ** For each sample in the aSample[] array, N samples are present in the
001643 ** effective sample array. In the above, samples 0 and 1 are based on
001644 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
001645 **
001646 ** Often, sample i of each block of N effective samples has (i+1) fields.
001647 ** Except, each sample may be extended to ensure that it is greater than or
001648 ** equal to the previous sample in the array. For example, in the above,
001649 ** sample 2 is the first sample of a block of N samples, so at first it
001650 ** appears that it should be 1 field in size. However, that would make it
001651 ** smaller than sample 1, so the binary search would not work. As a result,
001652 ** it is extended to two fields. The duplicates that this creates do not
001653 ** cause any problems.
001654 */
001655 if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
001656 nField = pIdx->nKeyCol;
001657 }else{
001658 nField = pIdx->nColumn;
001659 }
001660 nField = MIN(pRec->nField, nField);
001661 iCol = 0;
001662 iSample = pIdx->nSample * nField;
001663 do{
001664 int iSamp; /* Index in aSample[] of test sample */
001665 int n; /* Number of fields in test sample */
001666
001667 iTest = (iMin+iSample)/2;
001668 iSamp = iTest / nField;
001669 if( iSamp>0 ){
001670 /* The proposed effective sample is a prefix of sample aSample[iSamp].
001671 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
001672 ** fields that is greater than the previous effective sample. */
001673 for(n=(iTest % nField) + 1; n<nField; n++){
001674 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
001675 }
001676 }else{
001677 n = iTest + 1;
001678 }
001679
001680 pRec->nField = n;
001681 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec);
001682 if( res<0 ){
001683 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1];
001684 iMin = iTest+1;
001685 }else if( res==0 && n<nField ){
001686 iLower = aSample[iSamp].anLt[n-1];
001687 iMin = iTest+1;
001688 res = -1;
001689 }else{
001690 iSample = iTest;
001691 iCol = n-1;
001692 }
001693 }while( res && iMin<iSample );
001694 i = iSample / nField;
001695
001696 #ifdef SQLITE_DEBUG
001697 /* The following assert statements check that the binary search code
001698 ** above found the right answer. This block serves no purpose other
001699 ** than to invoke the asserts. */
001700 if( pParse->db->mallocFailed==0 ){
001701 if( res==0 ){
001702 /* If (res==0) is true, then pRec must be equal to sample i. */
001703 assert( i<pIdx->nSample );
001704 assert( iCol==nField-1 );
001705 pRec->nField = nField;
001706 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
001707 || pParse->db->mallocFailed
001708 );
001709 }else{
001710 /* Unless i==pIdx->nSample, indicating that pRec is larger than
001711 ** all samples in the aSample[] array, pRec must be smaller than the
001712 ** (iCol+1) field prefix of sample i. */
001713 assert( i<=pIdx->nSample && i>=0 );
001714 pRec->nField = iCol+1;
001715 assert( i==pIdx->nSample
001716 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
001717 || pParse->db->mallocFailed );
001718
001719 /* if i==0 and iCol==0, then record pRec is smaller than all samples
001720 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
001721 ** be greater than or equal to the (iCol) field prefix of sample i.
001722 ** If (i>0), then pRec must also be greater than sample (i-1). */
001723 if( iCol>0 ){
001724 pRec->nField = iCol;
001725 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0
001726 || pParse->db->mallocFailed || CORRUPT_DB );
001727 }
001728 if( i>0 ){
001729 pRec->nField = nField;
001730 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0
001731 || pParse->db->mallocFailed || CORRUPT_DB );
001732 }
001733 }
001734 }
001735 #endif /* ifdef SQLITE_DEBUG */
001736
001737 if( res==0 ){
001738 /* Record pRec is equal to sample i */
001739 assert( iCol==nField-1 );
001740 aStat[0] = aSample[i].anLt[iCol];
001741 aStat[1] = aSample[i].anEq[iCol];
001742 }else{
001743 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
001744 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
001745 ** is larger than all samples in the array. */
001746 tRowcnt iUpper, iGap;
001747 if( i>=pIdx->nSample ){
001748 iUpper = pIdx->nRowEst0;
001749 }else{
001750 iUpper = aSample[i].anLt[iCol];
001751 }
001752
001753 if( iLower>=iUpper ){
001754 iGap = 0;
001755 }else{
001756 iGap = iUpper - iLower;
001757 }
001758 if( roundUp ){
001759 iGap = (iGap*2)/3;
001760 }else{
001761 iGap = iGap/3;
001762 }
001763 aStat[0] = iLower + iGap;
001764 aStat[1] = pIdx->aAvgEq[nField-1];
001765 }
001766
001767 /* Restore the pRec->nField value before returning. */
001768 pRec->nField = nField;
001769 return i;
001770 }
001771 #endif /* SQLITE_ENABLE_STAT4 */
001772
001773 /*
001774 ** If it is not NULL, pTerm is a term that provides an upper or lower
001775 ** bound on a range scan. Without considering pTerm, it is estimated
001776 ** that the scan will visit nNew rows. This function returns the number
001777 ** estimated to be visited after taking pTerm into account.
001778 **
001779 ** If the user explicitly specified a likelihood() value for this term,
001780 ** then the return value is the likelihood multiplied by the number of
001781 ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
001782 ** has a likelihood of 0.50, and any other term a likelihood of 0.25.
001783 */
001784 static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
001785 LogEst nRet = nNew;
001786 if( pTerm ){
001787 if( pTerm->truthProb<=0 ){
001788 nRet += pTerm->truthProb;
001789 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){
001790 nRet -= 20; assert( 20==sqlite3LogEst(4) );
001791 }
001792 }
001793 return nRet;
001794 }
001795
001796
001797 #ifdef SQLITE_ENABLE_STAT4
001798 /*
001799 ** Return the affinity for a single column of an index.
001800 */
001801 char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
001802 assert( iCol>=0 && iCol<pIdx->nColumn );
001803 if( !pIdx->zColAff ){
001804 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
001805 }
001806 assert( pIdx->zColAff[iCol]!=0 );
001807 return pIdx->zColAff[iCol];
001808 }
001809 #endif
001810
001811
001812 #ifdef SQLITE_ENABLE_STAT4
001813 /*
001814 ** This function is called to estimate the number of rows visited by a
001815 ** range-scan on a skip-scan index. For example:
001816 **
001817 ** CREATE INDEX i1 ON t1(a, b, c);
001818 ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
001819 **
001820 ** Value pLoop->nOut is currently set to the estimated number of rows
001821 ** visited for scanning (a=? AND b=?). This function reduces that estimate
001822 ** by some factor to account for the (c BETWEEN ? AND ?) expression based
001823 ** on the stat4 data for the index. this scan will be performed multiple
001824 ** times (once for each (a,b) combination that matches a=?) is dealt with
001825 ** by the caller.
001826 **
001827 ** It does this by scanning through all stat4 samples, comparing values
001828 ** extracted from pLower and pUpper with the corresponding column in each
001829 ** sample. If L and U are the number of samples found to be less than or
001830 ** equal to the values extracted from pLower and pUpper respectively, and
001831 ** N is the total number of samples, the pLoop->nOut value is adjusted
001832 ** as follows:
001833 **
001834 ** nOut = nOut * ( min(U - L, 1) / N )
001835 **
001836 ** If pLower is NULL, or a value cannot be extracted from the term, L is
001837 ** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
001838 ** U is set to N.
001839 **
001840 ** Normally, this function sets *pbDone to 1 before returning. However,
001841 ** if no value can be extracted from either pLower or pUpper (and so the
001842 ** estimate of the number of rows delivered remains unchanged), *pbDone
001843 ** is left as is.
001844 **
001845 ** If an error occurs, an SQLite error code is returned. Otherwise,
001846 ** SQLITE_OK.
001847 */
001848 static int whereRangeSkipScanEst(
001849 Parse *pParse, /* Parsing & code generating context */
001850 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001851 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001852 WhereLoop *pLoop, /* Update the .nOut value of this loop */
001853 int *pbDone /* Set to true if at least one expr. value extracted */
001854 ){
001855 Index *p = pLoop->u.btree.pIndex;
001856 int nEq = pLoop->u.btree.nEq;
001857 sqlite3 *db = pParse->db;
001858 int nLower = -1;
001859 int nUpper = p->nSample+1;
001860 int rc = SQLITE_OK;
001861 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
001862 CollSeq *pColl;
001863
001864 sqlite3_value *p1 = 0; /* Value extracted from pLower */
001865 sqlite3_value *p2 = 0; /* Value extracted from pUpper */
001866 sqlite3_value *pVal = 0; /* Value extracted from record */
001867
001868 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]);
001869 if( pLower ){
001870 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1);
001871 nLower = 0;
001872 }
001873 if( pUpper && rc==SQLITE_OK ){
001874 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2);
001875 nUpper = p2 ? 0 : p->nSample;
001876 }
001877
001878 if( p1 || p2 ){
001879 int i;
001880 int nDiff;
001881 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){
001882 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal);
001883 if( rc==SQLITE_OK && p1 ){
001884 int res = sqlite3MemCompare(p1, pVal, pColl);
001885 if( res>=0 ) nLower++;
001886 }
001887 if( rc==SQLITE_OK && p2 ){
001888 int res = sqlite3MemCompare(p2, pVal, pColl);
001889 if( res>=0 ) nUpper++;
001890 }
001891 }
001892 nDiff = (nUpper - nLower);
001893 if( nDiff<=0 ) nDiff = 1;
001894
001895 /* If there is both an upper and lower bound specified, and the
001896 ** comparisons indicate that they are close together, use the fallback
001897 ** method (assume that the scan visits 1/64 of the rows) for estimating
001898 ** the number of rows visited. Otherwise, estimate the number of rows
001899 ** using the method described in the header comment for this function. */
001900 if( nDiff!=1 || pUpper==0 || pLower==0 ){
001901 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff));
001902 pLoop->nOut -= nAdjust;
001903 *pbDone = 1;
001904 WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
001905 nLower, nUpper, nAdjust*-1, pLoop->nOut));
001906 }
001907
001908 }else{
001909 assert( *pbDone==0 );
001910 }
001911
001912 sqlite3ValueFree(p1);
001913 sqlite3ValueFree(p2);
001914 sqlite3ValueFree(pVal);
001915
001916 return rc;
001917 }
001918 #endif /* SQLITE_ENABLE_STAT4 */
001919
001920 /*
001921 ** This function is used to estimate the number of rows that will be visited
001922 ** by scanning an index for a range of values. The range may have an upper
001923 ** bound, a lower bound, or both. The WHERE clause terms that set the upper
001924 ** and lower bounds are represented by pLower and pUpper respectively. For
001925 ** example, assuming that index p is on t1(a):
001926 **
001927 ** ... FROM t1 WHERE a > ? AND a < ? ...
001928 ** |_____| |_____|
001929 ** | |
001930 ** pLower pUpper
001931 **
001932 ** If either of the upper or lower bound is not present, then NULL is passed in
001933 ** place of the corresponding WhereTerm.
001934 **
001935 ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
001936 ** column subject to the range constraint. Or, equivalently, the number of
001937 ** equality constraints optimized by the proposed index scan. For example,
001938 ** assuming index p is on t1(a, b), and the SQL query is:
001939 **
001940 ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
001941 **
001942 ** then nEq is set to 1 (as the range restricted column, b, is the second
001943 ** left-most column of the index). Or, if the query is:
001944 **
001945 ** ... FROM t1 WHERE a > ? AND a < ? ...
001946 **
001947 ** then nEq is set to 0.
001948 **
001949 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the
001950 ** number of rows that the index scan is expected to visit without
001951 ** considering the range constraints. If nEq is 0, then *pnOut is the number of
001952 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
001953 ** to account for the range constraints pLower and pUpper.
001954 **
001955 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
001956 ** used, a single range inequality reduces the search space by a factor of 4.
001957 ** and a pair of constraints (x>? AND x<?) reduces the expected number of
001958 ** rows visited by a factor of 64.
001959 */
001960 static int whereRangeScanEst(
001961 Parse *pParse, /* Parsing & code generating context */
001962 WhereLoopBuilder *pBuilder,
001963 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001964 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001965 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */
001966 ){
001967 int rc = SQLITE_OK;
001968 int nOut = pLoop->nOut;
001969 LogEst nNew;
001970
001971 #ifdef SQLITE_ENABLE_STAT4
001972 Index *p = pLoop->u.btree.pIndex;
001973 int nEq = pLoop->u.btree.nEq;
001974
001975 if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol)
001976 && OptimizationEnabled(pParse->db, SQLITE_Stat4)
001977 ){
001978 if( nEq==pBuilder->nRecValid ){
001979 UnpackedRecord *pRec = pBuilder->pRec;
001980 tRowcnt a[2];
001981 int nBtm = pLoop->u.btree.nBtm;
001982 int nTop = pLoop->u.btree.nTop;
001983
001984 /* Variable iLower will be set to the estimate of the number of rows in
001985 ** the index that are less than the lower bound of the range query. The
001986 ** lower bound being the concatenation of $P and $L, where $P is the
001987 ** key-prefix formed by the nEq values matched against the nEq left-most
001988 ** columns of the index, and $L is the value in pLower.
001989 **
001990 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
001991 ** is not a simple variable or literal value), the lower bound of the
001992 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
001993 ** if $L is available, whereKeyStats() is called for both ($P) and
001994 ** ($P:$L) and the larger of the two returned values is used.
001995 **
001996 ** Similarly, iUpper is to be set to the estimate of the number of rows
001997 ** less than the upper bound of the range query. Where the upper bound
001998 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
001999 ** of iUpper are requested of whereKeyStats() and the smaller used.
002000 **
002001 ** The number of rows between the two bounds is then just iUpper-iLower.
002002 */
002003 tRowcnt iLower; /* Rows less than the lower bound */
002004 tRowcnt iUpper; /* Rows less than the upper bound */
002005 int iLwrIdx = -2; /* aSample[] for the lower bound */
002006 int iUprIdx = -1; /* aSample[] for the upper bound */
002007
002008 if( pRec ){
002009 testcase( pRec->nField!=pBuilder->nRecValid );
002010 pRec->nField = pBuilder->nRecValid;
002011 }
002012 /* Determine iLower and iUpper using ($P) only. */
002013 if( nEq==0 ){
002014 iLower = 0;
002015 iUpper = p->nRowEst0;
002016 }else{
002017 /* Note: this call could be optimized away - since the same values must
002018 ** have been requested when testing key $P in whereEqualScanEst(). */
002019 whereKeyStats(pParse, p, pRec, 0, a);
002020 iLower = a[0];
002021 iUpper = a[0] + a[1];
002022 }
002023
002024 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
002025 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
002026 assert( p->aSortOrder!=0 );
002027 if( p->aSortOrder[nEq] ){
002028 /* The roles of pLower and pUpper are swapped for a DESC index */
002029 SWAP(WhereTerm*, pLower, pUpper);
002030 SWAP(int, nBtm, nTop);
002031 }
002032
002033 /* If possible, improve on the iLower estimate using ($P:$L). */
002034 if( pLower ){
002035 int n; /* Values extracted from pExpr */
002036 Expr *pExpr = pLower->pExpr->pRight;
002037 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
002038 if( rc==SQLITE_OK && n ){
002039 tRowcnt iNew;
002040 u16 mask = WO_GT|WO_LE;
002041 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
002042 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
002043 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
002044 if( iNew>iLower ) iLower = iNew;
002045 nOut--;
002046 pLower = 0;
002047 }
002048 }
002049
002050 /* If possible, improve on the iUpper estimate using ($P:$U). */
002051 if( pUpper ){
002052 int n; /* Values extracted from pExpr */
002053 Expr *pExpr = pUpper->pExpr->pRight;
002054 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
002055 if( rc==SQLITE_OK && n ){
002056 tRowcnt iNew;
002057 u16 mask = WO_GT|WO_LE;
002058 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
002059 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
002060 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
002061 if( iNew<iUpper ) iUpper = iNew;
002062 nOut--;
002063 pUpper = 0;
002064 }
002065 }
002066
002067 pBuilder->pRec = pRec;
002068 if( rc==SQLITE_OK ){
002069 if( iUpper>iLower ){
002070 nNew = sqlite3LogEst(iUpper - iLower);
002071 /* TUNING: If both iUpper and iLower are derived from the same
002072 ** sample, then assume they are 4x more selective. This brings
002073 ** the estimated selectivity more in line with what it would be
002074 ** if estimated without the use of STAT4 tables. */
002075 if( iLwrIdx==iUprIdx ){ nNew -= 20; }
002076 assert( 20==sqlite3LogEst(4) );
002077 }else{
002078 nNew = 10; assert( 10==sqlite3LogEst(2) );
002079 }
002080 if( nNew<nOut ){
002081 nOut = nNew;
002082 }
002083 WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n",
002084 (u32)iLower, (u32)iUpper, nOut));
002085 }
002086 }else{
002087 int bDone = 0;
002088 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone);
002089 if( bDone ) return rc;
002090 }
002091 }
002092 #else
002093 UNUSED_PARAMETER(pParse);
002094 UNUSED_PARAMETER(pBuilder);
002095 assert( pLower || pUpper );
002096 #endif
002097 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 );
002098 nNew = whereRangeAdjust(pLower, nOut);
002099 nNew = whereRangeAdjust(pUpper, nNew);
002100
002101 /* TUNING: If there is both an upper and lower limit and neither limit
002102 ** has an application-defined likelihood(), assume the range is
002103 ** reduced by an additional 75%. This means that, by default, an open-ended
002104 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
002105 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
002106 ** match 1/64 of the index. */
002107 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
002108 nNew -= 20;
002109 }
002110
002111 nOut -= (pLower!=0) + (pUpper!=0);
002112 if( nNew<10 ) nNew = 10;
002113 if( nNew<nOut ) nOut = nNew;
002114 #if defined(WHERETRACE_ENABLED)
002115 if( pLoop->nOut>nOut ){
002116 WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n",
002117 pLoop->nOut, nOut));
002118 }
002119 #endif
002120 pLoop->nOut = (LogEst)nOut;
002121 return rc;
002122 }
002123
002124 #ifdef SQLITE_ENABLE_STAT4
002125 /*
002126 ** Estimate the number of rows that will be returned based on
002127 ** an equality constraint x=VALUE and where that VALUE occurs in
002128 ** the histogram data. This only works when x is the left-most
002129 ** column of an index and sqlite_stat4 histogram data is available
002130 ** for that index. When pExpr==NULL that means the constraint is
002131 ** "x IS NULL" instead of "x=VALUE".
002132 **
002133 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002134 ** If unable to make an estimate, leave *pnRow unchanged and return
002135 ** non-zero.
002136 **
002137 ** This routine can fail if it is unable to load a collating sequence
002138 ** required for string comparison, or if unable to allocate memory
002139 ** for a UTF conversion required for comparison. The error is stored
002140 ** in the pParse structure.
002141 */
002142 static int whereEqualScanEst(
002143 Parse *pParse, /* Parsing & code generating context */
002144 WhereLoopBuilder *pBuilder,
002145 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
002146 tRowcnt *pnRow /* Write the revised row estimate here */
002147 ){
002148 Index *p = pBuilder->pNew->u.btree.pIndex;
002149 int nEq = pBuilder->pNew->u.btree.nEq;
002150 UnpackedRecord *pRec = pBuilder->pRec;
002151 int rc; /* Subfunction return code */
002152 tRowcnt a[2]; /* Statistics */
002153 int bOk;
002154
002155 assert( nEq>=1 );
002156 assert( nEq<=p->nColumn );
002157 assert( p->aSample!=0 );
002158 assert( p->nSample>0 );
002159 assert( pBuilder->nRecValid<nEq );
002160
002161 /* If values are not available for all fields of the index to the left
002162 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
002163 if( pBuilder->nRecValid<(nEq-1) ){
002164 return SQLITE_NOTFOUND;
002165 }
002166
002167 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
002168 ** below would return the same value. */
002169 if( nEq>=p->nColumn ){
002170 *pnRow = 1;
002171 return SQLITE_OK;
002172 }
002173
002174 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
002175 pBuilder->pRec = pRec;
002176 if( rc!=SQLITE_OK ) return rc;
002177 if( bOk==0 ) return SQLITE_NOTFOUND;
002178 pBuilder->nRecValid = nEq;
002179
002180 whereKeyStats(pParse, p, pRec, 0, a);
002181 WHERETRACE(0x20,("equality scan regions %s(%d): %d\n",
002182 p->zName, nEq-1, (int)a[1]));
002183 *pnRow = a[1];
002184
002185 return rc;
002186 }
002187 #endif /* SQLITE_ENABLE_STAT4 */
002188
002189 #ifdef SQLITE_ENABLE_STAT4
002190 /*
002191 ** Estimate the number of rows that will be returned based on
002192 ** an IN constraint where the right-hand side of the IN operator
002193 ** is a list of values. Example:
002194 **
002195 ** WHERE x IN (1,2,3,4)
002196 **
002197 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002198 ** If unable to make an estimate, leave *pnRow unchanged and return
002199 ** non-zero.
002200 **
002201 ** This routine can fail if it is unable to load a collating sequence
002202 ** required for string comparison, or if unable to allocate memory
002203 ** for a UTF conversion required for comparison. The error is stored
002204 ** in the pParse structure.
002205 */
002206 static int whereInScanEst(
002207 Parse *pParse, /* Parsing & code generating context */
002208 WhereLoopBuilder *pBuilder,
002209 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
002210 tRowcnt *pnRow /* Write the revised row estimate here */
002211 ){
002212 Index *p = pBuilder->pNew->u.btree.pIndex;
002213 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]);
002214 int nRecValid = pBuilder->nRecValid;
002215 int rc = SQLITE_OK; /* Subfunction return code */
002216 tRowcnt nEst; /* Number of rows for a single term */
002217 tRowcnt nRowEst = 0; /* New estimate of the number of rows */
002218 int i; /* Loop counter */
002219
002220 assert( p->aSample!=0 );
002221 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
002222 nEst = nRow0;
002223 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst);
002224 nRowEst += nEst;
002225 pBuilder->nRecValid = nRecValid;
002226 }
002227
002228 if( rc==SQLITE_OK ){
002229 if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0;
002230 *pnRow = nRowEst;
002231 WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst));
002232 }
002233 assert( pBuilder->nRecValid==nRecValid );
002234 return rc;
002235 }
002236 #endif /* SQLITE_ENABLE_STAT4 */
002237
002238
002239 #ifdef WHERETRACE_ENABLED
002240 /*
002241 ** Print the content of a WhereTerm object
002242 */
002243 void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
002244 if( pTerm==0 ){
002245 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
002246 }else{
002247 char zType[8];
002248 char zLeft[50];
002249 memcpy(zType, "....", 5);
002250 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
002251 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E';
002252 if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L';
002253 if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C';
002254 if( pTerm->eOperator & WO_SINGLE ){
002255 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
002256 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
002257 pTerm->leftCursor, pTerm->u.x.leftColumn);
002258 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
002259 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx",
002260 pTerm->u.pOrInfo->indexable);
002261 }else{
002262 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
002263 }
002264 sqlite3DebugPrintf(
002265 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
002266 iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags);
002267 /* The 0x10000 .wheretrace flag causes extra information to be
002268 ** shown about each Term */
002269 if( sqlite3WhereTrace & 0x10000 ){
002270 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
002271 pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight);
002272 }
002273 if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){
002274 sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField);
002275 }
002276 if( pTerm->iParent>=0 ){
002277 sqlite3DebugPrintf(" iParent=%d", pTerm->iParent);
002278 }
002279 sqlite3DebugPrintf("\n");
002280 sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
002281 }
002282 }
002283 #endif
002284
002285 #ifdef WHERETRACE_ENABLED
002286 /*
002287 ** Show the complete content of a WhereClause
002288 */
002289 void sqlite3WhereClausePrint(WhereClause *pWC){
002290 int i;
002291 for(i=0; i<pWC->nTerm; i++){
002292 sqlite3WhereTermPrint(&pWC->a[i], i);
002293 }
002294 }
002295 #endif
002296
002297 #ifdef WHERETRACE_ENABLED
002298 /*
002299 ** Print a WhereLoop object for debugging purposes
002300 **
002301 ** Format example:
002302 **
002303 ** .--- Position in WHERE clause rSetup, rRun, nOut ---.
002304 ** | |
002305 ** | .--- selfMask nTerm ------. |
002306 ** | | | |
002307 ** | | .-- prereq Idx wsFlags----. | |
002308 ** | | | Name | | |
002309 ** | | | __|__ nEq ---. ___|__ | __|__
002310 ** | / \ / \ / \ | / \ / \ / \
002311 ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31
002312 */
002313 void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){
002314 if( pWC ){
002315 WhereInfo *pWInfo = pWC->pWInfo;
002316 int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
002317 SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
002318 Table *pTab = pItem->pTab;
002319 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
002320 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
002321 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
002322 sqlite3DebugPrintf(" %12s",
002323 pItem->zAlias ? pItem->zAlias : pTab->zName);
002324 }else{
002325 sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d",
002326 p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab);
002327 }
002328 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002329 const char *zName;
002330 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
002331 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
002332 int i = sqlite3Strlen30(zName) - 1;
002333 while( zName[i]!='_' ) i--;
002334 zName += i;
002335 }
002336 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
002337 }else{
002338 sqlite3DebugPrintf("%20s","");
002339 }
002340 }else{
002341 char *z;
002342 if( p->u.vtab.idxStr ){
002343 z = sqlite3_mprintf("(%d,\"%s\",%#x)",
002344 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
002345 }else{
002346 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
002347 }
002348 sqlite3DebugPrintf(" %-19s", z);
002349 sqlite3_free(z);
002350 }
002351 if( p->wsFlags & WHERE_SKIPSCAN ){
002352 sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
002353 }else{
002354 sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
002355 }
002356 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
002357 if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){
002358 int i;
002359 for(i=0; i<p->nLTerm; i++){
002360 sqlite3WhereTermPrint(p->aLTerm[i], i);
002361 }
002362 }
002363 }
002364 void sqlite3ShowWhereLoop(const WhereLoop *p){
002365 if( p ) sqlite3WhereLoopPrint(p, 0);
002366 }
002367 void sqlite3ShowWhereLoopList(const WhereLoop *p){
002368 while( p ){
002369 sqlite3ShowWhereLoop(p);
002370 p = p->pNextLoop;
002371 }
002372 }
002373 #endif
002374
002375 /*
002376 ** Convert bulk memory into a valid WhereLoop that can be passed
002377 ** to whereLoopClear harmlessly.
002378 */
002379 static void whereLoopInit(WhereLoop *p){
002380 p->aLTerm = p->aLTermSpace;
002381 p->nLTerm = 0;
002382 p->nLSlot = ArraySize(p->aLTermSpace);
002383 p->wsFlags = 0;
002384 }
002385
002386 /*
002387 ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
002388 */
002389 static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
002390 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
002391 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
002392 sqlite3_free(p->u.vtab.idxStr);
002393 p->u.vtab.needFree = 0;
002394 p->u.vtab.idxStr = 0;
002395 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
002396 sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
002397 sqlite3DbFreeNN(db, p->u.btree.pIndex);
002398 p->u.btree.pIndex = 0;
002399 }
002400 }
002401 }
002402
002403 /*
002404 ** Deallocate internal memory used by a WhereLoop object. Leave the
002405 ** object in an initialized state, as if it had been newly allocated.
002406 */
002407 static void whereLoopClear(sqlite3 *db, WhereLoop *p){
002408 if( p->aLTerm!=p->aLTermSpace ){
002409 sqlite3DbFreeNN(db, p->aLTerm);
002410 p->aLTerm = p->aLTermSpace;
002411 p->nLSlot = ArraySize(p->aLTermSpace);
002412 }
002413 whereLoopClearUnion(db, p);
002414 p->nLTerm = 0;
002415 p->wsFlags = 0;
002416 }
002417
002418 /*
002419 ** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
002420 */
002421 static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
002422 WhereTerm **paNew;
002423 if( p->nLSlot>=n ) return SQLITE_OK;
002424 n = (n+7)&~7;
002425 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
002426 if( paNew==0 ) return SQLITE_NOMEM_BKPT;
002427 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
002428 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm);
002429 p->aLTerm = paNew;
002430 p->nLSlot = n;
002431 return SQLITE_OK;
002432 }
002433
002434 /*
002435 ** Transfer content from the second pLoop into the first.
002436 */
002437 static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
002438 whereLoopClearUnion(db, pTo);
002439 if( pFrom->nLTerm > pTo->nLSlot
002440 && whereLoopResize(db, pTo, pFrom->nLTerm)
002441 ){
002442 memset(pTo, 0, WHERE_LOOP_XFER_SZ);
002443 return SQLITE_NOMEM_BKPT;
002444 }
002445 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
002446 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
002447 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
002448 pFrom->u.vtab.needFree = 0;
002449 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
002450 pFrom->u.btree.pIndex = 0;
002451 }
002452 return SQLITE_OK;
002453 }
002454
002455 /*
002456 ** Delete a WhereLoop object
002457 */
002458 static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
002459 assert( db!=0 );
002460 whereLoopClear(db, p);
002461 sqlite3DbNNFreeNN(db, p);
002462 }
002463
002464 /*
002465 ** Free a WhereInfo structure
002466 */
002467 static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
002468 assert( pWInfo!=0 );
002469 assert( db!=0 );
002470 sqlite3WhereClauseClear(&pWInfo->sWC);
002471 while( pWInfo->pLoops ){
002472 WhereLoop *p = pWInfo->pLoops;
002473 pWInfo->pLoops = p->pNextLoop;
002474 whereLoopDelete(db, p);
002475 }
002476 while( pWInfo->pMemToFree ){
002477 WhereMemBlock *pNext = pWInfo->pMemToFree->pNext;
002478 sqlite3DbNNFreeNN(db, pWInfo->pMemToFree);
002479 pWInfo->pMemToFree = pNext;
002480 }
002481 sqlite3DbNNFreeNN(db, pWInfo);
002482 }
002483
002484 /*
002485 ** Return TRUE if X is a proper subset of Y but is of equal or less cost.
002486 ** In other words, return true if all constraints of X are also part of Y
002487 ** and Y has additional constraints that might speed the search that X lacks
002488 ** but the cost of running X is not more than the cost of running Y.
002489 **
002490 ** In other words, return true if the cost relationwship between X and Y
002491 ** is inverted and needs to be adjusted.
002492 **
002493 ** Case 1:
002494 **
002495 ** (1a) X and Y use the same index.
002496 ** (1b) X has fewer == terms than Y
002497 ** (1c) Neither X nor Y use skip-scan
002498 ** (1d) X does not have a a greater cost than Y
002499 **
002500 ** Case 2:
002501 **
002502 ** (2a) X has the same or lower cost, or returns the same or fewer rows,
002503 ** than Y.
002504 ** (2b) X uses fewer WHERE clause terms than Y
002505 ** (2c) Every WHERE clause term used by X is also used by Y
002506 ** (2d) X skips at least as many columns as Y
002507 ** (2e) If X is a covering index, than Y is too
002508 */
002509 static int whereLoopCheaperProperSubset(
002510 const WhereLoop *pX, /* First WhereLoop to compare */
002511 const WhereLoop *pY /* Compare against this WhereLoop */
002512 ){
002513 int i, j;
002514 if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */
002515 assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 );
002516 assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 );
002517 if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */
002518 && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */
002519 && pX->nSkip==0 && pY->nSkip==0 /* (1c) */
002520 ){
002521 return 1; /* Case 1 is true */
002522 }
002523 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
002524 return 0; /* (2b) */
002525 }
002526 if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */
002527 for(i=pX->nLTerm-1; i>=0; i--){
002528 if( pX->aLTerm[i]==0 ) continue;
002529 for(j=pY->nLTerm-1; j>=0; j--){
002530 if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
002531 }
002532 if( j<0 ) return 0; /* (2c) */
002533 }
002534 if( (pX->wsFlags&WHERE_IDX_ONLY)!=0
002535 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
002536 return 0; /* (2e) */
002537 }
002538 return 1; /* Case 2 is true */
002539 }
002540
002541 /*
002542 ** Try to adjust the cost and number of output rows of WhereLoop pTemplate
002543 ** upwards or downwards so that:
002544 **
002545 ** (1) pTemplate costs less than any other WhereLoops that are a proper
002546 ** subset of pTemplate
002547 **
002548 ** (2) pTemplate costs more than any other WhereLoops for which pTemplate
002549 ** is a proper subset.
002550 **
002551 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
002552 ** WHERE clause terms than Y and that every WHERE clause term used by X is
002553 ** also used by Y.
002554 */
002555 static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
002556 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
002557 for(; p; p=p->pNextLoop){
002558 if( p->iTab!=pTemplate->iTab ) continue;
002559 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
002560 if( whereLoopCheaperProperSubset(p, pTemplate) ){
002561 /* Adjust pTemplate cost downward so that it is cheaper than its
002562 ** subset p. */
002563 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002564 pTemplate->rRun, pTemplate->nOut,
002565 MIN(p->rRun, pTemplate->rRun),
002566 MIN(p->nOut - 1, pTemplate->nOut)));
002567 pTemplate->rRun = MIN(p->rRun, pTemplate->rRun);
002568 pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut);
002569 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
002570 /* Adjust pTemplate cost upward so that it is costlier than p since
002571 ** pTemplate is a proper subset of p */
002572 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002573 pTemplate->rRun, pTemplate->nOut,
002574 MAX(p->rRun, pTemplate->rRun),
002575 MAX(p->nOut + 1, pTemplate->nOut)));
002576 pTemplate->rRun = MAX(p->rRun, pTemplate->rRun);
002577 pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut);
002578 }
002579 }
002580 }
002581
002582 /*
002583 ** Search the list of WhereLoops in *ppPrev looking for one that can be
002584 ** replaced by pTemplate.
002585 **
002586 ** Return NULL if pTemplate does not belong on the WhereLoop list.
002587 ** In other words if pTemplate ought to be dropped from further consideration.
002588 **
002589 ** If pX is a WhereLoop that pTemplate can replace, then return the
002590 ** link that points to pX.
002591 **
002592 ** If pTemplate cannot replace any existing element of the list but needs
002593 ** to be added to the list as a new entry, then return a pointer to the
002594 ** tail of the list.
002595 */
002596 static WhereLoop **whereLoopFindLesser(
002597 WhereLoop **ppPrev,
002598 const WhereLoop *pTemplate
002599 ){
002600 WhereLoop *p;
002601 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){
002602 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
002603 /* If either the iTab or iSortIdx values for two WhereLoop are different
002604 ** then those WhereLoops need to be considered separately. Neither is
002605 ** a candidate to replace the other. */
002606 continue;
002607 }
002608 /* In the current implementation, the rSetup value is either zero
002609 ** or the cost of building an automatic index (NlogN) and the NlogN
002610 ** is the same for compatible WhereLoops. */
002611 assert( p->rSetup==0 || pTemplate->rSetup==0
002612 || p->rSetup==pTemplate->rSetup );
002613
002614 /* whereLoopAddBtree() always generates and inserts the automatic index
002615 ** case first. Hence compatible candidate WhereLoops never have a larger
002616 ** rSetup. Call this SETUP-INVARIANT */
002617 assert( p->rSetup>=pTemplate->rSetup );
002618
002619 /* Any loop using an application-defined index (or PRIMARY KEY or
002620 ** UNIQUE constraint) with one or more == constraints is better
002621 ** than an automatic index. Unless it is a skip-scan. */
002622 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
002623 && (pTemplate->nSkip)==0
002624 && (pTemplate->wsFlags & WHERE_INDEXED)!=0
002625 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0
002626 && (p->prereq & pTemplate->prereq)==pTemplate->prereq
002627 ){
002628 break;
002629 }
002630
002631 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
002632 ** discarded. WhereLoop p is better if:
002633 ** (1) p has no more dependencies than pTemplate, and
002634 ** (2) p has an equal or lower cost than pTemplate
002635 */
002636 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */
002637 && p->rSetup<=pTemplate->rSetup /* (2a) */
002638 && p->rRun<=pTemplate->rRun /* (2b) */
002639 && p->nOut<=pTemplate->nOut /* (2c) */
002640 ){
002641 return 0; /* Discard pTemplate */
002642 }
002643
002644 /* If pTemplate is always better than p, then cause p to be overwritten
002645 ** with pTemplate. pTemplate is better than p if:
002646 ** (1) pTemplate has no more dependencies than p, and
002647 ** (2) pTemplate has an equal or lower cost than p.
002648 */
002649 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */
002650 && p->rRun>=pTemplate->rRun /* (2a) */
002651 && p->nOut>=pTemplate->nOut /* (2b) */
002652 ){
002653 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */
002654 break; /* Cause p to be overwritten by pTemplate */
002655 }
002656 }
002657 return ppPrev;
002658 }
002659
002660 /*
002661 ** Insert or replace a WhereLoop entry using the template supplied.
002662 **
002663 ** An existing WhereLoop entry might be overwritten if the new template
002664 ** is better and has fewer dependencies. Or the template will be ignored
002665 ** and no insert will occur if an existing WhereLoop is faster and has
002666 ** fewer dependencies than the template. Otherwise a new WhereLoop is
002667 ** added based on the template.
002668 **
002669 ** If pBuilder->pOrSet is not NULL then we care about only the
002670 ** prerequisites and rRun and nOut costs of the N best loops. That
002671 ** information is gathered in the pBuilder->pOrSet object. This special
002672 ** processing mode is used only for OR clause processing.
002673 **
002674 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
002675 ** still might overwrite similar loops with the new template if the
002676 ** new template is better. Loops may be overwritten if the following
002677 ** conditions are met:
002678 **
002679 ** (1) They have the same iTab.
002680 ** (2) They have the same iSortIdx.
002681 ** (3) The template has same or fewer dependencies than the current loop
002682 ** (4) The template has the same or lower cost than the current loop
002683 */
002684 static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
002685 WhereLoop **ppPrev, *p;
002686 WhereInfo *pWInfo = pBuilder->pWInfo;
002687 sqlite3 *db = pWInfo->pParse->db;
002688 int rc;
002689
002690 /* Stop the search once we hit the query planner search limit */
002691 if( pBuilder->iPlanLimit==0 ){
002692 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n"));
002693 if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0;
002694 return SQLITE_DONE;
002695 }
002696 pBuilder->iPlanLimit--;
002697
002698 whereLoopAdjustCost(pWInfo->pLoops, pTemplate);
002699
002700 /* If pBuilder->pOrSet is defined, then only keep track of the costs
002701 ** and prereqs.
002702 */
002703 if( pBuilder->pOrSet!=0 ){
002704 if( pTemplate->nLTerm ){
002705 #if WHERETRACE_ENABLED
002706 u16 n = pBuilder->pOrSet->n;
002707 int x =
002708 #endif
002709 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
002710 pTemplate->nOut);
002711 #if WHERETRACE_ENABLED /* 0x8 */
002712 if( sqlite3WhereTrace & 0x8 ){
002713 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n);
002714 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002715 }
002716 #endif
002717 }
002718 return SQLITE_OK;
002719 }
002720
002721 /* Look for an existing WhereLoop to replace with pTemplate
002722 */
002723 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate);
002724
002725 if( ppPrev==0 ){
002726 /* There already exists a WhereLoop on the list that is better
002727 ** than pTemplate, so just ignore pTemplate */
002728 #if WHERETRACE_ENABLED /* 0x8 */
002729 if( sqlite3WhereTrace & 0x8 ){
002730 sqlite3DebugPrintf(" skip: ");
002731 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002732 }
002733 #endif
002734 return SQLITE_OK;
002735 }else{
002736 p = *ppPrev;
002737 }
002738
002739 /* If we reach this point it means that either p[] should be overwritten
002740 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
002741 ** WhereLoop and insert it.
002742 */
002743 #if WHERETRACE_ENABLED /* 0x8 */
002744 if( sqlite3WhereTrace & 0x8 ){
002745 if( p!=0 ){
002746 sqlite3DebugPrintf("replace: ");
002747 sqlite3WhereLoopPrint(p, pBuilder->pWC);
002748 sqlite3DebugPrintf(" with: ");
002749 }else{
002750 sqlite3DebugPrintf(" add: ");
002751 }
002752 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002753 }
002754 #endif
002755 if( p==0 ){
002756 /* Allocate a new WhereLoop to add to the end of the list */
002757 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
002758 if( p==0 ) return SQLITE_NOMEM_BKPT;
002759 whereLoopInit(p);
002760 p->pNextLoop = 0;
002761 }else{
002762 /* We will be overwriting WhereLoop p[]. But before we do, first
002763 ** go through the rest of the list and delete any other entries besides
002764 ** p[] that are also supplanted by pTemplate */
002765 WhereLoop **ppTail = &p->pNextLoop;
002766 WhereLoop *pToDel;
002767 while( *ppTail ){
002768 ppTail = whereLoopFindLesser(ppTail, pTemplate);
002769 if( ppTail==0 ) break;
002770 pToDel = *ppTail;
002771 if( pToDel==0 ) break;
002772 *ppTail = pToDel->pNextLoop;
002773 #if WHERETRACE_ENABLED /* 0x8 */
002774 if( sqlite3WhereTrace & 0x8 ){
002775 sqlite3DebugPrintf(" delete: ");
002776 sqlite3WhereLoopPrint(pToDel, pBuilder->pWC);
002777 }
002778 #endif
002779 whereLoopDelete(db, pToDel);
002780 }
002781 }
002782 rc = whereLoopXfer(db, p, pTemplate);
002783 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002784 Index *pIndex = p->u.btree.pIndex;
002785 if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){
002786 p->u.btree.pIndex = 0;
002787 }
002788 }
002789 return rc;
002790 }
002791
002792 /*
002793 ** Adjust the WhereLoop.nOut value downward to account for terms of the
002794 ** WHERE clause that reference the loop but which are not used by an
002795 ** index.
002796 *
002797 ** For every WHERE clause term that is not used by the index
002798 ** and which has a truth probability assigned by one of the likelihood(),
002799 ** likely(), or unlikely() SQL functions, reduce the estimated number
002800 ** of output rows by the probability specified.
002801 **
002802 ** TUNING: For every WHERE clause term that is not used by the index
002803 ** and which does not have an assigned truth probability, heuristics
002804 ** described below are used to try to estimate the truth probability.
002805 ** TODO --> Perhaps this is something that could be improved by better
002806 ** table statistics.
002807 **
002808 ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
002809 ** value corresponds to -1 in LogEst notation, so this means decrement
002810 ** the WhereLoop.nOut field for every such WHERE clause term.
002811 **
002812 ** Heuristic 2: If there exists one or more WHERE clause terms of the
002813 ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
002814 ** final output row estimate is no greater than 1/4 of the total number
002815 ** of rows in the table. In other words, assume that x==EXPR will filter
002816 ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
002817 ** "x" column is boolean or else -1 or 0 or 1 is a common default value
002818 ** on the "x" column and so in that case only cap the output row estimate
002819 ** at 1/2 instead of 1/4.
002820 */
002821 static void whereLoopOutputAdjust(
002822 WhereClause *pWC, /* The WHERE clause */
002823 WhereLoop *pLoop, /* The loop to adjust downward */
002824 LogEst nRow /* Number of rows in the entire table */
002825 ){
002826 WhereTerm *pTerm, *pX;
002827 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
002828 int i, j;
002829 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */
002830
002831 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
002832 for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){
002833 assert( pTerm!=0 );
002834 if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
002835 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
002836 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue;
002837 for(j=pLoop->nLTerm-1; j>=0; j--){
002838 pX = pLoop->aLTerm[j];
002839 if( pX==0 ) continue;
002840 if( pX==pTerm ) break;
002841 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
002842 }
002843 if( j<0 ){
002844 sqlite3ProgressCheck(pWC->pWInfo->pParse);
002845 if( pLoop->maskSelf==pTerm->prereqAll ){
002846 /* If there are extra terms in the WHERE clause not used by an index
002847 ** that depend only on the table being scanned, and that will tend to
002848 ** cause many rows to be omitted, then mark that table as
002849 ** "self-culling".
002850 **
002851 ** 2022-03-24: Self-culling only applies if either the extra terms
002852 ** are straight comparison operators that are non-true with NULL
002853 ** operand, or if the loop is not an OUTER JOIN.
002854 */
002855 if( (pTerm->eOperator & 0x3f)!=0
002856 || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype
002857 & (JT_LEFT|JT_LTORJ))==0
002858 ){
002859 pLoop->wsFlags |= WHERE_SELFCULL;
002860 }
002861 }
002862 if( pTerm->truthProb<=0 ){
002863 /* If a truth probability is specified using the likelihood() hints,
002864 ** then use the probability provided by the application. */
002865 pLoop->nOut += pTerm->truthProb;
002866 }else{
002867 /* In the absence of explicit truth probabilities, use heuristics to
002868 ** guess a reasonable truth probability. */
002869 pLoop->nOut--;
002870 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0
002871 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */
002872 ){
002873 Expr *pRight = pTerm->pExpr->pRight;
002874 int k = 0;
002875 testcase( pTerm->pExpr->op==TK_IS );
002876 if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
002877 k = 10;
002878 }else{
002879 k = 20;
002880 }
002881 if( iReduce<k ){
002882 pTerm->wtFlags |= TERM_HEURTRUTH;
002883 iReduce = k;
002884 }
002885 }
002886 }
002887 }
002888 }
002889 if( pLoop->nOut > nRow-iReduce ){
002890 pLoop->nOut = nRow - iReduce;
002891 }
002892 }
002893
002894 /*
002895 ** Term pTerm is a vector range comparison operation. The first comparison
002896 ** in the vector can be optimized using column nEq of the index. This
002897 ** function returns the total number of vector elements that can be used
002898 ** as part of the range comparison.
002899 **
002900 ** For example, if the query is:
002901 **
002902 ** WHERE a = ? AND (b, c, d) > (?, ?, ?)
002903 **
002904 ** and the index:
002905 **
002906 ** CREATE INDEX ... ON (a, b, c, d, e)
002907 **
002908 ** then this function would be invoked with nEq=1. The value returned in
002909 ** this case is 3.
002910 */
002911 static int whereRangeVectorLen(
002912 Parse *pParse, /* Parsing context */
002913 int iCur, /* Cursor open on pIdx */
002914 Index *pIdx, /* The index to be used for a inequality constraint */
002915 int nEq, /* Number of prior equality constraints on same index */
002916 WhereTerm *pTerm /* The vector inequality constraint */
002917 ){
002918 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
002919 int i;
002920
002921 nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
002922 for(i=1; i<nCmp; i++){
002923 /* Test if comparison i of pTerm is compatible with column (i+nEq)
002924 ** of the index. If not, exit the loop. */
002925 char aff; /* Comparison affinity */
002926 char idxaff = 0; /* Indexed columns affinity */
002927 CollSeq *pColl; /* Comparison collation sequence */
002928 Expr *pLhs, *pRhs;
002929
002930 assert( ExprUseXList(pTerm->pExpr->pLeft) );
002931 pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr;
002932 pRhs = pTerm->pExpr->pRight;
002933 if( ExprUseXSelect(pRhs) ){
002934 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
002935 }else{
002936 pRhs = pRhs->x.pList->a[i].pExpr;
002937 }
002938
002939 /* Check that the LHS of the comparison is a column reference to
002940 ** the right column of the right source table. And that the sort
002941 ** order of the index column is the same as the sort order of the
002942 ** leftmost index column. */
002943 if( pLhs->op!=TK_COLUMN
002944 || pLhs->iTable!=iCur
002945 || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
002946 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
002947 ){
002948 break;
002949 }
002950
002951 testcase( pLhs->iColumn==XN_ROWID );
002952 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
002953 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
002954 if( aff!=idxaff ) break;
002955
002956 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
002957 if( pColl==0 ) break;
002958 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
002959 }
002960 return i;
002961 }
002962
002963 /*
002964 ** Adjust the cost C by the costMult factor T. This only occurs if
002965 ** compiled with -DSQLITE_ENABLE_COSTMULT
002966 */
002967 #ifdef SQLITE_ENABLE_COSTMULT
002968 # define ApplyCostMultiplier(C,T) C += T
002969 #else
002970 # define ApplyCostMultiplier(C,T)
002971 #endif
002972
002973 /*
002974 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
002975 ** index pIndex. Try to match one more.
002976 **
002977 ** When this function is called, pBuilder->pNew->nOut contains the
002978 ** number of rows expected to be visited by filtering using the nEq
002979 ** terms only. If it is modified, this value is restored before this
002980 ** function returns.
002981 **
002982 ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
002983 ** a fake index used for the INTEGER PRIMARY KEY.
002984 */
002985 static int whereLoopAddBtreeIndex(
002986 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */
002987 SrcItem *pSrc, /* FROM clause term being analyzed */
002988 Index *pProbe, /* An index on pSrc */
002989 LogEst nInMul /* log(Number of iterations due to IN) */
002990 ){
002991 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */
002992 Parse *pParse = pWInfo->pParse; /* Parsing context */
002993 sqlite3 *db = pParse->db; /* Database connection malloc context */
002994 WhereLoop *pNew; /* Template WhereLoop under construction */
002995 WhereTerm *pTerm; /* A WhereTerm under consideration */
002996 int opMask; /* Valid operators for constraints */
002997 WhereScan scan; /* Iterator for WHERE terms */
002998 Bitmask saved_prereq; /* Original value of pNew->prereq */
002999 u16 saved_nLTerm; /* Original value of pNew->nLTerm */
003000 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */
003001 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */
003002 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */
003003 u16 saved_nSkip; /* Original value of pNew->nSkip */
003004 u32 saved_wsFlags; /* Original value of pNew->wsFlags */
003005 LogEst saved_nOut; /* Original value of pNew->nOut */
003006 int rc = SQLITE_OK; /* Return code */
003007 LogEst rSize; /* Number of rows in the table */
003008 LogEst rLogSize; /* Logarithm of table size */
003009 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
003010
003011 pNew = pBuilder->pNew;
003012 assert( db->mallocFailed==0 || pParse->nErr>0 );
003013 if( pParse->nErr ){
003014 return pParse->rc;
003015 }
003016 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
003017 pProbe->pTable->zName,pProbe->zName,
003018 pNew->u.btree.nEq, pNew->nSkip, pNew->rRun));
003019
003020 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
003021 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
003022 if( pNew->wsFlags & WHERE_BTM_LIMIT ){
003023 opMask = WO_LT|WO_LE;
003024 }else{
003025 assert( pNew->u.btree.nBtm==0 );
003026 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
003027 }
003028 if( pProbe->bUnordered || pProbe->bLowQual ){
003029 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
003030 if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){
003031 opMask &= ~(WO_EQ|WO_IN|WO_IS);
003032 }
003033 }
003034
003035 assert( pNew->u.btree.nEq<pProbe->nColumn );
003036 assert( pNew->u.btree.nEq<pProbe->nKeyCol
003037 || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY );
003038
003039 saved_nEq = pNew->u.btree.nEq;
003040 saved_nBtm = pNew->u.btree.nBtm;
003041 saved_nTop = pNew->u.btree.nTop;
003042 saved_nSkip = pNew->nSkip;
003043 saved_nLTerm = pNew->nLTerm;
003044 saved_wsFlags = pNew->wsFlags;
003045 saved_prereq = pNew->prereq;
003046 saved_nOut = pNew->nOut;
003047 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
003048 opMask, pProbe);
003049 pNew->rSetup = 0;
003050 rSize = pProbe->aiRowLogEst[0];
003051 rLogSize = estLog(rSize);
003052 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
003053 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */
003054 LogEst rCostIdx;
003055 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */
003056 int nIn = 0;
003057 #ifdef SQLITE_ENABLE_STAT4
003058 int nRecValid = pBuilder->nRecValid;
003059 #endif
003060 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
003061 && indexColumnNotNull(pProbe, saved_nEq)
003062 ){
003063 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
003064 }
003065 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003066
003067 /* Do not allow the upper bound of a LIKE optimization range constraint
003068 ** to mix with a lower range bound from some other source */
003069 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
003070
003071 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
003072 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
003073 ){
003074 continue;
003075 }
003076 if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
003077 pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
003078 }else{
003079 pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED;
003080 }
003081 pNew->wsFlags = saved_wsFlags;
003082 pNew->u.btree.nEq = saved_nEq;
003083 pNew->u.btree.nBtm = saved_nBtm;
003084 pNew->u.btree.nTop = saved_nTop;
003085 pNew->nLTerm = saved_nLTerm;
003086 if( pNew->nLTerm>=pNew->nLSlot
003087 && whereLoopResize(db, pNew, pNew->nLTerm+1)
003088 ){
003089 break; /* OOM while trying to enlarge the pNew->aLTerm array */
003090 }
003091 pNew->aLTerm[pNew->nLTerm++] = pTerm;
003092 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
003093
003094 assert( nInMul==0
003095 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
003096 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
003097 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
003098 );
003099
003100 if( eOp & WO_IN ){
003101 Expr *pExpr = pTerm->pExpr;
003102 if( ExprUseXSelect(pExpr) ){
003103 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
003104 int i;
003105 nIn = 46; assert( 46==sqlite3LogEst(25) );
003106
003107 /* The expression may actually be of the form (x, y) IN (SELECT...).
003108 ** In this case there is a separate term for each of (x) and (y).
003109 ** However, the nIn multiplier should only be applied once, not once
003110 ** for each such term. The following loop checks that pTerm is the
003111 ** first such term in use, and sets nIn back to 0 if it is not. */
003112 for(i=0; i<pNew->nLTerm-1; i++){
003113 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
003114 }
003115 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
003116 /* "x IN (value, value, ...)" */
003117 nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
003118 }
003119 if( pProbe->hasStat1 && rLogSize>=10 ){
003120 LogEst M, logK, x;
003121 /* Let:
003122 ** N = the total number of rows in the table
003123 ** K = the number of entries on the RHS of the IN operator
003124 ** M = the number of rows in the table that match terms to the
003125 ** to the left in the same index. If the IN operator is on
003126 ** the left-most index column, M==N.
003127 **
003128 ** Given the definitions above, it is better to omit the IN operator
003129 ** from the index lookup and instead do a scan of the M elements,
003130 ** testing each scanned row against the IN operator separately, if:
003131 **
003132 ** M*log(K) < K*log(N)
003133 **
003134 ** Our estimates for M, K, and N might be inaccurate, so we build in
003135 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
003136 ** with the index, as using an index has better worst-case behavior.
003137 ** If we do not have real sqlite_stat1 data, always prefer to use
003138 ** the index. Do not bother with this optimization on very small
003139 ** tables (less than 2 rows) as it is pointless in that case.
003140 */
003141 M = pProbe->aiRowLogEst[saved_nEq];
003142 logK = estLog(nIn);
003143 /* TUNING v----- 10 to bias toward indexed IN */
003144 x = M + logK + 10 - (nIn + rLogSize);
003145 if( x>=0 ){
003146 WHERETRACE(0x40,
003147 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
003148 "prefers indexed lookup\n",
003149 saved_nEq, M, logK, nIn, rLogSize, x));
003150 }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){
003151 WHERETRACE(0x40,
003152 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003153 " nInMul=%d) prefers skip-scan\n",
003154 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003155 pNew->wsFlags |= WHERE_IN_SEEKSCAN;
003156 }else{
003157 WHERETRACE(0x40,
003158 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003159 " nInMul=%d) prefers normal scan\n",
003160 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003161 continue;
003162 }
003163 }
003164 pNew->wsFlags |= WHERE_COLUMN_IN;
003165 }else if( eOp & (WO_EQ|WO_IS) ){
003166 int iCol = pProbe->aiColumn[saved_nEq];
003167 pNew->wsFlags |= WHERE_COLUMN_EQ;
003168 assert( saved_nEq==pNew->u.btree.nEq );
003169 if( iCol==XN_ROWID
003170 || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
003171 ){
003172 if( iCol==XN_ROWID || pProbe->uniqNotNull
003173 || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ)
003174 ){
003175 pNew->wsFlags |= WHERE_ONEROW;
003176 }else{
003177 pNew->wsFlags |= WHERE_UNQ_WANTED;
003178 }
003179 }
003180 if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS;
003181 }else if( eOp & WO_ISNULL ){
003182 pNew->wsFlags |= WHERE_COLUMN_NULL;
003183 }else{
003184 int nVecLen = whereRangeVectorLen(
003185 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
003186 );
003187 if( eOp & (WO_GT|WO_GE) ){
003188 testcase( eOp & WO_GT );
003189 testcase( eOp & WO_GE );
003190 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
003191 pNew->u.btree.nBtm = nVecLen;
003192 pBtm = pTerm;
003193 pTop = 0;
003194 if( pTerm->wtFlags & TERM_LIKEOPT ){
003195 /* Range constraints that come from the LIKE optimization are
003196 ** always used in pairs. */
003197 pTop = &pTerm[1];
003198 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm );
003199 assert( pTop->wtFlags & TERM_LIKEOPT );
003200 assert( pTop->eOperator==WO_LT );
003201 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
003202 pNew->aLTerm[pNew->nLTerm++] = pTop;
003203 pNew->wsFlags |= WHERE_TOP_LIMIT;
003204 pNew->u.btree.nTop = 1;
003205 }
003206 }else{
003207 assert( eOp & (WO_LT|WO_LE) );
003208 testcase( eOp & WO_LT );
003209 testcase( eOp & WO_LE );
003210 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
003211 pNew->u.btree.nTop = nVecLen;
003212 pTop = pTerm;
003213 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
003214 pNew->aLTerm[pNew->nLTerm-2] : 0;
003215 }
003216 }
003217
003218 /* At this point pNew->nOut is set to the number of rows expected to
003219 ** be visited by the index scan before considering term pTerm, or the
003220 ** values of nIn and nInMul. In other words, assuming that all
003221 ** "x IN(...)" terms are replaced with "x = ?". This block updates
003222 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
003223 assert( pNew->nOut==saved_nOut );
003224 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003225 /* Adjust nOut using stat4 data. Or, if there is no stat4
003226 ** data, using some other estimate. */
003227 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew);
003228 }else{
003229 int nEq = ++pNew->u.btree.nEq;
003230 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
003231
003232 assert( pNew->nOut==saved_nOut );
003233 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
003234 assert( (eOp & WO_IN) || nIn==0 );
003235 testcase( eOp & WO_IN );
003236 pNew->nOut += pTerm->truthProb;
003237 pNew->nOut -= nIn;
003238 }else{
003239 #ifdef SQLITE_ENABLE_STAT4
003240 tRowcnt nOut = 0;
003241 if( nInMul==0
003242 && pProbe->nSample
003243 && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol)
003244 && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr))
003245 && OptimizationEnabled(db, SQLITE_Stat4)
003246 ){
003247 Expr *pExpr = pTerm->pExpr;
003248 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){
003249 testcase( eOp & WO_EQ );
003250 testcase( eOp & WO_IS );
003251 testcase( eOp & WO_ISNULL );
003252 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
003253 }else{
003254 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut);
003255 }
003256 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
003257 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */
003258 if( nOut ){
003259 pNew->nOut = sqlite3LogEst(nOut);
003260 if( nEq==1
003261 /* TUNING: Mark terms as "low selectivity" if they seem likely
003262 ** to be true for half or more of the rows in the table.
003263 ** See tag-202002240-1 */
003264 && pNew->nOut+10 > pProbe->aiRowLogEst[0]
003265 ){
003266 #if WHERETRACE_ENABLED /* 0x01 */
003267 if( sqlite3WhereTrace & 0x20 ){
003268 sqlite3DebugPrintf(
003269 "STAT4 determines term has low selectivity:\n");
003270 sqlite3WhereTermPrint(pTerm, 999);
003271 }
003272 #endif
003273 pTerm->wtFlags |= TERM_HIGHTRUTH;
003274 if( pTerm->wtFlags & TERM_HEURTRUTH ){
003275 /* If the term has previously been used with an assumption of
003276 ** higher selectivity, then set the flag to rerun the
003277 ** loop computations. */
003278 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
003279 }
003280 }
003281 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
003282 pNew->nOut -= nIn;
003283 }
003284 }
003285 if( nOut==0 )
003286 #endif
003287 {
003288 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
003289 if( eOp & WO_ISNULL ){
003290 /* TUNING: If there is no likelihood() value, assume that a
003291 ** "col IS NULL" expression matches twice as many rows
003292 ** as (col=?). */
003293 pNew->nOut += 10;
003294 }
003295 }
003296 }
003297 }
003298
003299 /* Set rCostIdx to the estimated cost of visiting selected rows in the
003300 ** index. The estimate is the sum of two values:
003301 ** 1. The cost of doing one search-by-key to find the first matching
003302 ** entry
003303 ** 2. Stepping forward in the index pNew->nOut times to find all
003304 ** additional matching entries.
003305 */
003306 assert( pSrc->pTab->szTabRow>0 );
003307 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003308 /* The pProbe->szIdxRow is low for an IPK table since the interior
003309 ** pages are small. Thus szIdxRow gives a good estimate of seek cost.
003310 ** But the leaf pages are full-size, so pProbe->szIdxRow would badly
003311 ** under-estimate the scanning cost. */
003312 rCostIdx = pNew->nOut + 16;
003313 }else{
003314 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow;
003315 }
003316 rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx);
003317
003318 /* Estimate the cost of running the loop. If all data is coming
003319 ** from the index, then this is just the cost of doing the index
003320 ** lookup and scan. But if some data is coming out of the main table,
003321 ** we also have to add in the cost of doing pNew->nOut searches to
003322 ** locate the row in the main table that corresponds to the index entry.
003323 */
003324 pNew->rRun = rCostIdx;
003325 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){
003326 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
003327 }
003328 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
003329
003330 nOutUnadjusted = pNew->nOut;
003331 pNew->rRun += nInMul + nIn;
003332 pNew->nOut += nInMul + nIn;
003333 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize);
003334 rc = whereLoopInsert(pBuilder, pNew);
003335
003336 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003337 pNew->nOut = saved_nOut;
003338 }else{
003339 pNew->nOut = nOutUnadjusted;
003340 }
003341
003342 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
003343 && pNew->u.btree.nEq<pProbe->nColumn
003344 && (pNew->u.btree.nEq<pProbe->nKeyCol ||
003345 pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
003346 ){
003347 if( pNew->u.btree.nEq>3 ){
003348 sqlite3ProgressCheck(pParse);
003349 }
003350 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
003351 }
003352 pNew->nOut = saved_nOut;
003353 #ifdef SQLITE_ENABLE_STAT4
003354 pBuilder->nRecValid = nRecValid;
003355 #endif
003356 }
003357 pNew->prereq = saved_prereq;
003358 pNew->u.btree.nEq = saved_nEq;
003359 pNew->u.btree.nBtm = saved_nBtm;
003360 pNew->u.btree.nTop = saved_nTop;
003361 pNew->nSkip = saved_nSkip;
003362 pNew->wsFlags = saved_wsFlags;
003363 pNew->nOut = saved_nOut;
003364 pNew->nLTerm = saved_nLTerm;
003365
003366 /* Consider using a skip-scan if there are no WHERE clause constraints
003367 ** available for the left-most terms of the index, and if the average
003368 ** number of repeats in the left-most terms is at least 18.
003369 **
003370 ** The magic number 18 is selected on the basis that scanning 17 rows
003371 ** is almost always quicker than an index seek (even though if the index
003372 ** contains fewer than 2^17 rows we assume otherwise in other parts of
003373 ** the code). And, even if it is not, it should not be too much slower.
003374 ** On the other hand, the extra seeks could end up being significantly
003375 ** more expensive. */
003376 assert( 42==sqlite3LogEst(18) );
003377 if( saved_nEq==saved_nSkip
003378 && saved_nEq+1<pProbe->nKeyCol
003379 && saved_nEq==pNew->nLTerm
003380 && pProbe->noSkipScan==0
003381 && pProbe->hasStat1!=0
003382 && OptimizationEnabled(db, SQLITE_SkipScan)
003383 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */
003384 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
003385 ){
003386 LogEst nIter;
003387 pNew->u.btree.nEq++;
003388 pNew->nSkip++;
003389 pNew->aLTerm[pNew->nLTerm++] = 0;
003390 pNew->wsFlags |= WHERE_SKIPSCAN;
003391 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
003392 pNew->nOut -= nIter;
003393 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
003394 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
003395 nIter += 5;
003396 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
003397 pNew->nOut = saved_nOut;
003398 pNew->u.btree.nEq = saved_nEq;
003399 pNew->nSkip = saved_nSkip;
003400 pNew->wsFlags = saved_wsFlags;
003401 }
003402
003403 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
003404 pProbe->pTable->zName, pProbe->zName, saved_nEq, rc));
003405 return rc;
003406 }
003407
003408 /*
003409 ** Return True if it is possible that pIndex might be useful in
003410 ** implementing the ORDER BY clause in pBuilder.
003411 **
003412 ** Return False if pBuilder does not contain an ORDER BY clause or
003413 ** if there is no way for pIndex to be useful in implementing that
003414 ** ORDER BY clause.
003415 */
003416 static int indexMightHelpWithOrderBy(
003417 WhereLoopBuilder *pBuilder,
003418 Index *pIndex,
003419 int iCursor
003420 ){
003421 ExprList *pOB;
003422 ExprList *aColExpr;
003423 int ii, jj;
003424
003425 if( pIndex->bUnordered ) return 0;
003426 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
003427 for(ii=0; ii<pOB->nExpr; ii++){
003428 Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
003429 if( NEVER(pExpr==0) ) continue;
003430 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN)
003431 && pExpr->iTable==iCursor
003432 ){
003433 if( pExpr->iColumn<0 ) return 1;
003434 for(jj=0; jj<pIndex->nKeyCol; jj++){
003435 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
003436 }
003437 }else if( (aColExpr = pIndex->aColExpr)!=0 ){
003438 for(jj=0; jj<pIndex->nKeyCol; jj++){
003439 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
003440 if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
003441 return 1;
003442 }
003443 }
003444 }
003445 }
003446 return 0;
003447 }
003448
003449 /* Check to see if a partial index with pPartIndexWhere can be used
003450 ** in the current query. Return true if it can be and false if not.
003451 */
003452 static int whereUsablePartialIndex(
003453 int iTab, /* The table for which we want an index */
003454 u8 jointype, /* The JT_* flags on the join */
003455 WhereClause *pWC, /* The WHERE clause of the query */
003456 Expr *pWhere /* The WHERE clause from the partial index */
003457 ){
003458 int i;
003459 WhereTerm *pTerm;
003460 Parse *pParse;
003461
003462 if( jointype & JT_LTORJ ) return 0;
003463 pParse = pWC->pWInfo->pParse;
003464 while( pWhere->op==TK_AND ){
003465 if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
003466 pWhere = pWhere->pRight;
003467 }
003468 if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0;
003469 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
003470 Expr *pExpr;
003471 pExpr = pTerm->pExpr;
003472 if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab)
003473 && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON))
003474 && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab)
003475 && (pTerm->wtFlags & TERM_VNULL)==0
003476 ){
003477 return 1;
003478 }
003479 }
003480 return 0;
003481 }
003482
003483 /*
003484 ** pIdx is an index containing expressions. Check it see if any of the
003485 ** expressions in the index match the pExpr expression.
003486 */
003487 static int exprIsCoveredByIndex(
003488 const Expr *pExpr,
003489 const Index *pIdx,
003490 int iTabCur
003491 ){
003492 int i;
003493 for(i=0; i<pIdx->nColumn; i++){
003494 if( pIdx->aiColumn[i]==XN_EXPR
003495 && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0
003496 ){
003497 return 1;
003498 }
003499 }
003500 return 0;
003501 }
003502
003503 /*
003504 ** Structure passed to the whereIsCoveringIndex Walker callback.
003505 */
003506 typedef struct CoveringIndexCheck CoveringIndexCheck;
003507 struct CoveringIndexCheck {
003508 Index *pIdx; /* The index */
003509 int iTabCur; /* Cursor number for the corresponding table */
003510 u8 bExpr; /* Uses an indexed expression */
003511 u8 bUnidx; /* Uses an unindexed column not within an indexed expr */
003512 };
003513
003514 /*
003515 ** Information passed in is pWalk->u.pCovIdxCk. Call it pCk.
003516 **
003517 ** If the Expr node references the table with cursor pCk->iTabCur, then
003518 ** make sure that column is covered by the index pCk->pIdx. We know that
003519 ** all columns less than 63 (really BMS-1) are covered, so we don't need
003520 ** to check them. But we do need to check any column at 63 or greater.
003521 **
003522 ** If the index does not cover the column, then set pWalk->eCode to
003523 ** non-zero and return WRC_Abort to stop the search.
003524 **
003525 ** If this node does not disprove that the index can be a covering index,
003526 ** then just return WRC_Continue, to continue the search.
003527 **
003528 ** If pCk->pIdx contains indexed expressions and one of those expressions
003529 ** matches pExpr, then prune the search.
003530 */
003531 static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
003532 int i; /* Loop counter */
003533 const Index *pIdx; /* The index of interest */
003534 const i16 *aiColumn; /* Columns contained in the index */
003535 u16 nColumn; /* Number of columns in the index */
003536 CoveringIndexCheck *pCk; /* Info about this search */
003537
003538 pCk = pWalk->u.pCovIdxCk;
003539 pIdx = pCk->pIdx;
003540 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){
003541 /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/
003542 if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue;
003543 pIdx = pWalk->u.pCovIdxCk->pIdx;
003544 aiColumn = pIdx->aiColumn;
003545 nColumn = pIdx->nColumn;
003546 for(i=0; i<nColumn; i++){
003547 if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
003548 }
003549 pCk->bUnidx = 1;
003550 return WRC_Abort;
003551 }else if( pIdx->bHasExpr
003552 && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){
003553 pCk->bExpr = 1;
003554 return WRC_Prune;
003555 }
003556 return WRC_Continue;
003557 }
003558
003559
003560 /*
003561 ** pIdx is an index that covers all of the low-number columns used by
003562 ** pWInfo->pSelect (columns from 0 through 62) or an index that has
003563 ** expressions terms. Hence, we cannot determine whether or not it is
003564 ** a covering index by using the colUsed bitmasks. We have to do a search
003565 ** to see if the index is covering. This routine does that search.
003566 **
003567 ** The return value is one of these:
003568 **
003569 ** 0 The index is definitely not a covering index
003570 **
003571 ** WHERE_IDX_ONLY The index is definitely a covering index
003572 **
003573 ** WHERE_EXPRIDX The index is likely a covering index, but it is
003574 ** difficult to determine precisely because of the
003575 ** expressions that are indexed. Score it as a
003576 ** covering index, but still keep the main table open
003577 ** just in case we need it.
003578 **
003579 ** This routine is an optimization. It is always safe to return zero.
003580 ** But returning one of the other two values when zero should have been
003581 ** returned can lead to incorrect bytecode and assertion faults.
003582 */
003583 static SQLITE_NOINLINE u32 whereIsCoveringIndex(
003584 WhereInfo *pWInfo, /* The WHERE clause context */
003585 Index *pIdx, /* Index that is being tested */
003586 int iTabCur /* Cursor for the table being indexed */
003587 ){
003588 int i, rc;
003589 struct CoveringIndexCheck ck;
003590 Walker w;
003591 if( pWInfo->pSelect==0 ){
003592 /* We don't have access to the full query, so we cannot check to see
003593 ** if pIdx is covering. Assume it is not. */
003594 return 0;
003595 }
003596 if( pIdx->bHasExpr==0 ){
003597 for(i=0; i<pIdx->nColumn; i++){
003598 if( pIdx->aiColumn[i]>=BMS-1 ) break;
003599 }
003600 if( i>=pIdx->nColumn ){
003601 /* pIdx does not index any columns greater than 62, but we know from
003602 ** colMask that columns greater than 62 are used, so this is not a
003603 ** covering index */
003604 return 0;
003605 }
003606 }
003607 ck.pIdx = pIdx;
003608 ck.iTabCur = iTabCur;
003609 ck.bExpr = 0;
003610 ck.bUnidx = 0;
003611 memset(&w, 0, sizeof(w));
003612 w.xExprCallback = whereIsCoveringIndexWalkCallback;
003613 w.xSelectCallback = sqlite3SelectWalkNoop;
003614 w.u.pCovIdxCk = &ck;
003615 sqlite3WalkSelect(&w, pWInfo->pSelect);
003616 if( ck.bUnidx ){
003617 rc = 0;
003618 }else if( ck.bExpr ){
003619 rc = WHERE_EXPRIDX;
003620 }else{
003621 rc = WHERE_IDX_ONLY;
003622 }
003623 return rc;
003624 }
003625
003626 /*
003627 ** This is an sqlite3ParserAddCleanup() callback that is invoked to
003628 ** free the Parse->pIdxEpr list when the Parse object is destroyed.
003629 */
003630 static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
003631 IndexedExpr **pp = (IndexedExpr**)pObject;
003632 while( *pp!=0 ){
003633 IndexedExpr *p = *pp;
003634 *pp = p->pIENext;
003635 sqlite3ExprDelete(db, p->pExpr);
003636 sqlite3DbFreeNN(db, p);
003637 }
003638 }
003639
003640 /*
003641 ** This function is called for a partial index - one with a WHERE clause - in
003642 ** two scenarios. In both cases, it determines whether or not the WHERE
003643 ** clause on the index implies that a column of the table may be safely
003644 ** replaced by a constant expression. For example, in the following
003645 ** SELECT:
003646 **
003647 ** CREATE INDEX i1 ON t1(b, c) WHERE a=<expr>;
003648 ** SELECT a, b, c FROM t1 WHERE a=<expr> AND b=?;
003649 **
003650 ** The "a" in the select-list may be replaced by <expr>, iff:
003651 **
003652 ** (a) <expr> is a constant expression, and
003653 ** (b) The (a=<expr>) comparison uses the BINARY collation sequence, and
003654 ** (c) Column "a" has an affinity other than NONE or BLOB.
003655 **
003656 ** If argument pItem is NULL, then pMask must not be NULL. In this case this
003657 ** function is being called as part of determining whether or not pIdx
003658 ** is a covering index. This function clears any bits in (*pMask)
003659 ** corresponding to columns that may be replaced by constants as described
003660 ** above.
003661 **
003662 ** Otherwise, if pItem is not NULL, then this function is being called
003663 ** as part of coding a loop that uses index pIdx. In this case, add entries
003664 ** to the Parse.pIdxPartExpr list for each column that can be replaced
003665 ** by a constant.
003666 */
003667 static void wherePartIdxExpr(
003668 Parse *pParse, /* Parse context */
003669 Index *pIdx, /* Partial index being processed */
003670 Expr *pPart, /* WHERE clause being processed */
003671 Bitmask *pMask, /* Mask to clear bits in */
003672 int iIdxCur, /* Cursor number for index */
003673 SrcItem *pItem /* The FROM clause entry for the table */
003674 ){
003675 assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 );
003676 assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) );
003677
003678 if( pPart->op==TK_AND ){
003679 wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem);
003680 pPart = pPart->pLeft;
003681 }
003682
003683 if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){
003684 Expr *pLeft = pPart->pLeft;
003685 Expr *pRight = pPart->pRight;
003686 u8 aff;
003687
003688 if( pLeft->op!=TK_COLUMN ) return;
003689 if( !sqlite3ExprIsConstant(0, pRight) ) return;
003690 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return;
003691 if( pLeft->iColumn<0 ) return;
003692 aff = pIdx->pTable->aCol[pLeft->iColumn].affinity;
003693 if( aff>=SQLITE_AFF_TEXT ){
003694 if( pItem ){
003695 sqlite3 *db = pParse->db;
003696 IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p));
003697 if( p ){
003698 int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0;
003699 p->pExpr = sqlite3ExprDup(db, pRight, 0);
003700 p->iDataCur = pItem->iCursor;
003701 p->iIdxCur = iIdxCur;
003702 p->iIdxCol = pLeft->iColumn;
003703 p->bMaybeNullRow = bNullRow;
003704 p->pIENext = pParse->pIdxPartExpr;
003705 p->aff = aff;
003706 pParse->pIdxPartExpr = p;
003707 if( p->pIENext==0 ){
003708 void *pArg = (void*)&pParse->pIdxPartExpr;
003709 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
003710 }
003711 }
003712 }else if( pLeft->iColumn<(BMS-1) ){
003713 *pMask &= ~((Bitmask)1 << pLeft->iColumn);
003714 }
003715 }
003716 }
003717 }
003718
003719
003720 /*
003721 ** Add all WhereLoop objects for a single table of the join where the table
003722 ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
003723 ** a b-tree table, not a virtual table.
003724 **
003725 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
003726 ** are calculated as follows:
003727 **
003728 ** For a full scan, assuming the table (or index) contains nRow rows:
003729 **
003730 ** cost = nRow * 3.0 // full-table scan
003731 ** cost = nRow * K // scan of covering index
003732 ** cost = nRow * (K+3.0) // scan of non-covering index
003733 **
003734 ** where K is a value between 1.1 and 3.0 set based on the relative
003735 ** estimated average size of the index and table records.
003736 **
003737 ** For an index scan, where nVisit is the number of index rows visited
003738 ** by the scan, and nSeek is the number of seek operations required on
003739 ** the index b-tree:
003740 **
003741 ** cost = nSeek * (log(nRow) + K * nVisit) // covering index
003742 ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
003743 **
003744 ** Normally, nSeek is 1. nSeek values greater than 1 come about if the
003745 ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
003746 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
003747 **
003748 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount
003749 ** of uncertainty. For this reason, scoring is designed to pick plans that
003750 ** "do the least harm" if the estimates are inaccurate. For example, a
003751 ** log(nRow) factor is omitted from a non-covering index scan in order to
003752 ** bias the scoring in favor of using an index, since the worst-case
003753 ** performance of using an index is far better than the worst-case performance
003754 ** of a full table scan.
003755 */
003756 static int whereLoopAddBtree(
003757 WhereLoopBuilder *pBuilder, /* WHERE clause information */
003758 Bitmask mPrereq /* Extra prerequisites for using this table */
003759 ){
003760 WhereInfo *pWInfo; /* WHERE analysis context */
003761 Index *pProbe; /* An index we are evaluating */
003762 Index sPk; /* A fake index object for the primary key */
003763 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */
003764 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */
003765 SrcList *pTabList; /* The FROM clause */
003766 SrcItem *pSrc; /* The FROM clause btree term to add */
003767 WhereLoop *pNew; /* Template WhereLoop object */
003768 int rc = SQLITE_OK; /* Return code */
003769 int iSortIdx = 1; /* Index number */
003770 int b; /* A boolean value */
003771 LogEst rSize; /* number of rows in the table */
003772 WhereClause *pWC; /* The parsed WHERE clause */
003773 Table *pTab; /* Table being queried */
003774
003775 pNew = pBuilder->pNew;
003776 pWInfo = pBuilder->pWInfo;
003777 pTabList = pWInfo->pTabList;
003778 pSrc = pTabList->a + pNew->iTab;
003779 pTab = pSrc->pTab;
003780 pWC = pBuilder->pWC;
003781 assert( !IsVirtual(pSrc->pTab) );
003782
003783 if( pSrc->fg.isIndexedBy ){
003784 assert( pSrc->fg.isCte==0 );
003785 /* An INDEXED BY clause specifies a particular index to use */
003786 pProbe = pSrc->u2.pIBIndex;
003787 }else if( !HasRowid(pTab) ){
003788 pProbe = pTab->pIndex;
003789 }else{
003790 /* There is no INDEXED BY clause. Create a fake Index object in local
003791 ** variable sPk to represent the rowid primary key index. Make this
003792 ** fake index the first in a chain of Index objects with all of the real
003793 ** indices to follow */
003794 Index *pFirst; /* First of real indices on the table */
003795 memset(&sPk, 0, sizeof(Index));
003796 sPk.nKeyCol = 1;
003797 sPk.nColumn = 1;
003798 sPk.aiColumn = &aiColumnPk;
003799 sPk.aiRowLogEst = aiRowEstPk;
003800 sPk.onError = OE_Replace;
003801 sPk.pTable = pTab;
003802 sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */
003803 sPk.idxType = SQLITE_IDXTYPE_IPK;
003804 aiRowEstPk[0] = pTab->nRowLogEst;
003805 aiRowEstPk[1] = 0;
003806 pFirst = pSrc->pTab->pIndex;
003807 if( pSrc->fg.notIndexed==0 ){
003808 /* The real indices of the table are only considered if the
003809 ** NOT INDEXED qualifier is omitted from the FROM clause */
003810 sPk.pNext = pFirst;
003811 }
003812 pProbe = &sPk;
003813 }
003814 rSize = pTab->nRowLogEst;
003815
003816 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
003817 /* Automatic indexes */
003818 if( !pBuilder->pOrSet /* Not part of an OR optimization */
003819 && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0
003820 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
003821 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */
003822 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */
003823 && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */
003824 && !pSrc->fg.isCorrelated /* Not a correlated subquery */
003825 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */
003826 && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */
003827 ){
003828 /* Generate auto-index WhereLoops */
003829 LogEst rLogSize; /* Logarithm of the number of rows in the table */
003830 WhereTerm *pTerm;
003831 WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
003832 rLogSize = estLog(rSize);
003833 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
003834 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003835 if( termCanDriveIndex(pTerm, pSrc, 0) ){
003836 pNew->u.btree.nEq = 1;
003837 pNew->nSkip = 0;
003838 pNew->u.btree.pIndex = 0;
003839 pNew->nLTerm = 1;
003840 pNew->aLTerm[0] = pTerm;
003841 /* TUNING: One-time cost for computing the automatic index is
003842 ** estimated to be X*N*log2(N) where N is the number of rows in
003843 ** the table being indexed and where X is 7 (LogEst=28) for normal
003844 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value
003845 ** of X is smaller for views and subqueries so that the query planner
003846 ** will be more aggressive about generating automatic indexes for
003847 ** those objects, since there is no opportunity to add schema
003848 ** indexes on subqueries and views. */
003849 pNew->rSetup = rLogSize + rSize;
003850 if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){
003851 pNew->rSetup += 28;
003852 }else{
003853 pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes
003854 ** on ephemeral materializations of views */
003855 }
003856 ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
003857 if( pNew->rSetup<0 ) pNew->rSetup = 0;
003858 /* TUNING: Each index lookup yields 20 rows in the table. This
003859 ** is more than the usual guess of 10 rows, since we have no way
003860 ** of knowing how selective the index will ultimately be. It would
003861 ** not be unreasonable to make this value much larger. */
003862 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) );
003863 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
003864 pNew->wsFlags = WHERE_AUTO_INDEX;
003865 pNew->prereq = mPrereq | pTerm->prereqRight;
003866 rc = whereLoopInsert(pBuilder, pNew);
003867 }
003868 }
003869 }
003870 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
003871
003872 /* Loop over all indices. If there was an INDEXED BY clause, then only
003873 ** consider index pProbe. */
003874 for(; rc==SQLITE_OK && pProbe;
003875 pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++
003876 ){
003877 if( pProbe->pPartIdxWhere!=0
003878 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC,
003879 pProbe->pPartIdxWhere)
003880 ){
003881 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
003882 continue; /* Partial index inappropriate for this query */
003883 }
003884 if( pProbe->bNoQuery ) continue;
003885 rSize = pProbe->aiRowLogEst[0];
003886 pNew->u.btree.nEq = 0;
003887 pNew->u.btree.nBtm = 0;
003888 pNew->u.btree.nTop = 0;
003889 pNew->nSkip = 0;
003890 pNew->nLTerm = 0;
003891 pNew->iSortIdx = 0;
003892 pNew->rSetup = 0;
003893 pNew->prereq = mPrereq;
003894 pNew->nOut = rSize;
003895 pNew->u.btree.pIndex = pProbe;
003896 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
003897
003898 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
003899 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
003900 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003901 /* Integer primary key index */
003902 pNew->wsFlags = WHERE_IPK;
003903
003904 /* Full table scan */
003905 pNew->iSortIdx = b ? iSortIdx : 0;
003906 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an
003907 ** extra cost designed to discourage the use of full table scans,
003908 ** since index lookups have better worst-case performance if our
003909 ** stat guesses are wrong. Reduce the 3.0 penalty slightly
003910 ** (to 2.75) if we have valid STAT4 information for the table.
003911 ** At 2.75, a full table scan is preferred over using an index on
003912 ** a column with just two distinct values where each value has about
003913 ** an equal number of appearances. Without STAT4 data, we still want
003914 ** to use an index in that case, since the constraint might be for
003915 ** the scarcer of the two values, and in that case an index lookup is
003916 ** better.
003917 */
003918 #ifdef SQLITE_ENABLE_STAT4
003919 pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
003920 #else
003921 pNew->rRun = rSize + 16;
003922 #endif
003923 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
003924 whereLoopOutputAdjust(pWC, pNew, rSize);
003925 rc = whereLoopInsert(pBuilder, pNew);
003926 pNew->nOut = rSize;
003927 if( rc ) break;
003928 }else{
003929 Bitmask m;
003930 if( pProbe->isCovering ){
003931 m = 0;
003932 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
003933 }else{
003934 m = pSrc->colUsed & pProbe->colNotIdxed;
003935 if( pProbe->pPartIdxWhere ){
003936 wherePartIdxExpr(
003937 pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0
003938 );
003939 }
003940 pNew->wsFlags = WHERE_INDEXED;
003941 if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){
003942 u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
003943 if( isCov==0 ){
003944 WHERETRACE(0x200,
003945 ("-> %s is not a covering index"
003946 " according to whereIsCoveringIndex()\n", pProbe->zName));
003947 assert( m!=0 );
003948 }else{
003949 m = 0;
003950 pNew->wsFlags |= isCov;
003951 if( isCov & WHERE_IDX_ONLY ){
003952 WHERETRACE(0x200,
003953 ("-> %s is a covering expression index"
003954 " according to whereIsCoveringIndex()\n", pProbe->zName));
003955 }else{
003956 assert( isCov==WHERE_EXPRIDX );
003957 WHERETRACE(0x200,
003958 ("-> %s might be a covering expression index"
003959 " according to whereIsCoveringIndex()\n", pProbe->zName));
003960 }
003961 }
003962 }else if( m==0
003963 && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700))
003964 ){
003965 WHERETRACE(0x200,
003966 ("-> %s a covering index according to bitmasks\n",
003967 pProbe->zName, m==0 ? "is" : "is not"));
003968 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
003969 }
003970 }
003971
003972 /* Full scan via index */
003973 if( b
003974 || !HasRowid(pTab)
003975 || pProbe->pPartIdxWhere!=0
003976 || pSrc->fg.isIndexedBy
003977 || ( m==0
003978 && pProbe->bUnordered==0
003979 && (pProbe->szIdxRow<pTab->szTabRow)
003980 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
003981 && sqlite3GlobalConfig.bUseCis
003982 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
003983 )
003984 ){
003985 pNew->iSortIdx = b ? iSortIdx : 0;
003986
003987 /* The cost of visiting the index rows is N*K, where K is
003988 ** between 1.1 and 3.0, depending on the relative sizes of the
003989 ** index and table rows. */
003990 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
003991 if( m!=0 ){
003992 /* If this is a non-covering index scan, add in the cost of
003993 ** doing table lookups. The cost will be 3x the number of
003994 ** lookups. Take into account WHERE clause terms that can be
003995 ** satisfied using just the index, and that do not require a
003996 ** table lookup. */
003997 LogEst nLookup = rSize + 16; /* Base cost: N*3 */
003998 int ii;
003999 int iCur = pSrc->iCursor;
004000 WhereClause *pWC2 = &pWInfo->sWC;
004001 for(ii=0; ii<pWC2->nTerm; ii++){
004002 WhereTerm *pTerm = &pWC2->a[ii];
004003 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
004004 break;
004005 }
004006 /* pTerm can be evaluated using just the index. So reduce
004007 ** the expected number of table lookups accordingly */
004008 if( pTerm->truthProb<=0 ){
004009 nLookup += pTerm->truthProb;
004010 }else{
004011 nLookup--;
004012 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
004013 }
004014 }
004015
004016 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
004017 }
004018 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
004019 whereLoopOutputAdjust(pWC, pNew, rSize);
004020 if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){
004021 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN
004022 ** because the cursor used to access the index might not be
004023 ** positioned to the correct row during the right-join no-match
004024 ** loop. */
004025 }else{
004026 rc = whereLoopInsert(pBuilder, pNew);
004027 }
004028 pNew->nOut = rSize;
004029 if( rc ) break;
004030 }
004031 }
004032
004033 pBuilder->bldFlags1 = 0;
004034 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
004035 if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){
004036 /* If a non-unique index is used, or if a prefix of the key for
004037 ** unique index is used (making the index functionally non-unique)
004038 ** then the sqlite_stat1 data becomes important for scoring the
004039 ** plan */
004040 pTab->tabFlags |= TF_MaybeReanalyze;
004041 }
004042 #ifdef SQLITE_ENABLE_STAT4
004043 sqlite3Stat4ProbeFree(pBuilder->pRec);
004044 pBuilder->nRecValid = 0;
004045 pBuilder->pRec = 0;
004046 #endif
004047 }
004048 return rc;
004049 }
004050
004051 #ifndef SQLITE_OMIT_VIRTUALTABLE
004052
004053 /*
004054 ** Return true if pTerm is a virtual table LIMIT or OFFSET term.
004055 */
004056 static int isLimitTerm(WhereTerm *pTerm){
004057 assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 );
004058 return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT
004059 && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET;
004060 }
004061
004062 /*
004063 ** Return true if the first nCons constraints in the pUsage array are
004064 ** marked as in-use (have argvIndex>0). False otherwise.
004065 */
004066 static int allConstraintsUsed(
004067 struct sqlite3_index_constraint_usage *aUsage,
004068 int nCons
004069 ){
004070 int ii;
004071 for(ii=0; ii<nCons; ii++){
004072 if( aUsage[ii].argvIndex<=0 ) return 0;
004073 }
004074 return 1;
004075 }
004076
004077 /*
004078 ** Argument pIdxInfo is already populated with all constraints that may
004079 ** be used by the virtual table identified by pBuilder->pNew->iTab. This
004080 ** function marks a subset of those constraints usable, invokes the
004081 ** xBestIndex method and adds the returned plan to pBuilder.
004082 **
004083 ** A constraint is marked usable if:
004084 **
004085 ** * Argument mUsable indicates that its prerequisites are available, and
004086 **
004087 ** * It is not one of the operators specified in the mExclude mask passed
004088 ** as the fourth argument (which in practice is either WO_IN or 0).
004089 **
004090 ** Argument mPrereq is a mask of tables that must be scanned before the
004091 ** virtual table in question. These are added to the plans prerequisites
004092 ** before it is added to pBuilder.
004093 **
004094 ** Output parameter *pbIn is set to true if the plan added to pBuilder
004095 ** uses one or more WO_IN terms, or false otherwise.
004096 */
004097 static int whereLoopAddVirtualOne(
004098 WhereLoopBuilder *pBuilder,
004099 Bitmask mPrereq, /* Mask of tables that must be used. */
004100 Bitmask mUsable, /* Mask of usable tables */
004101 u16 mExclude, /* Exclude terms using these operators */
004102 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */
004103 u16 mNoOmit, /* Do not omit these constraints */
004104 int *pbIn, /* OUT: True if plan uses an IN(...) op */
004105 int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */
004106 ){
004107 WhereClause *pWC = pBuilder->pWC;
004108 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004109 struct sqlite3_index_constraint *pIdxCons;
004110 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
004111 int i;
004112 int mxTerm;
004113 int rc = SQLITE_OK;
004114 WhereLoop *pNew = pBuilder->pNew;
004115 Parse *pParse = pBuilder->pWInfo->pParse;
004116 SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
004117 int nConstraint = pIdxInfo->nConstraint;
004118
004119 assert( (mUsable & mPrereq)==mPrereq );
004120 *pbIn = 0;
004121 pNew->prereq = mPrereq;
004122
004123 /* Set the usable flag on the subset of constraints identified by
004124 ** arguments mUsable and mExclude. */
004125 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004126 for(i=0; i<nConstraint; i++, pIdxCons++){
004127 WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset];
004128 pIdxCons->usable = 0;
004129 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
004130 && (pTerm->eOperator & mExclude)==0
004131 && (pbRetryLimit || !isLimitTerm(pTerm))
004132 ){
004133 pIdxCons->usable = 1;
004134 }
004135 }
004136
004137 /* Initialize the output fields of the sqlite3_index_info structure */
004138 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
004139 assert( pIdxInfo->needToFreeIdxStr==0 );
004140 pIdxInfo->idxStr = 0;
004141 pIdxInfo->idxNum = 0;
004142 pIdxInfo->orderByConsumed = 0;
004143 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
004144 pIdxInfo->estimatedRows = 25;
004145 pIdxInfo->idxFlags = 0;
004146 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
004147 pHidden->mHandleIn = 0;
004148
004149 /* Invoke the virtual table xBestIndex() method */
004150 rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo);
004151 if( rc ){
004152 if( rc==SQLITE_CONSTRAINT ){
004153 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
004154 ** that the particular combination of parameters provided is unusable.
004155 ** Make no entries in the loop table.
004156 */
004157 WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n"));
004158 return SQLITE_OK;
004159 }
004160 return rc;
004161 }
004162
004163 mxTerm = -1;
004164 assert( pNew->nLSlot>=nConstraint );
004165 memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint );
004166 memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab));
004167 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004168 for(i=0; i<nConstraint; i++, pIdxCons++){
004169 int iTerm;
004170 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
004171 WhereTerm *pTerm;
004172 int j = pIdxCons->iTermOffset;
004173 if( iTerm>=nConstraint
004174 || j<0
004175 || j>=pWC->nTerm
004176 || pNew->aLTerm[iTerm]!=0
004177 || pIdxCons->usable==0
004178 ){
004179 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
004180 testcase( pIdxInfo->needToFreeIdxStr );
004181 return SQLITE_ERROR;
004182 }
004183 testcase( iTerm==nConstraint-1 );
004184 testcase( j==0 );
004185 testcase( j==pWC->nTerm-1 );
004186 pTerm = &pWC->a[j];
004187 pNew->prereq |= pTerm->prereqRight;
004188 assert( iTerm<pNew->nLSlot );
004189 pNew->aLTerm[iTerm] = pTerm;
004190 if( iTerm>mxTerm ) mxTerm = iTerm;
004191 testcase( iTerm==15 );
004192 testcase( iTerm==16 );
004193 if( pUsage[i].omit ){
004194 if( i<16 && ((1<<i)&mNoOmit)==0 ){
004195 testcase( i!=iTerm );
004196 pNew->u.vtab.omitMask |= 1<<iTerm;
004197 }else{
004198 testcase( i!=iTerm );
004199 }
004200 if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){
004201 pNew->u.vtab.bOmitOffset = 1;
004202 }
004203 }
004204 if( SMASKBIT32(i) & pHidden->mHandleIn ){
004205 pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm);
004206 }else if( (pTerm->eOperator & WO_IN)!=0 ){
004207 /* A virtual table that is constrained by an IN clause may not
004208 ** consume the ORDER BY clause because (1) the order of IN terms
004209 ** is not necessarily related to the order of output terms and
004210 ** (2) Multiple outputs from a single IN value will not merge
004211 ** together. */
004212 pIdxInfo->orderByConsumed = 0;
004213 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
004214 *pbIn = 1; assert( (mExclude & WO_IN)==0 );
004215 }
004216
004217 /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET
004218 ** terms. And if there are any, they should follow all other terms. */
004219 assert( pbRetryLimit || !isLimitTerm(pTerm) );
004220 assert( !isLimitTerm(pTerm) || i>=nConstraint-2 );
004221 assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) );
004222
004223 if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){
004224 /* If there is an IN(...) term handled as an == (separate call to
004225 ** xFilter for each value on the RHS of the IN) and a LIMIT or
004226 ** OFFSET term handled as well, the plan is unusable. Similarly,
004227 ** if there is a LIMIT/OFFSET and there are other unused terms,
004228 ** the plan cannot be used. In these cases set variable *pbRetryLimit
004229 ** to true to tell the caller to retry with LIMIT and OFFSET
004230 ** disabled. */
004231 if( pIdxInfo->needToFreeIdxStr ){
004232 sqlite3_free(pIdxInfo->idxStr);
004233 pIdxInfo->idxStr = 0;
004234 pIdxInfo->needToFreeIdxStr = 0;
004235 }
004236 *pbRetryLimit = 1;
004237 return SQLITE_OK;
004238 }
004239 }
004240 }
004241
004242 pNew->nLTerm = mxTerm+1;
004243 for(i=0; i<=mxTerm; i++){
004244 if( pNew->aLTerm[i]==0 ){
004245 /* The non-zero argvIdx values must be contiguous. Raise an
004246 ** error if they are not */
004247 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
004248 testcase( pIdxInfo->needToFreeIdxStr );
004249 return SQLITE_ERROR;
004250 }
004251 }
004252 assert( pNew->nLTerm<=pNew->nLSlot );
004253 pNew->u.vtab.idxNum = pIdxInfo->idxNum;
004254 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
004255 pIdxInfo->needToFreeIdxStr = 0;
004256 pNew->u.vtab.idxStr = pIdxInfo->idxStr;
004257 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
004258 pIdxInfo->nOrderBy : 0);
004259 pNew->rSetup = 0;
004260 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
004261 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
004262
004263 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
004264 ** that the scan will visit at most one row. Clear it otherwise. */
004265 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
004266 pNew->wsFlags |= WHERE_ONEROW;
004267 }else{
004268 pNew->wsFlags &= ~WHERE_ONEROW;
004269 }
004270 rc = whereLoopInsert(pBuilder, pNew);
004271 if( pNew->u.vtab.needFree ){
004272 sqlite3_free(pNew->u.vtab.idxStr);
004273 pNew->u.vtab.needFree = 0;
004274 }
004275 WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
004276 *pbIn, (sqlite3_uint64)mPrereq,
004277 (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
004278
004279 return rc;
004280 }
004281
004282 /*
004283 ** Return the collating sequence for a constraint passed into xBestIndex.
004284 **
004285 ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex.
004286 ** This routine depends on there being a HiddenIndexInfo structure immediately
004287 ** following the sqlite3_index_info structure.
004288 **
004289 ** Return a pointer to the collation name:
004290 **
004291 ** 1. If there is an explicit COLLATE operator on the constraint, return it.
004292 **
004293 ** 2. Else, if the column has an alternative collation, return that.
004294 **
004295 ** 3. Otherwise, return "BINARY".
004296 */
004297 const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
004298 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004299 const char *zRet = 0;
004300 if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
004301 CollSeq *pC = 0;
004302 int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
004303 Expr *pX = pHidden->pWC->a[iTerm].pExpr;
004304 if( pX->pLeft ){
004305 pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX);
004306 }
004307 zRet = (pC ? pC->zName : sqlite3StrBINARY);
004308 }
004309 return zRet;
004310 }
004311
004312 /*
004313 ** Return true if constraint iCons is really an IN(...) constraint, or
004314 ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0)
004315 ** or clear (if bHandle==0) the flag to handle it using an iterator.
004316 */
004317 int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
004318 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004319 u32 m = SMASKBIT32(iCons);
004320 if( m & pHidden->mIn ){
004321 if( bHandle==0 ){
004322 pHidden->mHandleIn &= ~m;
004323 }else if( bHandle>0 ){
004324 pHidden->mHandleIn |= m;
004325 }
004326 return 1;
004327 }
004328 return 0;
004329 }
004330
004331 /*
004332 ** This interface is callable from within the xBestIndex callback only.
004333 **
004334 ** If possible, set (*ppVal) to point to an object containing the value
004335 ** on the right-hand-side of constraint iCons.
004336 */
004337 int sqlite3_vtab_rhs_value(
004338 sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */
004339 int iCons, /* Constraint for which RHS is wanted */
004340 sqlite3_value **ppVal /* Write value extracted here */
004341 ){
004342 HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1];
004343 sqlite3_value *pVal = 0;
004344 int rc = SQLITE_OK;
004345 if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
004346 rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */
004347 }else{
004348 if( pH->aRhs[iCons]==0 ){
004349 WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset];
004350 rc = sqlite3ValueFromExpr(
004351 pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db),
004352 SQLITE_AFF_BLOB, &pH->aRhs[iCons]
004353 );
004354 testcase( rc!=SQLITE_OK );
004355 }
004356 pVal = pH->aRhs[iCons];
004357 }
004358 *ppVal = pVal;
004359
004360 if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */
004361 rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */
004362 }
004363
004364 return rc;
004365 }
004366
004367 /*
004368 ** Return true if ORDER BY clause may be handled as DISTINCT.
004369 */
004370 int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
004371 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004372 assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 );
004373 return pHidden->eDistinct;
004374 }
004375
004376 /*
004377 ** Cause the prepared statement that is associated with a call to
004378 ** xBestIndex to potentially use all schemas. If the statement being
004379 ** prepared is read-only, then just start read transactions on all
004380 ** schemas. But if this is a write operation, start writes on all
004381 ** schemas.
004382 **
004383 ** This is used by the (built-in) sqlite_dbpage virtual table.
004384 */
004385 void sqlite3VtabUsesAllSchemas(Parse *pParse){
004386 int nDb = pParse->db->nDb;
004387 int i;
004388 for(i=0; i<nDb; i++){
004389 sqlite3CodeVerifySchema(pParse, i);
004390 }
004391 if( DbMaskNonZero(pParse->writeMask) ){
004392 for(i=0; i<nDb; i++){
004393 sqlite3BeginWriteOperation(pParse, 0, i);
004394 }
004395 }
004396 }
004397
004398 /*
004399 ** Add all WhereLoop objects for a table of the join identified by
004400 ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
004401 **
004402 ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
004403 ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
004404 ** entries that occur before the virtual table in the FROM clause and are
004405 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
004406 ** mUnusable mask contains all FROM clause entries that occur after the
004407 ** virtual table and are separated from it by at least one LEFT or
004408 ** CROSS JOIN.
004409 **
004410 ** For example, if the query were:
004411 **
004412 ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
004413 **
004414 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
004415 **
004416 ** All the tables in mPrereq must be scanned before the current virtual
004417 ** table. So any terms for which all prerequisites are satisfied by
004418 ** mPrereq may be specified as "usable" in all calls to xBestIndex.
004419 ** Conversely, all tables in mUnusable must be scanned after the current
004420 ** virtual table, so any terms for which the prerequisites overlap with
004421 ** mUnusable should always be configured as "not-usable" for xBestIndex.
004422 */
004423 static int whereLoopAddVirtual(
004424 WhereLoopBuilder *pBuilder, /* WHERE clause information */
004425 Bitmask mPrereq, /* Tables that must be scanned before this one */
004426 Bitmask mUnusable /* Tables that must be scanned after this one */
004427 ){
004428 int rc = SQLITE_OK; /* Return code */
004429 WhereInfo *pWInfo; /* WHERE analysis context */
004430 Parse *pParse; /* The parsing context */
004431 WhereClause *pWC; /* The WHERE clause */
004432 SrcItem *pSrc; /* The FROM clause term to search */
004433 sqlite3_index_info *p; /* Object to pass to xBestIndex() */
004434 int nConstraint; /* Number of constraints in p */
004435 int bIn; /* True if plan uses IN(...) operator */
004436 WhereLoop *pNew;
004437 Bitmask mBest; /* Tables used by best possible plan */
004438 u16 mNoOmit;
004439 int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */
004440
004441 assert( (mPrereq & mUnusable)==0 );
004442 pWInfo = pBuilder->pWInfo;
004443 pParse = pWInfo->pParse;
004444 pWC = pBuilder->pWC;
004445 pNew = pBuilder->pNew;
004446 pSrc = &pWInfo->pTabList->a[pNew->iTab];
004447 assert( IsVirtual(pSrc->pTab) );
004448 p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
004449 if( p==0 ) return SQLITE_NOMEM_BKPT;
004450 pNew->rSetup = 0;
004451 pNew->wsFlags = WHERE_VIRTUALTABLE;
004452 pNew->nLTerm = 0;
004453 pNew->u.vtab.needFree = 0;
004454 nConstraint = p->nConstraint;
004455 if( whereLoopResize(pParse->db, pNew, nConstraint) ){
004456 freeIndexInfo(pParse->db, p);
004457 return SQLITE_NOMEM_BKPT;
004458 }
004459
004460 /* First call xBestIndex() with all constraints usable. */
004461 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName));
004462 WHERETRACE(0x800, (" VirtualOne: all usable\n"));
004463 rc = whereLoopAddVirtualOne(
004464 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry
004465 );
004466 if( bRetry ){
004467 assert( rc==SQLITE_OK );
004468 rc = whereLoopAddVirtualOne(
004469 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0
004470 );
004471 }
004472
004473 /* If the call to xBestIndex() with all terms enabled produced a plan
004474 ** that does not require any source tables (IOW: a plan with mBest==0)
004475 ** and does not use an IN(...) operator, then there is no point in making
004476 ** any further calls to xBestIndex() since they will all return the same
004477 ** result (if the xBestIndex() implementation is sane). */
004478 if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){
004479 int seenZero = 0; /* True if a plan with no prereqs seen */
004480 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */
004481 Bitmask mPrev = 0;
004482 Bitmask mBestNoIn = 0;
004483
004484 /* If the plan produced by the earlier call uses an IN(...) term, call
004485 ** xBestIndex again, this time with IN(...) terms disabled. */
004486 if( bIn ){
004487 WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n"));
004488 rc = whereLoopAddVirtualOne(
004489 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0);
004490 assert( bIn==0 );
004491 mBestNoIn = pNew->prereq & ~mPrereq;
004492 if( mBestNoIn==0 ){
004493 seenZero = 1;
004494 seenZeroNoIN = 1;
004495 }
004496 }
004497
004498 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
004499 ** in the set of terms that apply to the current virtual table. */
004500 while( rc==SQLITE_OK ){
004501 int i;
004502 Bitmask mNext = ALLBITS;
004503 assert( mNext>0 );
004504 for(i=0; i<nConstraint; i++){
004505 Bitmask mThis = (
004506 pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq
004507 );
004508 if( mThis>mPrev && mThis<mNext ) mNext = mThis;
004509 }
004510 mPrev = mNext;
004511 if( mNext==ALLBITS ) break;
004512 if( mNext==mBest || mNext==mBestNoIn ) continue;
004513 WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
004514 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
004515 rc = whereLoopAddVirtualOne(
004516 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0);
004517 if( pNew->prereq==mPrereq ){
004518 seenZero = 1;
004519 if( bIn==0 ) seenZeroNoIN = 1;
004520 }
004521 }
004522
004523 /* If the calls to xBestIndex() in the above loop did not find a plan
004524 ** that requires no source tables at all (i.e. one guaranteed to be
004525 ** usable), make a call here with all source tables disabled */
004526 if( rc==SQLITE_OK && seenZero==0 ){
004527 WHERETRACE(0x800, (" VirtualOne: all disabled\n"));
004528 rc = whereLoopAddVirtualOne(
004529 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0);
004530 if( bIn==0 ) seenZeroNoIN = 1;
004531 }
004532
004533 /* If the calls to xBestIndex() have so far failed to find a plan
004534 ** that requires no source tables at all and does not use an IN(...)
004535 ** operator, make a final call to obtain one here. */
004536 if( rc==SQLITE_OK && seenZeroNoIN==0 ){
004537 WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n"));
004538 rc = whereLoopAddVirtualOne(
004539 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0);
004540 }
004541 }
004542
004543 if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
004544 freeIndexInfo(pParse->db, p);
004545 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc));
004546 return rc;
004547 }
004548 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004549
004550 /*
004551 ** Add WhereLoop entries to handle OR terms. This works for either
004552 ** btrees or virtual tables.
004553 */
004554 static int whereLoopAddOr(
004555 WhereLoopBuilder *pBuilder,
004556 Bitmask mPrereq,
004557 Bitmask mUnusable
004558 ){
004559 WhereInfo *pWInfo = pBuilder->pWInfo;
004560 WhereClause *pWC;
004561 WhereLoop *pNew;
004562 WhereTerm *pTerm, *pWCEnd;
004563 int rc = SQLITE_OK;
004564 int iCur;
004565 WhereClause tempWC;
004566 WhereLoopBuilder sSubBuild;
004567 WhereOrSet sSum, sCur;
004568 SrcItem *pItem;
004569
004570 pWC = pBuilder->pWC;
004571 pWCEnd = pWC->a + pWC->nTerm;
004572 pNew = pBuilder->pNew;
004573 memset(&sSum, 0, sizeof(sSum));
004574 pItem = pWInfo->pTabList->a + pNew->iTab;
004575 iCur = pItem->iCursor;
004576
004577 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */
004578 if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK;
004579
004580 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
004581 if( (pTerm->eOperator & WO_OR)!=0
004582 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
004583 ){
004584 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
004585 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
004586 WhereTerm *pOrTerm;
004587 int once = 1;
004588 int i, j;
004589
004590 sSubBuild = *pBuilder;
004591 sSubBuild.pOrSet = &sCur;
004592
004593 WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm));
004594 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
004595 if( (pOrTerm->eOperator & WO_AND)!=0 ){
004596 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
004597 }else if( pOrTerm->leftCursor==iCur ){
004598 tempWC.pWInfo = pWC->pWInfo;
004599 tempWC.pOuter = pWC;
004600 tempWC.op = TK_AND;
004601 tempWC.nTerm = 1;
004602 tempWC.nBase = 1;
004603 tempWC.a = pOrTerm;
004604 sSubBuild.pWC = &tempWC;
004605 }else{
004606 continue;
004607 }
004608 sCur.n = 0;
004609 #ifdef WHERETRACE_ENABLED
004610 WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n",
004611 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
004612 if( sqlite3WhereTrace & 0x20000 ){
004613 sqlite3WhereClausePrint(sSubBuild.pWC);
004614 }
004615 #endif
004616 #ifndef SQLITE_OMIT_VIRTUALTABLE
004617 if( IsVirtual(pItem->pTab) ){
004618 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
004619 }else
004620 #endif
004621 {
004622 rc = whereLoopAddBtree(&sSubBuild, mPrereq);
004623 }
004624 if( rc==SQLITE_OK ){
004625 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
004626 }
004627 testcase( rc==SQLITE_NOMEM && sCur.n>0 );
004628 testcase( rc==SQLITE_DONE );
004629 if( sCur.n==0 ){
004630 sSum.n = 0;
004631 break;
004632 }else if( once ){
004633 whereOrMove(&sSum, &sCur);
004634 once = 0;
004635 }else{
004636 WhereOrSet sPrev;
004637 whereOrMove(&sPrev, &sSum);
004638 sSum.n = 0;
004639 for(i=0; i<sPrev.n; i++){
004640 for(j=0; j<sCur.n; j++){
004641 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
004642 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
004643 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
004644 }
004645 }
004646 }
004647 }
004648 pNew->nLTerm = 1;
004649 pNew->aLTerm[0] = pTerm;
004650 pNew->wsFlags = WHERE_MULTI_OR;
004651 pNew->rSetup = 0;
004652 pNew->iSortIdx = 0;
004653 memset(&pNew->u, 0, sizeof(pNew->u));
004654 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
004655 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
004656 ** of all sub-scans required by the OR-scan. However, due to rounding
004657 ** errors, it may be that the cost of the OR-scan is equal to its
004658 ** most expensive sub-scan. Add the smallest possible penalty
004659 ** (equivalent to multiplying the cost by 1.07) to ensure that
004660 ** this does not happen. Otherwise, for WHERE clauses such as the
004661 ** following where there is an index on "y":
004662 **
004663 ** WHERE likelihood(x=?, 0.99) OR y=?
004664 **
004665 ** the planner may elect to "OR" together a full-table scan and an
004666 ** index lookup. And other similarly odd results. */
004667 pNew->rRun = sSum.a[i].rRun + 1;
004668 pNew->nOut = sSum.a[i].nOut;
004669 pNew->prereq = sSum.a[i].prereq;
004670 rc = whereLoopInsert(pBuilder, pNew);
004671 }
004672 WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm));
004673 }
004674 }
004675 return rc;
004676 }
004677
004678 /*
004679 ** Add all WhereLoop objects for all tables
004680 */
004681 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
004682 WhereInfo *pWInfo = pBuilder->pWInfo;
004683 Bitmask mPrereq = 0;
004684 Bitmask mPrior = 0;
004685 int iTab;
004686 SrcList *pTabList = pWInfo->pTabList;
004687 SrcItem *pItem;
004688 SrcItem *pEnd = &pTabList->a[pWInfo->nLevel];
004689 sqlite3 *db = pWInfo->pParse->db;
004690 int rc = SQLITE_OK;
004691 int bFirstPastRJ = 0;
004692 int hasRightJoin = 0;
004693 WhereLoop *pNew;
004694
004695
004696 /* Loop over the tables in the join, from left to right */
004697 pNew = pBuilder->pNew;
004698
004699 /* Verify that pNew has already been initialized */
004700 assert( pNew->nLTerm==0 );
004701 assert( pNew->wsFlags==0 );
004702 assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) );
004703 assert( pNew->aLTerm!=0 );
004704
004705 pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT;
004706 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){
004707 Bitmask mUnusable = 0;
004708 pNew->iTab = iTab;
004709 pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR;
004710 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
004711 if( bFirstPastRJ
004712 || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0
004713 ){
004714 /* Add prerequisites to prevent reordering of FROM clause terms
004715 ** across CROSS joins and outer joins. The bFirstPastRJ boolean
004716 ** prevents the right operand of a RIGHT JOIN from being swapped with
004717 ** other elements even further to the right.
004718 **
004719 ** The JT_LTORJ case and the hasRightJoin flag work together to
004720 ** prevent FROM-clause terms from moving from the right side of
004721 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN
004722 ** is itself on the left side of a RIGHT JOIN.
004723 */
004724 if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1;
004725 mPrereq |= mPrior;
004726 bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0;
004727 }else if( !hasRightJoin ){
004728 mPrereq = 0;
004729 }
004730 #ifndef SQLITE_OMIT_VIRTUALTABLE
004731 if( IsVirtual(pItem->pTab) ){
004732 SrcItem *p;
004733 for(p=&pItem[1]; p<pEnd; p++){
004734 if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){
004735 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor);
004736 }
004737 }
004738 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
004739 }else
004740 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004741 {
004742 rc = whereLoopAddBtree(pBuilder, mPrereq);
004743 }
004744 if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){
004745 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
004746 }
004747 mPrior |= pNew->maskSelf;
004748 if( rc || db->mallocFailed ){
004749 if( rc==SQLITE_DONE ){
004750 /* We hit the query planner search limit set by iPlanLimit */
004751 sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search");
004752 rc = SQLITE_OK;
004753 }else{
004754 break;
004755 }
004756 }
004757 }
004758
004759 whereLoopClear(db, pNew);
004760 return rc;
004761 }
004762
004763 /*
004764 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
004765 ** parameters) to see if it outputs rows in the requested ORDER BY
004766 ** (or GROUP BY) without requiring a separate sort operation. Return N:
004767 **
004768 ** N>0: N terms of the ORDER BY clause are satisfied
004769 ** N==0: No terms of the ORDER BY clause are satisfied
004770 ** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
004771 **
004772 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
004773 ** strict. With GROUP BY and DISTINCT the only requirement is that
004774 ** equivalent rows appear immediately adjacent to one another. GROUP BY
004775 ** and DISTINCT do not require rows to appear in any particular order as long
004776 ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
004777 ** the pOrderBy terms can be matched in any order. With ORDER BY, the
004778 ** pOrderBy terms must be matched in strict left-to-right order.
004779 */
004780 static i8 wherePathSatisfiesOrderBy(
004781 WhereInfo *pWInfo, /* The WHERE clause */
004782 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
004783 WherePath *pPath, /* The WherePath to check */
004784 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
004785 u16 nLoop, /* Number of entries in pPath->aLoop[] */
004786 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */
004787 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */
004788 ){
004789 u8 revSet; /* True if rev is known */
004790 u8 rev; /* Composite sort order */
004791 u8 revIdx; /* Index sort order */
004792 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */
004793 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */
004794 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */
004795 u16 eqOpMask; /* Allowed equality operators */
004796 u16 nKeyCol; /* Number of key columns in pIndex */
004797 u16 nColumn; /* Total number of ordered columns in the index */
004798 u16 nOrderBy; /* Number terms in the ORDER BY clause */
004799 int iLoop; /* Index of WhereLoop in pPath being processed */
004800 int i, j; /* Loop counters */
004801 int iCur; /* Cursor number for current WhereLoop */
004802 int iColumn; /* A column number within table iCur */
004803 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
004804 WhereTerm *pTerm; /* A single term of the WHERE clause */
004805 Expr *pOBExpr; /* An expression from the ORDER BY clause */
004806 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */
004807 Index *pIndex; /* The index associated with pLoop */
004808 sqlite3 *db = pWInfo->pParse->db; /* Database connection */
004809 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */
004810 Bitmask obDone; /* Mask of all ORDER BY terms */
004811 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */
004812 Bitmask ready; /* Mask of inner loops */
004813
004814 /*
004815 ** We say the WhereLoop is "one-row" if it generates no more than one
004816 ** row of output. A WhereLoop is one-row if all of the following are true:
004817 ** (a) All index columns match with WHERE_COLUMN_EQ.
004818 ** (b) The index is unique
004819 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
004820 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
004821 **
004822 ** We say the WhereLoop is "order-distinct" if the set of columns from
004823 ** that WhereLoop that are in the ORDER BY clause are different for every
004824 ** row of the WhereLoop. Every one-row WhereLoop is automatically
004825 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
004826 ** is not order-distinct. To be order-distinct is not quite the same as being
004827 ** UNIQUE since a UNIQUE column or index can have multiple rows that
004828 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
004829 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
004830 **
004831 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
004832 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
004833 ** automatically order-distinct.
004834 */
004835
004836 assert( pOrderBy!=0 );
004837 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
004838
004839 nOrderBy = pOrderBy->nExpr;
004840 testcase( nOrderBy==BMS-1 );
004841 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
004842 isOrderDistinct = 1;
004843 obDone = MASKBIT(nOrderBy)-1;
004844 orderDistinctMask = 0;
004845 ready = 0;
004846 eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
004847 if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){
004848 eqOpMask |= WO_IN;
004849 }
004850 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
004851 if( iLoop>0 ) ready |= pLoop->maskSelf;
004852 if( iLoop<nLoop ){
004853 pLoop = pPath->aLoop[iLoop];
004854 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
004855 }else{
004856 pLoop = pLast;
004857 }
004858 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
004859 if( pLoop->u.vtab.isOrdered
004860 && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
004861 ){
004862 obSat = obDone;
004863 }
004864 break;
004865 }else if( wctrlFlags & WHERE_DISTINCTBY ){
004866 pLoop->u.btree.nDistinctCol = 0;
004867 }
004868 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
004869
004870 /* Mark off any ORDER BY term X that is a column in the table of
004871 ** the current loop for which there is term in the WHERE
004872 ** clause of the form X IS NULL or X=? that reference only outer
004873 ** loops.
004874 */
004875 for(i=0; i<nOrderBy; i++){
004876 if( MASKBIT(i) & obSat ) continue;
004877 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
004878 if( NEVER(pOBExpr==0) ) continue;
004879 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
004880 if( pOBExpr->iTable!=iCur ) continue;
004881 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
004882 ~ready, eqOpMask, 0);
004883 if( pTerm==0 ) continue;
004884 if( pTerm->eOperator==WO_IN ){
004885 /* IN terms are only valid for sorting in the ORDER BY LIMIT
004886 ** optimization, and then only if they are actually used
004887 ** by the query plan */
004888 assert( wctrlFlags &
004889 (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) );
004890 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
004891 if( j>=pLoop->nLTerm ) continue;
004892 }
004893 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
004894 Parse *pParse = pWInfo->pParse;
004895 CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr);
004896 CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr);
004897 assert( pColl1 );
004898 if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){
004899 continue;
004900 }
004901 testcase( pTerm->pExpr->op==TK_IS );
004902 }
004903 obSat |= MASKBIT(i);
004904 }
004905
004906 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
004907 if( pLoop->wsFlags & WHERE_IPK ){
004908 pIndex = 0;
004909 nKeyCol = 0;
004910 nColumn = 1;
004911 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
004912 return 0;
004913 }else{
004914 nKeyCol = pIndex->nKeyCol;
004915 nColumn = pIndex->nColumn;
004916 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) );
004917 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
004918 || !HasRowid(pIndex->pTable));
004919 /* All relevant terms of the index must also be non-NULL in order
004920 ** for isOrderDistinct to be true. So the isOrderDistint value
004921 ** computed here might be a false positive. Corrections will be
004922 ** made at tag-20210426-1 below */
004923 isOrderDistinct = IsUniqueIndex(pIndex)
004924 && (pLoop->wsFlags & WHERE_SKIPSCAN)==0;
004925 }
004926
004927 /* Loop through all columns of the index and deal with the ones
004928 ** that are not constrained by == or IN.
004929 */
004930 rev = revSet = 0;
004931 distinctColumns = 0;
004932 for(j=0; j<nColumn; j++){
004933 u8 bOnce = 1; /* True to run the ORDER BY search loop */
004934
004935 assert( j>=pLoop->u.btree.nEq
004936 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
004937 );
004938 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
004939 u16 eOp = pLoop->aLTerm[j]->eOperator;
004940
004941 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
004942 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL
004943 ** terms imply that the index is not UNIQUE NOT NULL in which case
004944 ** the loop need to be marked as not order-distinct because it can
004945 ** have repeated NULL rows.
004946 **
004947 ** If the current term is a column of an ((?,?) IN (SELECT...))
004948 ** expression for which the SELECT returns more than one column,
004949 ** check that it is the only column used by this loop. Otherwise,
004950 ** if it is one of two or more, none of the columns can be
004951 ** considered to match an ORDER BY term.
004952 */
004953 if( (eOp & eqOpMask)!=0 ){
004954 if( eOp & (WO_ISNULL|WO_IS) ){
004955 testcase( eOp & WO_ISNULL );
004956 testcase( eOp & WO_IS );
004957 testcase( isOrderDistinct );
004958 isOrderDistinct = 0;
004959 }
004960 continue;
004961 }else if( ALWAYS(eOp & WO_IN) ){
004962 /* ALWAYS() justification: eOp is an equality operator due to the
004963 ** j<pLoop->u.btree.nEq constraint above. Any equality other
004964 ** than WO_IN is captured by the previous "if". So this one
004965 ** always has to be WO_IN. */
004966 Expr *pX = pLoop->aLTerm[j]->pExpr;
004967 for(i=j+1; i<pLoop->u.btree.nEq; i++){
004968 if( pLoop->aLTerm[i]->pExpr==pX ){
004969 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
004970 bOnce = 0;
004971 break;
004972 }
004973 }
004974 }
004975 }
004976
004977 /* Get the column number in the table (iColumn) and sort order
004978 ** (revIdx) for the j-th column of the index.
004979 */
004980 if( pIndex ){
004981 iColumn = pIndex->aiColumn[j];
004982 revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC;
004983 if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID;
004984 }else{
004985 iColumn = XN_ROWID;
004986 revIdx = 0;
004987 }
004988
004989 /* An unconstrained column that might be NULL means that this
004990 ** WhereLoop is not well-ordered. tag-20210426-1
004991 */
004992 if( isOrderDistinct ){
004993 if( iColumn>=0
004994 && j>=pLoop->u.btree.nEq
004995 && pIndex->pTable->aCol[iColumn].notNull==0
004996 ){
004997 isOrderDistinct = 0;
004998 }
004999 if( iColumn==XN_EXPR ){
005000 isOrderDistinct = 0;
005001 }
005002 }
005003
005004 /* Find the ORDER BY term that corresponds to the j-th column
005005 ** of the index and mark that ORDER BY term off
005006 */
005007 isMatch = 0;
005008 for(i=0; bOnce && i<nOrderBy; i++){
005009 if( MASKBIT(i) & obSat ) continue;
005010 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
005011 testcase( wctrlFlags & WHERE_GROUPBY );
005012 testcase( wctrlFlags & WHERE_DISTINCTBY );
005013 if( NEVER(pOBExpr==0) ) continue;
005014 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
005015 if( iColumn>=XN_ROWID ){
005016 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
005017 if( pOBExpr->iTable!=iCur ) continue;
005018 if( pOBExpr->iColumn!=iColumn ) continue;
005019 }else{
005020 Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr;
005021 if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){
005022 continue;
005023 }
005024 }
005025 if( iColumn!=XN_ROWID ){
005026 pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
005027 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
005028 }
005029 if( wctrlFlags & WHERE_DISTINCTBY ){
005030 pLoop->u.btree.nDistinctCol = j+1;
005031 }
005032 isMatch = 1;
005033 break;
005034 }
005035 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
005036 /* Make sure the sort order is compatible in an ORDER BY clause.
005037 ** Sort order is irrelevant for a GROUP BY clause. */
005038 if( revSet ){
005039 if( (rev ^ revIdx)
005040 != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC)
005041 ){
005042 isMatch = 0;
005043 }
005044 }else{
005045 rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC);
005046 if( rev ) *pRevMask |= MASKBIT(iLoop);
005047 revSet = 1;
005048 }
005049 }
005050 if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){
005051 if( j==pLoop->u.btree.nEq ){
005052 pLoop->wsFlags |= WHERE_BIGNULL_SORT;
005053 }else{
005054 isMatch = 0;
005055 }
005056 }
005057 if( isMatch ){
005058 if( iColumn==XN_ROWID ){
005059 testcase( distinctColumns==0 );
005060 distinctColumns = 1;
005061 }
005062 obSat |= MASKBIT(i);
005063 }else{
005064 /* No match found */
005065 if( j==0 || j<nKeyCol ){
005066 testcase( isOrderDistinct!=0 );
005067 isOrderDistinct = 0;
005068 }
005069 break;
005070 }
005071 } /* end Loop over all index columns */
005072 if( distinctColumns ){
005073 testcase( isOrderDistinct==0 );
005074 isOrderDistinct = 1;
005075 }
005076 } /* end-if not one-row */
005077
005078 /* Mark off any other ORDER BY terms that reference pLoop */
005079 if( isOrderDistinct ){
005080 orderDistinctMask |= pLoop->maskSelf;
005081 for(i=0; i<nOrderBy; i++){
005082 Expr *p;
005083 Bitmask mTerm;
005084 if( MASKBIT(i) & obSat ) continue;
005085 p = pOrderBy->a[i].pExpr;
005086 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
005087 if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue;
005088 if( (mTerm&~orderDistinctMask)==0 ){
005089 obSat |= MASKBIT(i);
005090 }
005091 }
005092 }
005093 } /* End the loop over all WhereLoops from outer-most down to inner-most */
005094 if( obSat==obDone ) return (i8)nOrderBy;
005095 if( !isOrderDistinct ){
005096 for(i=nOrderBy-1; i>0; i--){
005097 Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0;
005098 if( (obSat&m)==m ) return i;
005099 }
005100 return 0;
005101 }
005102 return -1;
005103 }
005104
005105
005106 /*
005107 ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
005108 ** the planner assumes that the specified pOrderBy list is actually a GROUP
005109 ** BY clause - and so any order that groups rows as required satisfies the
005110 ** request.
005111 **
005112 ** Normally, in this case it is not possible for the caller to determine
005113 ** whether or not the rows are really being delivered in sorted order, or
005114 ** just in some other order that provides the required grouping. However,
005115 ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
005116 ** this function may be called on the returned WhereInfo object. It returns
005117 ** true if the rows really will be sorted in the specified order, or false
005118 ** otherwise.
005119 **
005120 ** For example, assuming:
005121 **
005122 ** CREATE INDEX i1 ON t1(x, Y);
005123 **
005124 ** then
005125 **
005126 ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
005127 ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
005128 */
005129 int sqlite3WhereIsSorted(WhereInfo *pWInfo){
005130 assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) );
005131 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
005132 return pWInfo->sorted;
005133 }
005134
005135 #ifdef WHERETRACE_ENABLED
005136 /* For debugging use only: */
005137 static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
005138 static char zName[65];
005139 int i;
005140 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; }
005141 if( pLast ) zName[i++] = pLast->cId;
005142 zName[i] = 0;
005143 return zName;
005144 }
005145 #endif
005146
005147 /*
005148 ** Return the cost of sorting nRow rows, assuming that the keys have
005149 ** nOrderby columns and that the first nSorted columns are already in
005150 ** order.
005151 */
005152 static LogEst whereSortingCost(
005153 WhereInfo *pWInfo, /* Query planning context */
005154 LogEst nRow, /* Estimated number of rows to sort */
005155 int nOrderBy, /* Number of ORDER BY clause terms */
005156 int nSorted /* Number of initial ORDER BY terms naturally in order */
005157 ){
005158 /* Estimated cost of a full external sort, where N is
005159 ** the number of rows to sort is:
005160 **
005161 ** cost = (K * N * log(N)).
005162 **
005163 ** Or, if the order-by clause has X terms but only the last Y
005164 ** terms are out of order, then block-sorting will reduce the
005165 ** sorting cost to:
005166 **
005167 ** cost = (K * N * log(N)) * (Y/X)
005168 **
005169 ** The constant K is at least 2.0 but will be larger if there are a
005170 ** large number of columns to be sorted, as the sorting time is
005171 ** proportional to the amount of content to be sorted. The algorithm
005172 ** does not currently distinguish between fat columns (BLOBs and TEXTs)
005173 ** and skinny columns (INTs). It just uses the number of columns as
005174 ** an approximation for the row width.
005175 **
005176 ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort
005177 ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert.
005178 */
005179 LogEst rSortCost, nCol;
005180 assert( pWInfo->pSelect!=0 );
005181 assert( pWInfo->pSelect->pEList!=0 );
005182 /* TUNING: sorting cost proportional to the number of output columns: */
005183 nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30);
005184 rSortCost = nRow + nCol;
005185 if( nSorted>0 ){
005186 /* Scale the result by (Y/X) */
005187 rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
005188 }
005189
005190 /* Multiple by log(M) where M is the number of output rows.
005191 ** Use the LIMIT for M if it is smaller. Or if this sort is for
005192 ** a DISTINCT operator, M will be the number of distinct output
005193 ** rows, so fudge it downwards a bit.
005194 */
005195 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){
005196 rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */
005197 if( nSorted!=0 ){
005198 rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */
005199 }
005200 if( pWInfo->iLimit<nRow ){
005201 nRow = pWInfo->iLimit;
005202 }
005203 }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){
005204 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
005205 ** reduces the number of output rows by a factor of 2 */
005206 if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); }
005207 }
005208 rSortCost += estLog(nRow);
005209 return rSortCost;
005210 }
005211
005212 /*
005213 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
005214 ** attempts to find the lowest cost path that visits each WhereLoop
005215 ** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
005216 **
005217 ** Assume that the total number of output rows that will need to be sorted
005218 ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
005219 ** costs if nRowEst==0.
005220 **
005221 ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
005222 ** error occurs.
005223 */
005224 static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
005225 int mxChoice; /* Maximum number of simultaneous paths tracked */
005226 int nLoop; /* Number of terms in the join */
005227 Parse *pParse; /* Parsing context */
005228 int iLoop; /* Loop counter over the terms of the join */
005229 int ii, jj; /* Loop counters */
005230 int mxI = 0; /* Index of next entry to replace */
005231 int nOrderBy; /* Number of ORDER BY clause terms */
005232 LogEst mxCost = 0; /* Maximum cost of a set of paths */
005233 LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */
005234 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
005235 WherePath *aFrom; /* All nFrom paths at the previous level */
005236 WherePath *aTo; /* The nTo best paths at the current level */
005237 WherePath *pFrom; /* An element of aFrom[] that we are working on */
005238 WherePath *pTo; /* An element of aTo[] that we are working on */
005239 WhereLoop *pWLoop; /* One of the WhereLoop objects */
005240 WhereLoop **pX; /* Used to divy up the pSpace memory */
005241 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
005242 char *pSpace; /* Temporary memory used by this routine */
005243 int nSpace; /* Bytes of space allocated at pSpace */
005244
005245 pParse = pWInfo->pParse;
005246 nLoop = pWInfo->nLevel;
005247 /* TUNING: For simple queries, only the best path is tracked.
005248 ** For 2-way joins, the 5 best paths are followed.
005249 ** For joins of 3 or more tables, track the 10 best paths */
005250 mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10);
005251 assert( nLoop<=pWInfo->pTabList->nSrc );
005252 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n",
005253 nRowEst, pParse->nQueryLoop));
005254
005255 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
005256 ** case the purpose of this call is to estimate the number of rows returned
005257 ** by the overall query. Once this estimate has been obtained, the caller
005258 ** will invoke this function a second time, passing the estimate as the
005259 ** nRowEst parameter. */
005260 if( pWInfo->pOrderBy==0 || nRowEst==0 ){
005261 nOrderBy = 0;
005262 }else{
005263 nOrderBy = pWInfo->pOrderBy->nExpr;
005264 }
005265
005266 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
005267 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
005268 nSpace += sizeof(LogEst) * nOrderBy;
005269 pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace);
005270 if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
005271 aTo = (WherePath*)pSpace;
005272 aFrom = aTo+mxChoice;
005273 memset(aFrom, 0, sizeof(aFrom[0]));
005274 pX = (WhereLoop**)(aFrom+mxChoice);
005275 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
005276 pFrom->aLoop = pX;
005277 }
005278 if( nOrderBy ){
005279 /* If there is an ORDER BY clause and it is not being ignored, set up
005280 ** space for the aSortCost[] array. Each element of the aSortCost array
005281 ** is either zero - meaning it has not yet been initialized - or the
005282 ** cost of sorting nRowEst rows of data where the first X terms of
005283 ** the ORDER BY clause are already in order, where X is the array
005284 ** index. */
005285 aSortCost = (LogEst*)pX;
005286 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
005287 }
005288 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
005289 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
005290
005291 /* Seed the search with a single WherePath containing zero WhereLoops.
005292 **
005293 ** TUNING: Do not let the number of iterations go above 28. If the cost
005294 ** of computing an automatic index is not paid back within the first 28
005295 ** rows, then do not use the automatic index. */
005296 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) );
005297 nFrom = 1;
005298 assert( aFrom[0].isOrdered==0 );
005299 if( nOrderBy ){
005300 /* If nLoop is zero, then there are no FROM terms in the query. Since
005301 ** in this case the query may return a maximum of one row, the results
005302 ** are already in the requested order. Set isOrdered to nOrderBy to
005303 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
005304 ** -1, indicating that the result set may or may not be ordered,
005305 ** depending on the loops added to the current plan. */
005306 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
005307 }
005308
005309 /* Compute successively longer WherePaths using the previous generation
005310 ** of WherePaths as the basis for the next. Keep track of the mxChoice
005311 ** best paths at each generation */
005312 for(iLoop=0; iLoop<nLoop; iLoop++){
005313 nTo = 0;
005314 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
005315 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005316 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
005317 LogEst rCost; /* Cost of path (pFrom+pWLoop) */
005318 LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
005319 i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */
005320 Bitmask maskNew; /* Mask of src visited by (..) */
005321 Bitmask revMask; /* Mask of rev-order loops for (..) */
005322
005323 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
005324 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
005325 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){
005326 /* Do not use an automatic index if the this loop is expected
005327 ** to run less than 1.25 times. It is tempting to also exclude
005328 ** automatic index usage on an outer loop, but sometimes an automatic
005329 ** index is useful in the outer loop of a correlated subquery. */
005330 assert( 10==sqlite3LogEst(2) );
005331 continue;
005332 }
005333
005334 /* At this point, pWLoop is a candidate to be the next loop.
005335 ** Compute its cost */
005336 rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
005337 rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
005338 nOut = pFrom->nRow + pWLoop->nOut;
005339 maskNew = pFrom->maskLoop | pWLoop->maskSelf;
005340 isOrdered = pFrom->isOrdered;
005341 if( isOrdered<0 ){
005342 revMask = 0;
005343 isOrdered = wherePathSatisfiesOrderBy(pWInfo,
005344 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
005345 iLoop, pWLoop, &revMask);
005346 }else{
005347 revMask = pFrom->revLoop;
005348 }
005349 if( isOrdered>=0 && isOrdered<nOrderBy ){
005350 if( aSortCost[isOrdered]==0 ){
005351 aSortCost[isOrdered] = whereSortingCost(
005352 pWInfo, nRowEst, nOrderBy, isOrdered
005353 );
005354 }
005355 /* TUNING: Add a small extra penalty (3) to sorting as an
005356 ** extra encouragement to the query planner to select a plan
005357 ** where the rows emerge in the correct order without any sorting
005358 ** required. */
005359 rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3;
005360
005361 WHERETRACE(0x002,
005362 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
005363 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
005364 rUnsorted, rCost));
005365 }else{
005366 rCost = rUnsorted;
005367 rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */
005368 }
005369
005370 /* Check to see if pWLoop should be added to the set of
005371 ** mxChoice best-so-far paths.
005372 **
005373 ** First look for an existing path among best-so-far paths
005374 ** that covers the same set of loops and has the same isOrdered
005375 ** setting as the current path candidate.
005376 **
005377 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
005378 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
005379 ** of legal values for isOrdered, -1..64.
005380 */
005381 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
005382 if( pTo->maskLoop==maskNew
005383 && ((pTo->isOrdered^isOrdered)&0x80)==0
005384 ){
005385 testcase( jj==nTo-1 );
005386 break;
005387 }
005388 }
005389 if( jj>=nTo ){
005390 /* None of the existing best-so-far paths match the candidate. */
005391 if( nTo>=mxChoice
005392 && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
005393 ){
005394 /* The current candidate is no better than any of the mxChoice
005395 ** paths currently in the best-so-far buffer. So discard
005396 ** this candidate as not viable. */
005397 #ifdef WHERETRACE_ENABLED /* 0x4 */
005398 if( sqlite3WhereTrace&0x4 ){
005399 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n",
005400 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005401 isOrdered>=0 ? isOrdered+'0' : '?');
005402 }
005403 #endif
005404 continue;
005405 }
005406 /* If we reach this points it means that the new candidate path
005407 ** needs to be added to the set of best-so-far paths. */
005408 if( nTo<mxChoice ){
005409 /* Increase the size of the aTo set by one */
005410 jj = nTo++;
005411 }else{
005412 /* New path replaces the prior worst to keep count below mxChoice */
005413 jj = mxI;
005414 }
005415 pTo = &aTo[jj];
005416 #ifdef WHERETRACE_ENABLED /* 0x4 */
005417 if( sqlite3WhereTrace&0x4 ){
005418 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n",
005419 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005420 isOrdered>=0 ? isOrdered+'0' : '?');
005421 }
005422 #endif
005423 }else{
005424 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
005425 ** same set of loops and has the same isOrdered setting as the
005426 ** candidate path. Check to see if the candidate should replace
005427 ** pTo or if the candidate should be skipped.
005428 **
005429 ** The conditional is an expanded vector comparison equivalent to:
005430 ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted)
005431 */
005432 if( pTo->rCost<rCost
005433 || (pTo->rCost==rCost
005434 && (pTo->nRow<nOut
005435 || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted)
005436 )
005437 )
005438 ){
005439 #ifdef WHERETRACE_ENABLED /* 0x4 */
005440 if( sqlite3WhereTrace&0x4 ){
005441 sqlite3DebugPrintf(
005442 "Skip %s cost=%-3d,%3d,%3d order=%c",
005443 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005444 isOrdered>=0 ? isOrdered+'0' : '?');
005445 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n",
005446 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005447 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005448 }
005449 #endif
005450 /* Discard the candidate path from further consideration */
005451 testcase( pTo->rCost==rCost );
005452 continue;
005453 }
005454 testcase( pTo->rCost==rCost+1 );
005455 /* Control reaches here if the candidate path is better than the
005456 ** pTo path. Replace pTo with the candidate. */
005457 #ifdef WHERETRACE_ENABLED /* 0x4 */
005458 if( sqlite3WhereTrace&0x4 ){
005459 sqlite3DebugPrintf(
005460 "Update %s cost=%-3d,%3d,%3d order=%c",
005461 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005462 isOrdered>=0 ? isOrdered+'0' : '?');
005463 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n",
005464 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005465 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005466 }
005467 #endif
005468 }
005469 /* pWLoop is a winner. Add it to the set of best so far */
005470 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf;
005471 pTo->revLoop = revMask;
005472 pTo->nRow = nOut;
005473 pTo->rCost = rCost;
005474 pTo->rUnsorted = rUnsorted;
005475 pTo->isOrdered = isOrdered;
005476 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
005477 pTo->aLoop[iLoop] = pWLoop;
005478 if( nTo>=mxChoice ){
005479 mxI = 0;
005480 mxCost = aTo[0].rCost;
005481 mxUnsorted = aTo[0].nRow;
005482 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
005483 if( pTo->rCost>mxCost
005484 || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted)
005485 ){
005486 mxCost = pTo->rCost;
005487 mxUnsorted = pTo->rUnsorted;
005488 mxI = jj;
005489 }
005490 }
005491 }
005492 }
005493 }
005494
005495 #ifdef WHERETRACE_ENABLED /* >=2 */
005496 if( sqlite3WhereTrace & 0x02 ){
005497 sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
005498 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
005499 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
005500 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005501 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
005502 if( pTo->isOrdered>0 ){
005503 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
005504 }else{
005505 sqlite3DebugPrintf("\n");
005506 }
005507 }
005508 }
005509 #endif
005510
005511 /* Swap the roles of aFrom and aTo for the next generation */
005512 pFrom = aTo;
005513 aTo = aFrom;
005514 aFrom = pFrom;
005515 nFrom = nTo;
005516 }
005517
005518 if( nFrom==0 ){
005519 sqlite3ErrorMsg(pParse, "no query solution");
005520 sqlite3StackFreeNN(pParse->db, pSpace);
005521 return SQLITE_ERROR;
005522 }
005523
005524 /* Find the lowest cost path. pFrom will be left pointing to that path */
005525 pFrom = aFrom;
005526 for(ii=1; ii<nFrom; ii++){
005527 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
005528 }
005529 assert( pWInfo->nLevel==nLoop );
005530 /* Load the lowest cost path into pWInfo */
005531 for(iLoop=0; iLoop<nLoop; iLoop++){
005532 WhereLevel *pLevel = pWInfo->a + iLoop;
005533 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop];
005534 pLevel->iFrom = pWLoop->iTab;
005535 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;
005536 }
005537 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0
005538 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0
005539 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP
005540 && nRowEst
005541 ){
005542 Bitmask notUsed;
005543 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
005544 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used);
005545 if( rc==pWInfo->pResultSet->nExpr ){
005546 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005547 }
005548 }
005549 pWInfo->bOrderedInnerLoop = 0;
005550 if( pWInfo->pOrderBy ){
005551 pWInfo->nOBSat = pFrom->isOrdered;
005552 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
005553 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
005554 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005555 }
005556 /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */
005557 assert( pWInfo->pSelect->pOrderBy==0
005558 || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr );
005559 }else{
005560 pWInfo->revMask = pFrom->revLoop;
005561 if( pWInfo->nOBSat<=0 ){
005562 pWInfo->nOBSat = 0;
005563 if( nLoop>0 ){
005564 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
005565 if( (wsFlags & WHERE_ONEROW)==0
005566 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
005567 ){
005568 Bitmask m = 0;
005569 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
005570 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
005571 testcase( wsFlags & WHERE_IPK );
005572 testcase( wsFlags & WHERE_COLUMN_IN );
005573 if( rc==pWInfo->pOrderBy->nExpr ){
005574 pWInfo->bOrderedInnerLoop = 1;
005575 pWInfo->revMask = m;
005576 }
005577 }
005578 }
005579 }else if( nLoop
005580 && pWInfo->nOBSat==1
005581 && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0
005582 ){
005583 pWInfo->bOrderedInnerLoop = 1;
005584 }
005585 }
005586 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
005587 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
005588 ){
005589 Bitmask revMask = 0;
005590 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
005591 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
005592 );
005593 assert( pWInfo->sorted==0 );
005594 if( nOrder==pWInfo->pOrderBy->nExpr ){
005595 pWInfo->sorted = 1;
005596 pWInfo->revMask = revMask;
005597 }
005598 }
005599 }
005600
005601 pWInfo->nRowOut = pFrom->nRow;
005602
005603 /* Free temporary memory and return success */
005604 sqlite3StackFreeNN(pParse->db, pSpace);
005605 return SQLITE_OK;
005606 }
005607
005608 /*
005609 ** This routine implements a heuristic designed to improve query planning.
005610 ** This routine is called in between the first and second call to
005611 ** wherePathSolver(). Hence the name "Interstage" "Heuristic".
005612 **
005613 ** The first call to wherePathSolver() (hereafter just "solver()") computes
005614 ** the best path without regard to the order of the outputs. The second call
005615 ** to the solver() builds upon the first call to try to find an alternative
005616 ** path that satisfies the ORDER BY clause.
005617 **
005618 ** This routine looks at the results of the first solver() run, and for
005619 ** every FROM clause term in the resulting query plan that uses an equality
005620 ** constraint against an index, disable other WhereLoops for that same
005621 ** FROM clause term that would try to do a full-table scan. This prevents
005622 ** an index search from being converted into a full-table scan in order to
005623 ** satisfy an ORDER BY clause, since even though we might get slightly better
005624 ** performance using the full-scan without sorting if the output size
005625 ** estimates are very precise, we might also get severe performance
005626 ** degradation using the full-scan if the output size estimate is too large.
005627 ** It is better to err on the side of caution.
005628 **
005629 ** Except, if the first solver() call generated a full-table scan in an outer
005630 ** loop then stop this analysis at the first full-scan, since the second
005631 ** solver() run might try to swap that full-scan for another in order to
005632 ** get the output into the correct order. In other words, we allow a
005633 ** rewrite like this:
005634 **
005635 ** First Solver() Second Solver()
005636 ** |-- SCAN t1 |-- SCAN t2
005637 ** |-- SEARCH t2 `-- SEARCH t1
005638 ** `-- SORT USING B-TREE
005639 **
005640 ** The purpose of this routine is to disallow rewrites such as:
005641 **
005642 ** First Solver() Second Solver()
005643 ** |-- SEARCH t1 |-- SCAN t2 <--- bad!
005644 ** |-- SEARCH t2 `-- SEARCH t1
005645 ** `-- SORT USING B-TREE
005646 **
005647 ** See test cases in test/whereN.test for the real-world query that
005648 ** originally provoked this heuristic.
005649 */
005650 static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){
005651 int i;
005652 #ifdef WHERETRACE_ENABLED
005653 int once = 0;
005654 #endif
005655 for(i=0; i<pWInfo->nLevel; i++){
005656 WhereLoop *p = pWInfo->a[i].pWLoop;
005657 if( p==0 ) break;
005658 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue;
005659 if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){
005660 u8 iTab = p->iTab;
005661 WhereLoop *pLoop;
005662 for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){
005663 if( pLoop->iTab!=iTab ) continue;
005664 if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){
005665 /* Auto-index and index-constrained loops allowed to remain */
005666 continue;
005667 }
005668 #ifdef WHERETRACE_ENABLED
005669 if( sqlite3WhereTrace & 0x80 ){
005670 if( once==0 ){
005671 sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n");
005672 once = 1;
005673 }
005674 sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC);
005675 }
005676 #endif /* WHERETRACE_ENABLED */
005677 pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */
005678 }
005679 }else{
005680 break;
005681 }
005682 }
005683 }
005684
005685 /*
005686 ** Most queries use only a single table (they are not joins) and have
005687 ** simple == constraints against indexed fields. This routine attempts
005688 ** to plan those simple cases using much less ceremony than the
005689 ** general-purpose query planner, and thereby yield faster sqlite3_prepare()
005690 ** times for the common case.
005691 **
005692 ** Return non-zero on success, if this query can be handled by this
005693 ** no-frills query planner. Return zero if this query needs the
005694 ** general-purpose query planner.
005695 */
005696 static int whereShortCut(WhereLoopBuilder *pBuilder){
005697 WhereInfo *pWInfo;
005698 SrcItem *pItem;
005699 WhereClause *pWC;
005700 WhereTerm *pTerm;
005701 WhereLoop *pLoop;
005702 int iCur;
005703 int j;
005704 Table *pTab;
005705 Index *pIdx;
005706 WhereScan scan;
005707
005708 pWInfo = pBuilder->pWInfo;
005709 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
005710 assert( pWInfo->pTabList->nSrc>=1 );
005711 pItem = pWInfo->pTabList->a;
005712 pTab = pItem->pTab;
005713 if( IsVirtual(pTab) ) return 0;
005714 if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){
005715 testcase( pItem->fg.isIndexedBy );
005716 testcase( pItem->fg.notIndexed );
005717 return 0;
005718 }
005719 iCur = pItem->iCursor;
005720 pWC = &pWInfo->sWC;
005721 pLoop = pBuilder->pNew;
005722 pLoop->wsFlags = 0;
005723 pLoop->nSkip = 0;
005724 pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0);
005725 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
005726 if( pTerm ){
005727 testcase( pTerm->eOperator & WO_IS );
005728 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
005729 pLoop->aLTerm[0] = pTerm;
005730 pLoop->nLTerm = 1;
005731 pLoop->u.btree.nEq = 1;
005732 /* TUNING: Cost of a rowid lookup is 10 */
005733 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */
005734 }else{
005735 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
005736 int opMask;
005737 assert( pLoop->aLTermSpace==pLoop->aLTerm );
005738 if( !IsUniqueIndex(pIdx)
005739 || pIdx->pPartIdxWhere!=0
005740 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
005741 ) continue;
005742 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
005743 for(j=0; j<pIdx->nKeyCol; j++){
005744 pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx);
005745 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
005746 if( pTerm==0 ) break;
005747 testcase( pTerm->eOperator & WO_IS );
005748 pLoop->aLTerm[j] = pTerm;
005749 }
005750 if( j!=pIdx->nKeyCol ) continue;
005751 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED;
005752 if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){
005753 pLoop->wsFlags |= WHERE_IDX_ONLY;
005754 }
005755 pLoop->nLTerm = j;
005756 pLoop->u.btree.nEq = j;
005757 pLoop->u.btree.pIndex = pIdx;
005758 /* TUNING: Cost of a unique index lookup is 15 */
005759 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */
005760 break;
005761 }
005762 }
005763 if( pLoop->wsFlags ){
005764 pLoop->nOut = (LogEst)1;
005765 pWInfo->a[0].pWLoop = pLoop;
005766 assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] );
005767 pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */
005768 pWInfo->a[0].iTabCur = iCur;
005769 pWInfo->nRowOut = 1;
005770 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr;
005771 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
005772 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
005773 }
005774 if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS;
005775 #ifdef SQLITE_DEBUG
005776 pLoop->cId = '0';
005777 #endif
005778 #ifdef WHERETRACE_ENABLED
005779 if( sqlite3WhereTrace & 0x02 ){
005780 sqlite3DebugPrintf("whereShortCut() used to compute solution\n");
005781 }
005782 #endif
005783 return 1;
005784 }
005785 return 0;
005786 }
005787
005788 /*
005789 ** Helper function for exprIsDeterministic().
005790 */
005791 static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){
005792 if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){
005793 pWalker->eCode = 0;
005794 return WRC_Abort;
005795 }
005796 return WRC_Continue;
005797 }
005798
005799 /*
005800 ** Return true if the expression contains no non-deterministic SQL
005801 ** functions. Do not consider non-deterministic SQL functions that are
005802 ** part of sub-select statements.
005803 */
005804 static int exprIsDeterministic(Expr *p){
005805 Walker w;
005806 memset(&w, 0, sizeof(w));
005807 w.eCode = 1;
005808 w.xExprCallback = exprNodeIsDeterministic;
005809 w.xSelectCallback = sqlite3SelectWalkFail;
005810 sqlite3WalkExpr(&w, p);
005811 return w.eCode;
005812 }
005813
005814
005815 #ifdef WHERETRACE_ENABLED
005816 /*
005817 ** Display all WhereLoops in pWInfo
005818 */
005819 static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
005820 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */
005821 WhereLoop *p;
005822 int i;
005823 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
005824 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
005825 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
005826 p->cId = zLabel[i%(sizeof(zLabel)-1)];
005827 sqlite3WhereLoopPrint(p, pWC);
005828 }
005829 }
005830 }
005831 # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
005832 #else
005833 # define WHERETRACE_ALL_LOOPS(W,C)
005834 #endif
005835
005836 /* Attempt to omit tables from a join that do not affect the result.
005837 ** For a table to not affect the result, the following must be true:
005838 **
005839 ** 1) The query must not be an aggregate.
005840 ** 2) The table must be the RHS of a LEFT JOIN.
005841 ** 3) Either the query must be DISTINCT, or else the ON or USING clause
005842 ** must contain a constraint that limits the scan of the table to
005843 ** at most a single row.
005844 ** 4) The table must not be referenced by any part of the query apart
005845 ** from its own USING or ON clause.
005846 ** 5) The table must not have an inner-join ON or USING clause if there is
005847 ** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause
005848 ** might move from the right side to the left side of the RIGHT JOIN.
005849 ** Note: Due to (2), this condition can only arise if the table is
005850 ** the right-most table of a subquery that was flattened into the
005851 ** main query and that subquery was the right-hand operand of an
005852 ** inner join that held an ON or USING clause.
005853 ** 6) The ORDER BY clause has 63 or fewer terms
005854 ** 7) The omit-noop-join optimization is enabled.
005855 **
005856 ** Items (1), (6), and (7) are checked by the caller.
005857 **
005858 ** For example, given:
005859 **
005860 ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
005861 ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
005862 ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
005863 **
005864 ** then table t2 can be omitted from the following:
005865 **
005866 ** SELECT v1, v3 FROM t1
005867 ** LEFT JOIN t2 ON (t1.ipk=t2.ipk)
005868 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
005869 **
005870 ** or from:
005871 **
005872 ** SELECT DISTINCT v1, v3 FROM t1
005873 ** LEFT JOIN t2
005874 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
005875 */
005876 static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
005877 WhereInfo *pWInfo,
005878 Bitmask notReady
005879 ){
005880 int i;
005881 Bitmask tabUsed;
005882 int hasRightJoin;
005883
005884 /* Preconditions checked by the caller */
005885 assert( pWInfo->nLevel>=2 );
005886 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) );
005887
005888 /* These two preconditions checked by the caller combine to guarantee
005889 ** condition (1) of the header comment */
005890 assert( pWInfo->pResultSet!=0 );
005891 assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) );
005892
005893 tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet);
005894 if( pWInfo->pOrderBy ){
005895 tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy);
005896 }
005897 hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0;
005898 for(i=pWInfo->nLevel-1; i>=1; i--){
005899 WhereTerm *pTerm, *pEnd;
005900 SrcItem *pItem;
005901 WhereLoop *pLoop;
005902 pLoop = pWInfo->a[i].pWLoop;
005903 pItem = &pWInfo->pTabList->a[pLoop->iTab];
005904 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue;
005905 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0
005906 && (pLoop->wsFlags & WHERE_ONEROW)==0
005907 ){
005908 continue;
005909 }
005910 if( (tabUsed & pLoop->maskSelf)!=0 ) continue;
005911 pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm;
005912 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
005913 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
005914 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON)
005915 || pTerm->pExpr->w.iJoin!=pItem->iCursor
005916 ){
005917 break;
005918 }
005919 }
005920 if( hasRightJoin
005921 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
005922 && pTerm->pExpr->w.iJoin==pItem->iCursor
005923 ){
005924 break; /* restriction (5) */
005925 }
005926 }
005927 if( pTerm<pEnd ) continue;
005928 WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId));
005929 notReady &= ~pLoop->maskSelf;
005930 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
005931 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
005932 pTerm->wtFlags |= TERM_CODED;
005933 }
005934 }
005935 if( i!=pWInfo->nLevel-1 ){
005936 int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
005937 memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
005938 }
005939 pWInfo->nLevel--;
005940 assert( pWInfo->nLevel>0 );
005941 }
005942 return notReady;
005943 }
005944
005945 /*
005946 ** Check to see if there are any SEARCH loops that might benefit from
005947 ** using a Bloom filter. Consider a Bloom filter if:
005948 **
005949 ** (1) The SEARCH happens more than N times where N is the number
005950 ** of rows in the table that is being considered for the Bloom
005951 ** filter.
005952 ** (2) Some searches are expected to find zero rows. (This is determined
005953 ** by the WHERE_SELFCULL flag on the term.)
005954 ** (3) Bloom-filter processing is not disabled. (Checked by the
005955 ** caller.)
005956 ** (4) The size of the table being searched is known by ANALYZE.
005957 **
005958 ** This block of code merely checks to see if a Bloom filter would be
005959 ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
005960 ** WhereLoop. The implementation of the Bloom filter comes further
005961 ** down where the code for each WhereLoop is generated.
005962 */
005963 static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
005964 const WhereInfo *pWInfo
005965 ){
005966 int i;
005967 LogEst nSearch = 0;
005968
005969 assert( pWInfo->nLevel>=2 );
005970 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
005971 for(i=0; i<pWInfo->nLevel; i++){
005972 WhereLoop *pLoop = pWInfo->a[i].pWLoop;
005973 const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
005974 SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
005975 Table *pTab = pItem->pTab;
005976 if( (pTab->tabFlags & TF_HasStat1)==0 ) break;
005977 pTab->tabFlags |= TF_MaybeReanalyze;
005978 if( i>=1
005979 && (pLoop->wsFlags & reqFlags)==reqFlags
005980 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
005981 && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
005982 ){
005983 if( nSearch > pTab->nRowLogEst ){
005984 testcase( pItem->fg.jointype & JT_LEFT );
005985 pLoop->wsFlags |= WHERE_BLOOMFILTER;
005986 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
005987 WHERETRACE(0xffffffff, (
005988 "-> use Bloom-filter on loop %c because there are ~%.1e "
005989 "lookups into %s which has only ~%.1e rows\n",
005990 pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
005991 (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
005992 }
005993 }
005994 nSearch += pLoop->nOut;
005995 }
005996 }
005997
005998 /*
005999 ** Expression Node callback for sqlite3ExprCanReturnSubtype().
006000 **
006001 ** Only a function call is able to return a subtype. So if the node
006002 ** is not a function call, return WRC_Prune immediately.
006003 **
006004 ** A function call is able to return a subtype if it has the
006005 ** SQLITE_RESULT_SUBTYPE property.
006006 **
006007 ** Assume that every function is able to pass-through a subtype from
006008 ** one of its argument (using sqlite3_result_value()). Most functions
006009 ** are not this way, but we don't have a mechanism to distinguish those
006010 ** that are from those that are not, so assume they all work this way.
006011 ** That means that if one of its arguments is another function and that
006012 ** other function is able to return a subtype, then this function is
006013 ** able to return a subtype.
006014 */
006015 static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){
006016 int n;
006017 FuncDef *pDef;
006018 sqlite3 *db;
006019 if( pExpr->op!=TK_FUNCTION ){
006020 return WRC_Prune;
006021 }
006022 assert( ExprUseXList(pExpr) );
006023 db = pWalker->pParse->db;
006024 n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0;
006025 pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0);
006026 if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){
006027 pWalker->eCode = 1;
006028 return WRC_Prune;
006029 }
006030 return WRC_Continue;
006031 }
006032
006033 /*
006034 ** Return TRUE if expression pExpr is able to return a subtype.
006035 **
006036 ** A TRUE return does not guarantee that a subtype will be returned.
006037 ** It only indicates that a subtype return is possible. False positives
006038 ** are acceptable as they only disable an optimization. False negatives,
006039 ** on the other hand, can lead to incorrect answers.
006040 */
006041 static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){
006042 Walker w;
006043 memset(&w, 0, sizeof(w));
006044 w.pParse = pParse;
006045 w.xExprCallback = exprNodeCanReturnSubtype;
006046 sqlite3WalkExpr(&w, pExpr);
006047 return w.eCode;
006048 }
006049
006050 /*
006051 ** The index pIdx is used by a query and contains one or more expressions.
006052 ** In other words pIdx is an index on an expression. iIdxCur is the cursor
006053 ** number for the index and iDataCur is the cursor number for the corresponding
006054 ** table.
006055 **
006056 ** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for
006057 ** each of the expressions in the index so that the expression code generator
006058 ** will know to replace occurrences of the indexed expression with
006059 ** references to the corresponding column of the index.
006060 */
006061 static SQLITE_NOINLINE void whereAddIndexedExpr(
006062 Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */
006063 Index *pIdx, /* The index-on-expression that contains the expressions */
006064 int iIdxCur, /* Cursor number for pIdx */
006065 SrcItem *pTabItem /* The FROM clause entry for the table */
006066 ){
006067 int i;
006068 IndexedExpr *p;
006069 Table *pTab;
006070 assert( pIdx->bHasExpr );
006071 pTab = pIdx->pTable;
006072 for(i=0; i<pIdx->nColumn; i++){
006073 Expr *pExpr;
006074 int j = pIdx->aiColumn[i];
006075 if( j==XN_EXPR ){
006076 pExpr = pIdx->aColExpr->a[i].pExpr;
006077 }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
006078 pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
006079 }else{
006080 continue;
006081 }
006082 if( sqlite3ExprIsConstant(0,pExpr) ) continue;
006083 if( pExpr->op==TK_FUNCTION && sqlite3ExprCanReturnSubtype(pParse,pExpr) ){
006084 /* Functions that might set a subtype should not be replaced by the
006085 ** value taken from an expression index since the index omits the
006086 ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */
006087 continue;
006088 }
006089 p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
006090 if( p==0 ) break;
006091 p->pIENext = pParse->pIdxEpr;
006092 #ifdef WHERETRACE_ENABLED
006093 if( sqlite3WhereTrace & 0x200 ){
006094 sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i);
006095 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr);
006096 }
006097 #endif
006098 p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
006099 p->iDataCur = pTabItem->iCursor;
006100 p->iIdxCur = iIdxCur;
006101 p->iIdxCol = i;
006102 p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
006103 if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){
006104 p->aff = pIdx->zColAff[i];
006105 }
006106 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
006107 p->zIdxName = pIdx->zName;
006108 #endif
006109 pParse->pIdxEpr = p;
006110 if( p->pIENext==0 ){
006111 void *pArg = (void*)&pParse->pIdxEpr;
006112 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
006113 }
006114 }
006115 }
006116
006117 /*
006118 ** Set the reverse-scan order mask to one for all tables in the query
006119 ** with the exception of MATERIALIZED common table expressions that have
006120 ** their own internal ORDER BY clauses.
006121 **
006122 ** This implements the PRAGMA reverse_unordered_selects=ON setting.
006123 ** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
006124 */
006125 static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){
006126 int ii;
006127 for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){
006128 SrcItem *pItem = &pWInfo->pTabList->a[ii];
006129 if( !pItem->fg.isCte
006130 || pItem->u2.pCteUse->eM10d!=M10d_Yes
006131 || NEVER(pItem->pSelect==0)
006132 || pItem->pSelect->pOrderBy==0
006133 ){
006134 pWInfo->revMask |= MASKBIT(ii);
006135 }
006136 }
006137 }
006138
006139 /*
006140 ** Generate the beginning of the loop used for WHERE clause processing.
006141 ** The return value is a pointer to an opaque structure that contains
006142 ** information needed to terminate the loop. Later, the calling routine
006143 ** should invoke sqlite3WhereEnd() with the return value of this function
006144 ** in order to complete the WHERE clause processing.
006145 **
006146 ** If an error occurs, this routine returns NULL.
006147 **
006148 ** The basic idea is to do a nested loop, one loop for each table in
006149 ** the FROM clause of a select. (INSERT and UPDATE statements are the
006150 ** same as a SELECT with only a single table in the FROM clause.) For
006151 ** example, if the SQL is this:
006152 **
006153 ** SELECT * FROM t1, t2, t3 WHERE ...;
006154 **
006155 ** Then the code generated is conceptually like the following:
006156 **
006157 ** foreach row1 in t1 do \ Code generated
006158 ** foreach row2 in t2 do |-- by sqlite3WhereBegin()
006159 ** foreach row3 in t3 do /
006160 ** ...
006161 ** end \ Code generated
006162 ** end |-- by sqlite3WhereEnd()
006163 ** end /
006164 **
006165 ** Note that the loops might not be nested in the order in which they
006166 ** appear in the FROM clause if a different order is better able to make
006167 ** use of indices. Note also that when the IN operator appears in
006168 ** the WHERE clause, it might result in additional nested loops for
006169 ** scanning through all values on the right-hand side of the IN.
006170 **
006171 ** There are Btree cursors associated with each table. t1 uses cursor
006172 ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
006173 ** And so forth. This routine generates code to open those VDBE cursors
006174 ** and sqlite3WhereEnd() generates the code to close them.
006175 **
006176 ** The code that sqlite3WhereBegin() generates leaves the cursors named
006177 ** in pTabList pointing at their appropriate entries. The [...] code
006178 ** can use OP_Column and OP_Rowid opcodes on these cursors to extract
006179 ** data from the various tables of the loop.
006180 **
006181 ** If the WHERE clause is empty, the foreach loops must each scan their
006182 ** entire tables. Thus a three-way join is an O(N^3) operation. But if
006183 ** the tables have indices and there are terms in the WHERE clause that
006184 ** refer to those indices, a complete table scan can be avoided and the
006185 ** code will run much faster. Most of the work of this routine is checking
006186 ** to see if there are indices that can be used to speed up the loop.
006187 **
006188 ** Terms of the WHERE clause are also used to limit which rows actually
006189 ** make it to the "..." in the middle of the loop. After each "foreach",
006190 ** terms of the WHERE clause that use only terms in that loop and outer
006191 ** loops are evaluated and if false a jump is made around all subsequent
006192 ** inner loops (or around the "..." if the test occurs within the inner-
006193 ** most loop)
006194 **
006195 ** OUTER JOINS
006196 **
006197 ** An outer join of tables t1 and t2 is conceptually coded as follows:
006198 **
006199 ** foreach row1 in t1 do
006200 ** flag = 0
006201 ** foreach row2 in t2 do
006202 ** start:
006203 ** ...
006204 ** flag = 1
006205 ** end
006206 ** if flag==0 then
006207 ** move the row2 cursor to a null row
006208 ** goto start
006209 ** fi
006210 ** end
006211 **
006212 ** ORDER BY CLAUSE PROCESSING
006213 **
006214 ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
006215 ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
006216 ** if there is one. If there is no ORDER BY clause or if this routine
006217 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
006218 **
006219 ** The iIdxCur parameter is the cursor number of an index. If
006220 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
006221 ** to use for OR clause processing. The WHERE clause should use this
006222 ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
006223 ** the first cursor in an array of cursors for all indices. iIdxCur should
006224 ** be used to compute the appropriate cursor depending on which index is
006225 ** used.
006226 */
006227 WhereInfo *sqlite3WhereBegin(
006228 Parse *pParse, /* The parser context */
006229 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
006230 Expr *pWhere, /* The WHERE clause */
006231 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
006232 ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */
006233 Select *pSelect, /* The entire SELECT statement */
006234 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
006235 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
006236 ** If WHERE_USE_LIMIT, then the limit amount */
006237 ){
006238 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
006239 int nTabList; /* Number of elements in pTabList */
006240 WhereInfo *pWInfo; /* Will become the return value of this function */
006241 Vdbe *v = pParse->pVdbe; /* The virtual database engine */
006242 Bitmask notReady; /* Cursors that are not yet positioned */
006243 WhereLoopBuilder sWLB; /* The WhereLoop builder */
006244 WhereMaskSet *pMaskSet; /* The expression mask set */
006245 WhereLevel *pLevel; /* A single level in pWInfo->a[] */
006246 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */
006247 int ii; /* Loop counter */
006248 sqlite3 *db; /* Database connection */
006249 int rc; /* Return code */
006250 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */
006251
006252 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
006253 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
006254 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006255 ));
006256
006257 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
006258 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006259 || (wctrlFlags & WHERE_USE_LIMIT)==0 );
006260
006261 /* Variable initialization */
006262 db = pParse->db;
006263 memset(&sWLB, 0, sizeof(sWLB));
006264
006265 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
006266 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
006267 if( pOrderBy && pOrderBy->nExpr>=BMS ){
006268 pOrderBy = 0;
006269 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006270 wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */
006271 }
006272
006273 /* The number of tables in the FROM clause is limited by the number of
006274 ** bits in a Bitmask
006275 */
006276 testcase( pTabList->nSrc==BMS );
006277 if( pTabList->nSrc>BMS ){
006278 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
006279 return 0;
006280 }
006281
006282 /* This function normally generates a nested loop for all tables in
006283 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
006284 ** only generate code for the first table in pTabList and assume that
006285 ** any cursors associated with subsequent tables are uninitialized.
006286 */
006287 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
006288
006289 /* Allocate and initialize the WhereInfo structure that will become the
006290 ** return value. A single allocation is used to store the WhereInfo
006291 ** struct, the contents of WhereInfo.a[], the WhereClause structure
006292 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
006293 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
006294 ** some architectures. Hence the ROUND8() below.
006295 */
006296 nByteWInfo = ROUND8P(sizeof(WhereInfo));
006297 if( nTabList>1 ){
006298 nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel));
006299 }
006300 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
006301 if( db->mallocFailed ){
006302 sqlite3DbFree(db, pWInfo);
006303 pWInfo = 0;
006304 goto whereBeginError;
006305 }
006306 pWInfo->pParse = pParse;
006307 pWInfo->pTabList = pTabList;
006308 pWInfo->pOrderBy = pOrderBy;
006309 #if WHERETRACE_ENABLED
006310 pWInfo->pWhere = pWhere;
006311 #endif
006312 pWInfo->pResultSet = pResultSet;
006313 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
006314 pWInfo->nLevel = nTabList;
006315 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse);
006316 pWInfo->wctrlFlags = wctrlFlags;
006317 pWInfo->iLimit = iAuxArg;
006318 pWInfo->savedNQueryLoop = pParse->nQueryLoop;
006319 pWInfo->pSelect = pSelect;
006320 memset(&pWInfo->nOBSat, 0,
006321 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
006322 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
006323 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */
006324 pMaskSet = &pWInfo->sMaskSet;
006325 pMaskSet->n = 0;
006326 pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be
006327 ** a valid cursor number, to avoid an initial
006328 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */
006329 sWLB.pWInfo = pWInfo;
006330 sWLB.pWC = &pWInfo->sWC;
006331 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo);
006332 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) );
006333 whereLoopInit(sWLB.pNew);
006334 #ifdef SQLITE_DEBUG
006335 sWLB.pNew->cId = '*';
006336 #endif
006337
006338 /* Split the WHERE clause into separate subexpressions where each
006339 ** subexpression is separated by an AND operator.
006340 */
006341 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
006342 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
006343
006344 /* Special case: No FROM clause
006345 */
006346 if( nTabList==0 ){
006347 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
006348 if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0
006349 && OptimizationEnabled(db, SQLITE_DistinctOpt)
006350 ){
006351 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006352 }
006353 if( ALWAYS(pWInfo->pSelect)
006354 && (pWInfo->pSelect->selFlags & SF_MultiValue)==0
006355 ){
006356 ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
006357 }
006358 }else{
006359 /* Assign a bit from the bitmask to every term in the FROM clause.
006360 **
006361 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
006362 **
006363 ** The rule of the previous sentence ensures that if X is the bitmask for
006364 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
006365 ** Knowing the bitmask for all tables to the left of a left join is
006366 ** important. Ticket #3015.
006367 **
006368 ** Note that bitmasks are created for all pTabList->nSrc tables in
006369 ** pTabList, not just the first nTabList tables. nTabList is normally
006370 ** equal to pTabList->nSrc but might be shortened to 1 if the
006371 ** WHERE_OR_SUBCLAUSE flag is set.
006372 */
006373 ii = 0;
006374 do{
006375 createMask(pMaskSet, pTabList->a[ii].iCursor);
006376 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
006377 }while( (++ii)<pTabList->nSrc );
006378 #ifdef SQLITE_DEBUG
006379 {
006380 Bitmask mx = 0;
006381 for(ii=0; ii<pTabList->nSrc; ii++){
006382 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
006383 assert( m>=mx );
006384 mx = m;
006385 }
006386 }
006387 #endif
006388 }
006389
006390 /* Analyze all of the subexpressions. */
006391 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
006392 if( pSelect && pSelect->pLimit ){
006393 sqlite3WhereAddLimit(&pWInfo->sWC, pSelect);
006394 }
006395 if( pParse->nErr ) goto whereBeginError;
006396
006397 /* The False-WHERE-Term-Bypass optimization:
006398 **
006399 ** If there are WHERE terms that are false, then no rows will be output,
006400 ** so skip over all of the code generated here.
006401 **
006402 ** Conditions:
006403 **
006404 ** (1) The WHERE term must not refer to any tables in the join.
006405 ** (2) The term must not come from an ON clause on the
006406 ** right-hand side of a LEFT or FULL JOIN.
006407 ** (3) The term must not come from an ON clause, or there must be
006408 ** no RIGHT or FULL OUTER joins in pTabList.
006409 ** (4) If the expression contains non-deterministic functions
006410 ** that are not within a sub-select. This is not required
006411 ** for correctness but rather to preserves SQLite's legacy
006412 ** behaviour in the following two cases:
006413 **
006414 ** WHERE random()>0; -- eval random() once per row
006415 ** WHERE (SELECT random())>0; -- eval random() just once overall
006416 **
006417 ** Note that the Where term need not be a constant in order for this
006418 ** optimization to apply, though it does need to be constant relative to
006419 ** the current subquery (condition 1). The term might include variables
006420 ** from outer queries so that the value of the term changes from one
006421 ** invocation of the current subquery to the next.
006422 */
006423 for(ii=0; ii<sWLB.pWC->nBase; ii++){
006424 WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */
006425 Expr *pX; /* The expression of pT */
006426 if( pT->wtFlags & TERM_VIRTUAL ) continue;
006427 pX = pT->pExpr;
006428 assert( pX!=0 );
006429 assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) );
006430 if( pT->prereqAll==0 /* Conditions (1) and (2) */
006431 && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */
006432 && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */
006433 && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 )
006434 ){
006435 sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL);
006436 pT->wtFlags |= TERM_CODED;
006437 }
006438 }
006439
006440 if( wctrlFlags & WHERE_WANT_DISTINCT ){
006441 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){
006442 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
006443 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
006444 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006445 pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT;
006446 }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){
006447 /* The DISTINCT marking is pointless. Ignore it. */
006448 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006449 }else if( pOrderBy==0 ){
006450 /* Try to ORDER BY the result set to make distinct processing easier */
006451 pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
006452 pWInfo->pOrderBy = pResultSet;
006453 }
006454 }
006455
006456 /* Construct the WhereLoop objects */
006457 #if defined(WHERETRACE_ENABLED)
006458 if( sqlite3WhereTrace & 0xffffffff ){
006459 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
006460 if( wctrlFlags & WHERE_USE_LIMIT ){
006461 sqlite3DebugPrintf(", limit: %d", iAuxArg);
006462 }
006463 sqlite3DebugPrintf(")\n");
006464 if( sqlite3WhereTrace & 0x8000 ){
006465 Select sSelect;
006466 memset(&sSelect, 0, sizeof(sSelect));
006467 sSelect.selFlags = SF_WhereBegin;
006468 sSelect.pSrc = pTabList;
006469 sSelect.pWhere = pWhere;
006470 sSelect.pOrderBy = pOrderBy;
006471 sSelect.pEList = pResultSet;
006472 sqlite3TreeViewSelect(0, &sSelect, 0);
006473 }
006474 if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */
006475 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n");
006476 sqlite3WhereClausePrint(sWLB.pWC);
006477 }
006478 }
006479 #endif
006480
006481 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
006482 rc = whereLoopAddAll(&sWLB);
006483 if( rc ) goto whereBeginError;
006484
006485 #ifdef SQLITE_ENABLE_STAT4
006486 /* If one or more WhereTerm.truthProb values were used in estimating
006487 ** loop parameters, but then those truthProb values were subsequently
006488 ** changed based on STAT4 information while computing subsequent loops,
006489 ** then we need to rerun the whole loop building process so that all
006490 ** loops will be built using the revised truthProb values. */
006491 if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
006492 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006493 WHERETRACE(0xffffffff,
006494 ("**** Redo all loop computations due to"
006495 " TERM_HIGHTRUTH changes ****\n"));
006496 while( pWInfo->pLoops ){
006497 WhereLoop *p = pWInfo->pLoops;
006498 pWInfo->pLoops = p->pNextLoop;
006499 whereLoopDelete(db, p);
006500 }
006501 rc = whereLoopAddAll(&sWLB);
006502 if( rc ) goto whereBeginError;
006503 }
006504 #endif
006505 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006506
006507 wherePathSolver(pWInfo, 0);
006508 if( db->mallocFailed ) goto whereBeginError;
006509 if( pWInfo->pOrderBy ){
006510 whereInterstageHeuristic(pWInfo);
006511 wherePathSolver(pWInfo, pWInfo->nRowOut+1);
006512 if( db->mallocFailed ) goto whereBeginError;
006513 }
006514
006515 /* TUNING: Assume that a DISTINCT clause on a subquery reduces
006516 ** the output size by a factor of 8 (LogEst -30).
006517 */
006518 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){
006519 WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
006520 pWInfo->nRowOut, pWInfo->nRowOut-30));
006521 pWInfo->nRowOut -= 30;
006522 }
006523
006524 }
006525 assert( pWInfo->pTabList!=0 );
006526 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
006527 whereReverseScanOrder(pWInfo);
006528 }
006529 if( pParse->nErr ){
006530 goto whereBeginError;
006531 }
006532 assert( db->mallocFailed==0 );
006533 #ifdef WHERETRACE_ENABLED
006534 if( sqlite3WhereTrace ){
006535 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut);
006536 if( pWInfo->nOBSat>0 ){
006537 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
006538 }
006539 switch( pWInfo->eDistinct ){
006540 case WHERE_DISTINCT_UNIQUE: {
006541 sqlite3DebugPrintf(" DISTINCT=unique");
006542 break;
006543 }
006544 case WHERE_DISTINCT_ORDERED: {
006545 sqlite3DebugPrintf(" DISTINCT=ordered");
006546 break;
006547 }
006548 case WHERE_DISTINCT_UNORDERED: {
006549 sqlite3DebugPrintf(" DISTINCT=unordered");
006550 break;
006551 }
006552 }
006553 sqlite3DebugPrintf("\n");
006554 for(ii=0; ii<pWInfo->nLevel; ii++){
006555 sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC);
006556 }
006557 }
006558 #endif
006559
006560 /* Attempt to omit tables from a join that do not affect the result.
006561 ** See the comment on whereOmitNoopJoin() for further information.
006562 **
006563 ** This query optimization is factored out into a separate "no-inline"
006564 ** procedure to keep the sqlite3WhereBegin() procedure from becoming
006565 ** too large. If sqlite3WhereBegin() becomes too large, that prevents
006566 ** some C-compiler optimizers from in-lining the
006567 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
006568 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
006569 */
006570 notReady = ~(Bitmask)0;
006571 if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */
006572 && pResultSet!=0 /* Condition (1) */
006573 && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */
006574 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */
006575 ){
006576 notReady = whereOmitNoopJoin(pWInfo, notReady);
006577 nTabList = pWInfo->nLevel;
006578 assert( nTabList>0 );
006579 }
006580
006581 /* Check to see if there are any SEARCH loops that might benefit from
006582 ** using a Bloom filter.
006583 */
006584 if( pWInfo->nLevel>=2
006585 && OptimizationEnabled(db, SQLITE_BloomFilter)
006586 ){
006587 whereCheckIfBloomFilterIsUseful(pWInfo);
006588 }
006589
006590 #if defined(WHERETRACE_ENABLED)
006591 if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */
006592 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
006593 sqlite3WhereClausePrint(sWLB.pWC);
006594 }
006595 WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n"));
006596 #endif
006597 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut;
006598
006599 /* If the caller is an UPDATE or DELETE statement that is requesting
006600 ** to use a one-pass algorithm, determine if this is appropriate.
006601 **
006602 ** A one-pass approach can be used if the caller has requested one
006603 ** and either (a) the scan visits at most one row or (b) each
006604 ** of the following are true:
006605 **
006606 ** * the caller has indicated that a one-pass approach can be used
006607 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
006608 ** * the table is not a virtual table, and
006609 ** * either the scan does not use the OR optimization or the caller
006610 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified
006611 ** for DELETE).
006612 **
006613 ** The last qualification is because an UPDATE statement uses
006614 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
006615 ** use a one-pass approach, and this is not set accurately for scans
006616 ** that use the OR optimization.
006617 */
006618 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
006619 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
006620 int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
006621 int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
006622 assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) );
006623 if( bOnerow || (
006624 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
006625 && !IsVirtual(pTabList->a[0].pTab)
006626 && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
006627 && OptimizationEnabled(db, SQLITE_OnePass)
006628 )){
006629 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
006630 if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){
006631 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
006632 bFordelete = OPFLAG_FORDELETE;
006633 }
006634 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY);
006635 }
006636 }
006637 }
006638
006639 /* Open all tables in the pTabList and any indices selected for
006640 ** searching those tables.
006641 */
006642 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){
006643 Table *pTab; /* Table to open */
006644 int iDb; /* Index of database containing table/index */
006645 SrcItem *pTabItem;
006646
006647 pTabItem = &pTabList->a[pLevel->iFrom];
006648 pTab = pTabItem->pTab;
006649 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
006650 pLoop = pLevel->pWLoop;
006651 if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){
006652 /* Do nothing */
006653 }else
006654 #ifndef SQLITE_OMIT_VIRTUALTABLE
006655 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
006656 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab);
006657 int iCur = pTabItem->iCursor;
006658 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
006659 }else if( IsVirtual(pTab) ){
006660 /* noop */
006661 }else
006662 #endif
006663 if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0
006664 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0)
006665 || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0
006666 ){
006667 int op = OP_OpenRead;
006668 if( pWInfo->eOnePass!=ONEPASS_OFF ){
006669 op = OP_OpenWrite;
006670 pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
006671 };
006672 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
006673 assert( pTabItem->iCursor==pLevel->iTabCur );
006674 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
006675 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
006676 if( pWInfo->eOnePass==ONEPASS_OFF
006677 && pTab->nCol<BMS
006678 && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0
006679 && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0
006680 ){
006681 /* If we know that only a prefix of the record will be used,
006682 ** it is advantageous to reduce the "column count" field in
006683 ** the P4 operand of the OP_OpenRead/Write opcode. */
006684 Bitmask b = pTabItem->colUsed;
006685 int n = 0;
006686 for(; b; b=b>>1, n++){}
006687 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
006688 assert( n<=pTab->nCol );
006689 }
006690 #ifdef SQLITE_ENABLE_CURSOR_HINTS
006691 if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){
006692 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
006693 }else
006694 #endif
006695 {
006696 sqlite3VdbeChangeP5(v, bFordelete);
006697 }
006698 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
006699 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
006700 (const u8*)&pTabItem->colUsed, P4_INT64);
006701 #endif
006702 }else{
006703 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
006704 }
006705 if( pLoop->wsFlags & WHERE_INDEXED ){
006706 Index *pIx = pLoop->u.btree.pIndex;
006707 int iIndexCur;
006708 int op = OP_OpenRead;
006709 /* iAuxArg is always set to a positive value if ONEPASS is possible */
006710 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
006711 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
006712 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
006713 ){
006714 /* This is one term of an OR-optimization using the PRIMARY KEY of a
006715 ** WITHOUT ROWID table. No need for a separate index */
006716 iIndexCur = pLevel->iTabCur;
006717 op = 0;
006718 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
006719 Index *pJ = pTabItem->pTab->pIndex;
006720 iIndexCur = iAuxArg;
006721 assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
006722 while( ALWAYS(pJ) && pJ!=pIx ){
006723 iIndexCur++;
006724 pJ = pJ->pNext;
006725 }
006726 op = OP_OpenWrite;
006727 pWInfo->aiCurOnePass[1] = iIndexCur;
006728 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
006729 iIndexCur = iAuxArg;
006730 op = OP_ReopenIdx;
006731 }else{
006732 iIndexCur = pParse->nTab++;
006733 if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
006734 whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
006735 }
006736 if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){
006737 wherePartIdxExpr(
006738 pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem
006739 );
006740 }
006741 }
006742 pLevel->iIdxCur = iIndexCur;
006743 assert( pIx!=0 );
006744 assert( pIx->pSchema==pTab->pSchema );
006745 assert( iIndexCur>=0 );
006746 if( op ){
006747 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
006748 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
006749 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
006750 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
006751 && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0
006752 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
006753 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
006754 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
006755 ){
006756 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ);
006757 }
006758 VdbeComment((v, "%s", pIx->zName));
006759 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
006760 {
006761 u64 colUsed = 0;
006762 int ii, jj;
006763 for(ii=0; ii<pIx->nColumn; ii++){
006764 jj = pIx->aiColumn[ii];
006765 if( jj<0 ) continue;
006766 if( jj>63 ) jj = 63;
006767 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue;
006768 colUsed |= ((u64)1)<<(ii<63 ? ii : 63);
006769 }
006770 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0,
006771 (u8*)&colUsed, P4_INT64);
006772 }
006773 #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
006774 }
006775 }
006776 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb);
006777 if( (pTabItem->fg.jointype & JT_RIGHT)!=0
006778 && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0
006779 ){
006780 WhereRightJoin *pRJ = pLevel->pRJ;
006781 pRJ->iMatch = pParse->nTab++;
006782 pRJ->regBloom = ++pParse->nMem;
006783 sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom);
006784 pRJ->regReturn = ++pParse->nMem;
006785 sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn);
006786 assert( pTab==pTabItem->pTab );
006787 if( HasRowid(pTab) ){
006788 KeyInfo *pInfo;
006789 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1);
006790 pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0);
006791 if( pInfo ){
006792 pInfo->aColl[0] = 0;
006793 pInfo->aSortFlags[0] = 0;
006794 sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO);
006795 }
006796 }else{
006797 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
006798 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol);
006799 sqlite3VdbeSetP4KeyInfo(pParse, pPk);
006800 }
006801 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
006802 /* The nature of RIGHT JOIN processing is such that it messes up
006803 ** the output order. So omit any ORDER BY/GROUP BY elimination
006804 ** optimizations. We need to do an actual sort for RIGHT JOIN. */
006805 pWInfo->nOBSat = 0;
006806 pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED;
006807 }
006808 }
006809 pWInfo->iTop = sqlite3VdbeCurrentAddr(v);
006810 if( db->mallocFailed ) goto whereBeginError;
006811
006812 /* Generate the code to do the search. Each iteration of the for
006813 ** loop below generates code for a single nested loop of the VM
006814 ** program.
006815 */
006816 for(ii=0; ii<nTabList; ii++){
006817 int addrExplain;
006818 int wsFlags;
006819 SrcItem *pSrc;
006820 if( pParse->nErr ) goto whereBeginError;
006821 pLevel = &pWInfo->a[ii];
006822 wsFlags = pLevel->pWLoop->wsFlags;
006823 pSrc = &pTabList->a[pLevel->iFrom];
006824 if( pSrc->fg.isMaterialized ){
006825 if( pSrc->fg.isCorrelated ){
006826 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
006827 }else{
006828 int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
006829 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
006830 sqlite3VdbeJumpHere(v, iOnce);
006831 }
006832 }
006833 assert( pTabList == pWInfo->pTabList );
006834 if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
006835 if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
006836 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
006837 constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel);
006838 #endif
006839 }else{
006840 sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady);
006841 }
006842 if( db->mallocFailed ) goto whereBeginError;
006843 }
006844 addrExplain = sqlite3WhereExplainOneScan(
006845 pParse, pTabList, pLevel, wctrlFlags
006846 );
006847 pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
006848 notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady);
006849 pWInfo->iContinue = pLevel->addrCont;
006850 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
006851 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
006852 }
006853 }
006854
006855 /* Done. */
006856 VdbeModuleComment((v, "Begin WHERE-core"));
006857 pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v);
006858 return pWInfo;
006859
006860 /* Jump here if malloc fails */
006861 whereBeginError:
006862 if( pWInfo ){
006863 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
006864 whereInfoFree(db, pWInfo);
006865 }
006866 #ifdef WHERETRACE_ENABLED
006867 /* Prevent harmless compiler warnings about debugging routines
006868 ** being declared but never used */
006869 sqlite3ShowWhereLoopList(0);
006870 #endif /* WHERETRACE_ENABLED */
006871 return 0;
006872 }
006873
006874 /*
006875 ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the
006876 ** index rather than the main table. In SQLITE_DEBUG mode, we want
006877 ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine
006878 ** does that.
006879 */
006880 #ifndef SQLITE_DEBUG
006881 # define OpcodeRewriteTrace(D,K,P) /* no-op */
006882 #else
006883 # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P)
006884 static void sqlite3WhereOpcodeRewriteTrace(
006885 sqlite3 *db,
006886 int pc,
006887 VdbeOp *pOp
006888 ){
006889 if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return;
006890 sqlite3VdbePrintOp(0, pc, pOp);
006891 }
006892 #endif
006893
006894 /*
006895 ** Generate the end of the WHERE loop. See comments on
006896 ** sqlite3WhereBegin() for additional information.
006897 */
006898 void sqlite3WhereEnd(WhereInfo *pWInfo){
006899 Parse *pParse = pWInfo->pParse;
006900 Vdbe *v = pParse->pVdbe;
006901 int i;
006902 WhereLevel *pLevel;
006903 WhereLoop *pLoop;
006904 SrcList *pTabList = pWInfo->pTabList;
006905 sqlite3 *db = pParse->db;
006906 int iEnd = sqlite3VdbeCurrentAddr(v);
006907 int nRJ = 0;
006908
006909 /* Generate loop termination code.
006910 */
006911 VdbeModuleComment((v, "End WHERE-core"));
006912 for(i=pWInfo->nLevel-1; i>=0; i--){
006913 int addr;
006914 pLevel = &pWInfo->a[i];
006915 if( pLevel->pRJ ){
006916 /* Terminate the subroutine that forms the interior of the loop of
006917 ** the RIGHT JOIN table */
006918 WhereRightJoin *pRJ = pLevel->pRJ;
006919 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006920 pLevel->addrCont = 0;
006921 pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v);
006922 sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1);
006923 VdbeCoverage(v);
006924 nRJ++;
006925 }
006926 pLoop = pLevel->pWLoop;
006927 if( pLevel->op!=OP_Noop ){
006928 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
006929 int addrSeek = 0;
006930 Index *pIdx;
006931 int n;
006932 if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
006933 && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */
006934 && (pLoop->wsFlags & WHERE_INDEXED)!=0
006935 && (pIdx = pLoop->u.btree.pIndex)->hasStat1
006936 && (n = pLoop->u.btree.nDistinctCol)>0
006937 && pIdx->aiRowLogEst[n]>=36
006938 ){
006939 int r1 = pParse->nMem+1;
006940 int j, op;
006941 for(j=0; j<n; j++){
006942 sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
006943 }
006944 pParse->nMem += n+1;
006945 op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
006946 addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
006947 VdbeCoverageIf(v, op==OP_SeekLT);
006948 VdbeCoverageIf(v, op==OP_SeekGT);
006949 sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
006950 }
006951 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
006952 /* The common case: Advance to the next row */
006953 if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006954 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
006955 sqlite3VdbeChangeP5(v, pLevel->p5);
006956 VdbeCoverage(v);
006957 VdbeCoverageIf(v, pLevel->op==OP_Next);
006958 VdbeCoverageIf(v, pLevel->op==OP_Prev);
006959 VdbeCoverageIf(v, pLevel->op==OP_VNext);
006960 if( pLevel->regBignull ){
006961 sqlite3VdbeResolveLabel(v, pLevel->addrBignull);
006962 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1);
006963 VdbeCoverage(v);
006964 }
006965 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
006966 if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
006967 #endif
006968 }else if( pLevel->addrCont ){
006969 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006970 }
006971 if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){
006972 struct InLoop *pIn;
006973 int j;
006974 sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
006975 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
006976 assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull
006977 || pParse->db->mallocFailed );
006978 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
006979 if( pIn->eEndLoopOp!=OP_Noop ){
006980 if( pIn->nPrefix ){
006981 int bEarlyOut =
006982 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
006983 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
006984 if( pLevel->iLeftJoin ){
006985 /* For LEFT JOIN queries, cursor pIn->iCur may not have been
006986 ** opened yet. This occurs for WHERE clauses such as
006987 ** "a = ? AND b IN (...)", where the index is on (a, b). If
006988 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may
006989 ** never have been coded, but the body of the loop run to
006990 ** return the null-row. So, if the cursor is not open yet,
006991 ** jump over the OP_Next or OP_Prev instruction about to
006992 ** be coded. */
006993 sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur,
006994 sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
006995 VdbeCoverage(v);
006996 }
006997 if( bEarlyOut ){
006998 sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur,
006999 sqlite3VdbeCurrentAddr(v)+2,
007000 pIn->iBase, pIn->nPrefix);
007001 VdbeCoverage(v);
007002 /* Retarget the OP_IsNull against the left operand of IN so
007003 ** it jumps past the OP_IfNoHope. This is because the
007004 ** OP_IsNull also bypasses the OP_Affinity opcode that is
007005 ** required by OP_IfNoHope. */
007006 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
007007 }
007008 }
007009 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
007010 VdbeCoverage(v);
007011 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev);
007012 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next);
007013 }
007014 sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
007015 }
007016 }
007017 sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
007018 if( pLevel->pRJ ){
007019 sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1);
007020 VdbeCoverage(v);
007021 }
007022 if( pLevel->addrSkip ){
007023 sqlite3VdbeGoto(v, pLevel->addrSkip);
007024 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName));
007025 sqlite3VdbeJumpHere(v, pLevel->addrSkip);
007026 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2);
007027 }
007028 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
007029 if( pLevel->addrLikeRep ){
007030 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
007031 pLevel->addrLikeRep);
007032 VdbeCoverage(v);
007033 }
007034 #endif
007035 if( pLevel->iLeftJoin ){
007036 int ws = pLoop->wsFlags;
007037 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
007038 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
007039 if( (ws & WHERE_IDX_ONLY)==0 ){
007040 SrcItem *pSrc = &pTabList->a[pLevel->iFrom];
007041 assert( pLevel->iTabCur==pSrc->iCursor );
007042 if( pSrc->fg.viaCoroutine ){
007043 int m, n;
007044 n = pSrc->regResult;
007045 assert( pSrc->pTab!=0 );
007046 m = pSrc->pTab->nCol;
007047 sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1);
007048 }
007049 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
007050 }
007051 if( (ws & WHERE_INDEXED)
007052 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx)
007053 ){
007054 if( ws & WHERE_MULTI_OR ){
007055 Index *pIx = pLevel->u.pCoveringIdx;
007056 int iDb = sqlite3SchemaToIndex(db, pIx->pSchema);
007057 sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb);
007058 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
007059 }
007060 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
007061 }
007062 if( pLevel->op==OP_Return ){
007063 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
007064 }else{
007065 sqlite3VdbeGoto(v, pLevel->addrFirst);
007066 }
007067 sqlite3VdbeJumpHere(v, addr);
007068 }
007069 VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
007070 pWInfo->pTabList->a[pLevel->iFrom].pTab->zName));
007071 }
007072
007073 assert( pWInfo->nLevel<=pTabList->nSrc );
007074 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
007075 int k, last;
007076 VdbeOp *pOp, *pLastOp;
007077 Index *pIdx = 0;
007078 SrcItem *pTabItem = &pTabList->a[pLevel->iFrom];
007079 Table *pTab = pTabItem->pTab;
007080 assert( pTab!=0 );
007081 pLoop = pLevel->pWLoop;
007082
007083 /* Do RIGHT JOIN processing. Generate code that will output the
007084 ** unmatched rows of the right operand of the RIGHT JOIN with
007085 ** all of the columns of the left operand set to NULL.
007086 */
007087 if( pLevel->pRJ ){
007088 sqlite3WhereRightJoinLoop(pWInfo, i, pLevel);
007089 continue;
007090 }
007091
007092 /* For a co-routine, change all OP_Column references to the table of
007093 ** the co-routine into OP_Copy of result contained in a register.
007094 ** OP_Rowid becomes OP_Null.
007095 */
007096 if( pTabItem->fg.viaCoroutine ){
007097 testcase( pParse->db->mallocFailed );
007098 assert( pTabItem->regResult>=0 );
007099 translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
007100 pTabItem->regResult, 0);
007101 continue;
007102 }
007103
007104 /* If this scan uses an index, make VDBE code substitutions to read data
007105 ** from the index instead of from the table where possible. In some cases
007106 ** this optimization prevents the table from ever being read, which can
007107 ** yield a significant performance boost.
007108 **
007109 ** Calls to the code generator in between sqlite3WhereBegin and
007110 ** sqlite3WhereEnd will have created code that references the table
007111 ** directly. This loop scans all that code looking for opcodes
007112 ** that reference the table and converts them into opcodes that
007113 ** reference the index.
007114 */
007115 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){
007116 pIdx = pLoop->u.btree.pIndex;
007117 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){
007118 pIdx = pLevel->u.pCoveringIdx;
007119 }
007120 if( pIdx
007121 && !db->mallocFailed
007122 ){
007123 if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
007124 last = iEnd;
007125 }else{
007126 last = pWInfo->iEndWhere;
007127 }
007128 if( pIdx->bHasExpr ){
007129 IndexedExpr *p = pParse->pIdxEpr;
007130 while( p ){
007131 if( p->iIdxCur==pLevel->iIdxCur ){
007132 #ifdef WHERETRACE_ENABLED
007133 if( sqlite3WhereTrace & 0x200 ){
007134 sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n",
007135 p->iIdxCur, p->iIdxCol);
007136 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr);
007137 }
007138 #endif
007139 p->iDataCur = -1;
007140 p->iIdxCur = -1;
007141 }
007142 p = p->pIENext;
007143 }
007144 }
007145 k = pLevel->addrBody + 1;
007146 #ifdef SQLITE_DEBUG
007147 if( db->flags & SQLITE_VdbeAddopTrace ){
007148 printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n",
007149 pLevel->iTabCur, pLevel->iIdxCur, k, last-1);
007150 }
007151 /* Proof that the "+1" on the k value above is safe */
007152 pOp = sqlite3VdbeGetOp(v, k - 1);
007153 assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur );
007154 assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur );
007155 assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur );
007156 #endif
007157 pOp = sqlite3VdbeGetOp(v, k);
007158 pLastOp = pOp + (last - k);
007159 assert( pOp<=pLastOp );
007160 do{
007161 if( pOp->p1!=pLevel->iTabCur ){
007162 /* no-op */
007163 }else if( pOp->opcode==OP_Column
007164 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
007165 || pOp->opcode==OP_Offset
007166 #endif
007167 ){
007168 int x = pOp->p2;
007169 assert( pIdx->pTable==pTab );
007170 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
007171 if( pOp->opcode==OP_Offset ){
007172 /* Do not need to translate the column number */
007173 }else
007174 #endif
007175 if( !HasRowid(pTab) ){
007176 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
007177 x = pPk->aiColumn[x];
007178 assert( x>=0 );
007179 }else{
007180 testcase( x!=sqlite3StorageColumnToTable(pTab,x) );
007181 x = sqlite3StorageColumnToTable(pTab,x);
007182 }
007183 x = sqlite3TableColumnToIndex(pIdx, x);
007184 if( x>=0 ){
007185 pOp->p2 = x;
007186 pOp->p1 = pLevel->iIdxCur;
007187 OpcodeRewriteTrace(db, k, pOp);
007188 }else{
007189 /* Unable to translate the table reference into an index
007190 ** reference. Verify that this is harmless - that the
007191 ** table being referenced really is open.
007192 */
007193 if( pLoop->wsFlags & WHERE_IDX_ONLY ){
007194 sqlite3ErrorMsg(pParse, "internal query planner error");
007195 pParse->rc = SQLITE_INTERNAL;
007196 }
007197 }
007198 }else if( pOp->opcode==OP_Rowid ){
007199 pOp->p1 = pLevel->iIdxCur;
007200 pOp->opcode = OP_IdxRowid;
007201 OpcodeRewriteTrace(db, k, pOp);
007202 }else if( pOp->opcode==OP_IfNullRow ){
007203 pOp->p1 = pLevel->iIdxCur;
007204 OpcodeRewriteTrace(db, k, pOp);
007205 }
007206 #ifdef SQLITE_DEBUG
007207 k++;
007208 #endif
007209 }while( (++pOp)<pLastOp );
007210 #ifdef SQLITE_DEBUG
007211 if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n");
007212 #endif
007213 }
007214 }
007215
007216 /* The "break" point is here, just past the end of the outer loop.
007217 ** Set it.
007218 */
007219 sqlite3VdbeResolveLabel(v, pWInfo->iBreak);
007220
007221 /* Final cleanup
007222 */
007223 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
007224 whereInfoFree(db, pWInfo);
007225 pParse->withinRJSubrtn -= nRJ;
007226 return;
007227 }