000001 /* 000002 ** 2001 September 15 000003 ** 000004 ** The author disclaims copyright to this source code. In place of 000005 ** a legal notice, here is a blessing: 000006 ** 000007 ** May you do good and not evil. 000008 ** May you find forgiveness for yourself and forgive others. 000009 ** May you share freely, never taking more than you give. 000010 ** 000011 ************************************************************************* 000012 ** This module contains C code that generates VDBE code used to process 000013 ** the WHERE clause of SQL statements. This module is responsible for 000014 ** generating the code that loops through a table looking for applicable 000015 ** rows. Indices are selected and used to speed the search when doing 000016 ** so is applicable. Because this module is responsible for selecting 000017 ** indices, you might also think of this module as the "query optimizer". 000018 */ 000019 #include "sqliteInt.h" 000020 #include "whereInt.h" 000021 000022 /* 000023 ** Extra information appended to the end of sqlite3_index_info but not 000024 ** visible to the xBestIndex function, at least not directly. The 000025 ** sqlite3_vtab_collation() interface knows how to reach it, however. 000026 ** 000027 ** This object is not an API and can be changed from one release to the 000028 ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation() 000029 ** agree on the structure, all will be well. 000030 */ 000031 typedef struct HiddenIndexInfo HiddenIndexInfo; 000032 struct HiddenIndexInfo { 000033 WhereClause *pWC; /* The Where clause being analyzed */ 000034 Parse *pParse; /* The parsing context */ 000035 int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ 000036 u32 mIn; /* Mask of terms that are <col> IN (...) */ 000037 u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */ 000038 sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST 000039 ** because extra space is allocated to hold up 000040 ** to nTerm such values */ 000041 }; 000042 000043 /* Forward declaration of methods */ 000044 static int whereLoopResize(sqlite3*, WhereLoop*, int); 000045 000046 /* 000047 ** Return the estimated number of output rows from a WHERE clause 000048 */ 000049 LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ 000050 return pWInfo->nRowOut; 000051 } 000052 000053 /* 000054 ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this 000055 ** WHERE clause returns outputs for DISTINCT processing. 000056 */ 000057 int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ 000058 return pWInfo->eDistinct; 000059 } 000060 000061 /* 000062 ** Return the number of ORDER BY terms that are satisfied by the 000063 ** WHERE clause. A return of 0 means that the output must be 000064 ** completely sorted. A return equal to the number of ORDER BY 000065 ** terms means that no sorting is needed at all. A return that 000066 ** is positive but less than the number of ORDER BY terms means that 000067 ** block sorting is required. 000068 */ 000069 int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ 000070 return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat; 000071 } 000072 000073 /* 000074 ** In the ORDER BY LIMIT optimization, if the inner-most loop is known 000075 ** to emit rows in increasing order, and if the last row emitted by the 000076 ** inner-most loop did not fit within the sorter, then we can skip all 000077 ** subsequent rows for the current iteration of the inner loop (because they 000078 ** will not fit in the sorter either) and continue with the second inner 000079 ** loop - the loop immediately outside the inner-most. 000080 ** 000081 ** When a row does not fit in the sorter (because the sorter already 000082 ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the 000083 ** label returned by this function. 000084 ** 000085 ** If the ORDER BY LIMIT optimization applies, the jump destination should 000086 ** be the continuation for the second-inner-most loop. If the ORDER BY 000087 ** LIMIT optimization does not apply, then the jump destination should 000088 ** be the continuation for the inner-most loop. 000089 ** 000090 ** It is always safe for this routine to return the continuation of the 000091 ** inner-most loop, in the sense that a correct answer will result. 000092 ** Returning the continuation the second inner loop is an optimization 000093 ** that might make the code run a little faster, but should not change 000094 ** the final answer. 000095 */ 000096 int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ 000097 WhereLevel *pInner; 000098 if( !pWInfo->bOrderedInnerLoop ){ 000099 /* The ORDER BY LIMIT optimization does not apply. Jump to the 000100 ** continuation of the inner-most loop. */ 000101 return pWInfo->iContinue; 000102 } 000103 pInner = &pWInfo->a[pWInfo->nLevel-1]; 000104 assert( pInner->addrNxt!=0 ); 000105 return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt; 000106 } 000107 000108 /* 000109 ** While generating code for the min/max optimization, after handling 000110 ** the aggregate-step call to min() or max(), check to see if any 000111 ** additional looping is required. If the output order is such that 000112 ** we are certain that the correct answer has already been found, then 000113 ** code an OP_Goto to by pass subsequent processing. 000114 ** 000115 ** Any extra OP_Goto that is coded here is an optimization. The 000116 ** correct answer should be obtained regardless. This OP_Goto just 000117 ** makes the answer appear faster. 000118 */ 000119 void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){ 000120 WhereLevel *pInner; 000121 int i; 000122 if( !pWInfo->bOrderedInnerLoop ) return; 000123 if( pWInfo->nOBSat==0 ) return; 000124 for(i=pWInfo->nLevel-1; i>=0; i--){ 000125 pInner = &pWInfo->a[i]; 000126 if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){ 000127 sqlite3VdbeGoto(v, pInner->addrNxt); 000128 return; 000129 } 000130 } 000131 sqlite3VdbeGoto(v, pWInfo->iBreak); 000132 } 000133 000134 /* 000135 ** Return the VDBE address or label to jump to in order to continue 000136 ** immediately with the next row of a WHERE clause. 000137 */ 000138 int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ 000139 assert( pWInfo->iContinue!=0 ); 000140 return pWInfo->iContinue; 000141 } 000142 000143 /* 000144 ** Return the VDBE address or label to jump to in order to break 000145 ** out of a WHERE loop. 000146 */ 000147 int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ 000148 return pWInfo->iBreak; 000149 } 000150 000151 /* 000152 ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to 000153 ** operate directly on the rowids returned by a WHERE clause. Return 000154 ** ONEPASS_SINGLE (1) if the statement can operation directly because only 000155 ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass 000156 ** optimization can be used on multiple 000157 ** 000158 ** If the ONEPASS optimization is used (if this routine returns true) 000159 ** then also write the indices of open cursors used by ONEPASS 000160 ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data 000161 ** table and iaCur[1] gets the cursor used by an auxiliary index. 000162 ** Either value may be -1, indicating that cursor is not used. 000163 ** Any cursors returned will have been opened for writing. 000164 ** 000165 ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is 000166 ** unable to use the ONEPASS optimization. 000167 */ 000168 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ 000169 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); 000170 #ifdef WHERETRACE_ENABLED 000171 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ 000172 sqlite3DebugPrintf("%s cursors: %d %d\n", 000173 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", 000174 aiCur[0], aiCur[1]); 000175 } 000176 #endif 000177 return pWInfo->eOnePass; 000178 } 000179 000180 /* 000181 ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move 000182 ** the data cursor to the row selected by the index cursor. 000183 */ 000184 int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){ 000185 return pWInfo->bDeferredSeek; 000186 } 000187 000188 /* 000189 ** Move the content of pSrc into pDest 000190 */ 000191 static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ 000192 pDest->n = pSrc->n; 000193 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); 000194 } 000195 000196 /* 000197 ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. 000198 ** 000199 ** The new entry might overwrite an existing entry, or it might be 000200 ** appended, or it might be discarded. Do whatever is the right thing 000201 ** so that pSet keeps the N_OR_COST best entries seen so far. 000202 */ 000203 static int whereOrInsert( 000204 WhereOrSet *pSet, /* The WhereOrSet to be updated */ 000205 Bitmask prereq, /* Prerequisites of the new entry */ 000206 LogEst rRun, /* Run-cost of the new entry */ 000207 LogEst nOut /* Number of outputs for the new entry */ 000208 ){ 000209 u16 i; 000210 WhereOrCost *p; 000211 for(i=pSet->n, p=pSet->a; i>0; i--, p++){ 000212 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ 000213 goto whereOrInsert_done; 000214 } 000215 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ 000216 return 0; 000217 } 000218 } 000219 if( pSet->n<N_OR_COST ){ 000220 p = &pSet->a[pSet->n++]; 000221 p->nOut = nOut; 000222 }else{ 000223 p = pSet->a; 000224 for(i=1; i<pSet->n; i++){ 000225 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; 000226 } 000227 if( p->rRun<=rRun ) return 0; 000228 } 000229 whereOrInsert_done: 000230 p->prereq = prereq; 000231 p->rRun = rRun; 000232 if( p->nOut>nOut ) p->nOut = nOut; 000233 return 1; 000234 } 000235 000236 /* 000237 ** Return the bitmask for the given cursor number. Return 0 if 000238 ** iCursor is not in the set. 000239 */ 000240 Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ 000241 int i; 000242 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); 000243 assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 ); 000244 assert( iCursor>=-1 ); 000245 if( pMaskSet->ix[0]==iCursor ){ 000246 return 1; 000247 } 000248 for(i=1; i<pMaskSet->n; i++){ 000249 if( pMaskSet->ix[i]==iCursor ){ 000250 return MASKBIT(i); 000251 } 000252 } 000253 return 0; 000254 } 000255 000256 /* Allocate memory that is automatically freed when pWInfo is freed. 000257 */ 000258 void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){ 000259 WhereMemBlock *pBlock; 000260 pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock)); 000261 if( pBlock ){ 000262 pBlock->pNext = pWInfo->pMemToFree; 000263 pBlock->sz = nByte; 000264 pWInfo->pMemToFree = pBlock; 000265 pBlock++; 000266 } 000267 return (void*)pBlock; 000268 } 000269 void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){ 000270 void *pNew = sqlite3WhereMalloc(pWInfo, nByte); 000271 if( pNew && pOld ){ 000272 WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld; 000273 pOldBlk--; 000274 assert( pOldBlk->sz<nByte ); 000275 memcpy(pNew, pOld, pOldBlk->sz); 000276 } 000277 return pNew; 000278 } 000279 000280 /* 000281 ** Create a new mask for cursor iCursor. 000282 ** 000283 ** There is one cursor per table in the FROM clause. The number of 000284 ** tables in the FROM clause is limited by a test early in the 000285 ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] 000286 ** array will never overflow. 000287 */ 000288 static void createMask(WhereMaskSet *pMaskSet, int iCursor){ 000289 assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); 000290 pMaskSet->ix[pMaskSet->n++] = iCursor; 000291 } 000292 000293 /* 000294 ** If the right-hand branch of the expression is a TK_COLUMN, then return 000295 ** a pointer to the right-hand branch. Otherwise, return NULL. 000296 */ 000297 static Expr *whereRightSubexprIsColumn(Expr *p){ 000298 p = sqlite3ExprSkipCollateAndLikely(p->pRight); 000299 if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ 000300 return p; 000301 } 000302 return 0; 000303 } 000304 000305 /* 000306 ** Advance to the next WhereTerm that matches according to the criteria 000307 ** established when the pScan object was initialized by whereScanInit(). 000308 ** Return NULL if there are no more matching WhereTerms. 000309 */ 000310 static WhereTerm *whereScanNext(WhereScan *pScan){ 000311 int iCur; /* The cursor on the LHS of the term */ 000312 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ 000313 Expr *pX; /* An expression being tested */ 000314 WhereClause *pWC; /* Shorthand for pScan->pWC */ 000315 WhereTerm *pTerm; /* The term being tested */ 000316 int k = pScan->k; /* Where to start scanning */ 000317 000318 assert( pScan->iEquiv<=pScan->nEquiv ); 000319 pWC = pScan->pWC; 000320 while(1){ 000321 iColumn = pScan->aiColumn[pScan->iEquiv-1]; 000322 iCur = pScan->aiCur[pScan->iEquiv-1]; 000323 assert( pWC!=0 ); 000324 assert( iCur>=0 ); 000325 do{ 000326 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){ 000327 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 ); 000328 if( pTerm->leftCursor==iCur 000329 && pTerm->u.x.leftColumn==iColumn 000330 && (iColumn!=XN_EXPR 000331 || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft, 000332 pScan->pIdxExpr,iCur)==0) 000333 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON)) 000334 ){ 000335 if( (pTerm->eOperator & WO_EQUIV)!=0 000336 && pScan->nEquiv<ArraySize(pScan->aiCur) 000337 && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0 000338 ){ 000339 int j; 000340 for(j=0; j<pScan->nEquiv; j++){ 000341 if( pScan->aiCur[j]==pX->iTable 000342 && pScan->aiColumn[j]==pX->iColumn ){ 000343 break; 000344 } 000345 } 000346 if( j==pScan->nEquiv ){ 000347 pScan->aiCur[j] = pX->iTable; 000348 pScan->aiColumn[j] = pX->iColumn; 000349 pScan->nEquiv++; 000350 } 000351 } 000352 if( (pTerm->eOperator & pScan->opMask)!=0 ){ 000353 /* Verify the affinity and collating sequence match */ 000354 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ 000355 CollSeq *pColl; 000356 Parse *pParse = pWC->pWInfo->pParse; 000357 pX = pTerm->pExpr; 000358 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ 000359 continue; 000360 } 000361 assert(pX->pLeft); 000362 pColl = sqlite3ExprCompareCollSeq(pParse, pX); 000363 if( pColl==0 ) pColl = pParse->db->pDfltColl; 000364 if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ 000365 continue; 000366 } 000367 } 000368 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 000369 && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0)) 000370 && pX->op==TK_COLUMN 000371 && pX->iTable==pScan->aiCur[0] 000372 && pX->iColumn==pScan->aiColumn[0] 000373 ){ 000374 testcase( pTerm->eOperator & WO_IS ); 000375 continue; 000376 } 000377 pScan->pWC = pWC; 000378 pScan->k = k+1; 000379 #ifdef WHERETRACE_ENABLED 000380 if( sqlite3WhereTrace & 0x20000 ){ 000381 int ii; 000382 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d", 000383 pTerm, pScan->nEquiv); 000384 for(ii=0; ii<pScan->nEquiv; ii++){ 000385 sqlite3DebugPrintf(" {%d:%d}", 000386 pScan->aiCur[ii], pScan->aiColumn[ii]); 000387 } 000388 sqlite3DebugPrintf("\n"); 000389 } 000390 #endif 000391 return pTerm; 000392 } 000393 } 000394 } 000395 pWC = pWC->pOuter; 000396 k = 0; 000397 }while( pWC!=0 ); 000398 if( pScan->iEquiv>=pScan->nEquiv ) break; 000399 pWC = pScan->pOrigWC; 000400 k = 0; 000401 pScan->iEquiv++; 000402 } 000403 return 0; 000404 } 000405 000406 /* 000407 ** This is whereScanInit() for the case of an index on an expression. 000408 ** It is factored out into a separate tail-recursion subroutine so that 000409 ** the normal whereScanInit() routine, which is a high-runner, does not 000410 ** need to push registers onto the stack as part of its prologue. 000411 */ 000412 static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){ 000413 pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr); 000414 return whereScanNext(pScan); 000415 } 000416 000417 /* 000418 ** Initialize a WHERE clause scanner object. Return a pointer to the 000419 ** first match. Return NULL if there are no matches. 000420 ** 000421 ** The scanner will be searching the WHERE clause pWC. It will look 000422 ** for terms of the form "X <op> <expr>" where X is column iColumn of table 000423 ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx 000424 ** must be one of the indexes of table iCur. 000425 ** 000426 ** The <op> must be one of the operators described by opMask. 000427 ** 000428 ** If the search is for X and the WHERE clause contains terms of the 000429 ** form X=Y then this routine might also return terms of the form 000430 ** "Y <op> <expr>". The number of levels of transitivity is limited, 000431 ** but is enough to handle most commonly occurring SQL statements. 000432 ** 000433 ** If X is not the INTEGER PRIMARY KEY then X must be compatible with 000434 ** index pIdx. 000435 */ 000436 static WhereTerm *whereScanInit( 000437 WhereScan *pScan, /* The WhereScan object being initialized */ 000438 WhereClause *pWC, /* The WHERE clause to be scanned */ 000439 int iCur, /* Cursor to scan for */ 000440 int iColumn, /* Column to scan for */ 000441 u32 opMask, /* Operator(s) to scan for */ 000442 Index *pIdx /* Must be compatible with this index */ 000443 ){ 000444 pScan->pOrigWC = pWC; 000445 pScan->pWC = pWC; 000446 pScan->pIdxExpr = 0; 000447 pScan->idxaff = 0; 000448 pScan->zCollName = 0; 000449 pScan->opMask = opMask; 000450 pScan->k = 0; 000451 pScan->aiCur[0] = iCur; 000452 pScan->nEquiv = 1; 000453 pScan->iEquiv = 1; 000454 if( pIdx ){ 000455 int j = iColumn; 000456 iColumn = pIdx->aiColumn[j]; 000457 if( iColumn==pIdx->pTable->iPKey ){ 000458 iColumn = XN_ROWID; 000459 }else if( iColumn>=0 ){ 000460 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; 000461 pScan->zCollName = pIdx->azColl[j]; 000462 }else if( iColumn==XN_EXPR ){ 000463 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; 000464 pScan->zCollName = pIdx->azColl[j]; 000465 pScan->aiColumn[0] = XN_EXPR; 000466 return whereScanInitIndexExpr(pScan); 000467 } 000468 }else if( iColumn==XN_EXPR ){ 000469 return 0; 000470 } 000471 pScan->aiColumn[0] = iColumn; 000472 return whereScanNext(pScan); 000473 } 000474 000475 /* 000476 ** Search for a term in the WHERE clause that is of the form "X <op> <expr>" 000477 ** where X is a reference to the iColumn of table iCur or of index pIdx 000478 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by 000479 ** the op parameter. Return a pointer to the term. Return 0 if not found. 000480 ** 000481 ** If pIdx!=0 then it must be one of the indexes of table iCur. 000482 ** Search for terms matching the iColumn-th column of pIdx 000483 ** rather than the iColumn-th column of table iCur. 000484 ** 000485 ** The term returned might by Y=<expr> if there is another constraint in 000486 ** the WHERE clause that specifies that X=Y. Any such constraints will be 000487 ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The 000488 ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 000489 ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 000490 ** other equivalent values. Hence a search for X will return <expr> if X=A1 000491 ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>. 000492 ** 000493 ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>" 000494 ** then try for the one with no dependencies on <expr> - in other words where 000495 ** <expr> is a constant expression of some kind. Only return entries of 000496 ** the form "X <op> Y" where Y is a column in another table if no terms of 000497 ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS 000498 ** exist, try to return a term that does not use WO_EQUIV. 000499 */ 000500 WhereTerm *sqlite3WhereFindTerm( 000501 WhereClause *pWC, /* The WHERE clause to be searched */ 000502 int iCur, /* Cursor number of LHS */ 000503 int iColumn, /* Column number of LHS */ 000504 Bitmask notReady, /* RHS must not overlap with this mask */ 000505 u32 op, /* Mask of WO_xx values describing operator */ 000506 Index *pIdx /* Must be compatible with this index, if not NULL */ 000507 ){ 000508 WhereTerm *pResult = 0; 000509 WhereTerm *p; 000510 WhereScan scan; 000511 000512 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); 000513 op &= WO_EQ|WO_IS; 000514 while( p ){ 000515 if( (p->prereqRight & notReady)==0 ){ 000516 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){ 000517 testcase( p->eOperator & WO_IS ); 000518 return p; 000519 } 000520 if( pResult==0 ) pResult = p; 000521 } 000522 p = whereScanNext(&scan); 000523 } 000524 return pResult; 000525 } 000526 000527 /* 000528 ** This function searches pList for an entry that matches the iCol-th column 000529 ** of index pIdx. 000530 ** 000531 ** If such an expression is found, its index in pList->a[] is returned. If 000532 ** no expression is found, -1 is returned. 000533 */ 000534 static int findIndexCol( 000535 Parse *pParse, /* Parse context */ 000536 ExprList *pList, /* Expression list to search */ 000537 int iBase, /* Cursor for table associated with pIdx */ 000538 Index *pIdx, /* Index to match column of */ 000539 int iCol /* Column of index to match */ 000540 ){ 000541 int i; 000542 const char *zColl = pIdx->azColl[iCol]; 000543 000544 for(i=0; i<pList->nExpr; i++){ 000545 Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr); 000546 if( ALWAYS(p!=0) 000547 && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) 000548 && p->iColumn==pIdx->aiColumn[iCol] 000549 && p->iTable==iBase 000550 ){ 000551 CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr); 000552 if( 0==sqlite3StrICmp(pColl->zName, zColl) ){ 000553 return i; 000554 } 000555 } 000556 } 000557 000558 return -1; 000559 } 000560 000561 /* 000562 ** Return TRUE if the iCol-th column of index pIdx is NOT NULL 000563 */ 000564 static int indexColumnNotNull(Index *pIdx, int iCol){ 000565 int j; 000566 assert( pIdx!=0 ); 000567 assert( iCol>=0 && iCol<pIdx->nColumn ); 000568 j = pIdx->aiColumn[iCol]; 000569 if( j>=0 ){ 000570 return pIdx->pTable->aCol[j].notNull; 000571 }else if( j==(-1) ){ 000572 return 1; 000573 }else{ 000574 assert( j==(-2) ); 000575 return 0; /* Assume an indexed expression can always yield a NULL */ 000576 000577 } 000578 } 000579 000580 /* 000581 ** Return true if the DISTINCT expression-list passed as the third argument 000582 ** is redundant. 000583 ** 000584 ** A DISTINCT list is redundant if any subset of the columns in the 000585 ** DISTINCT list are collectively unique and individually non-null. 000586 */ 000587 static int isDistinctRedundant( 000588 Parse *pParse, /* Parsing context */ 000589 SrcList *pTabList, /* The FROM clause */ 000590 WhereClause *pWC, /* The WHERE clause */ 000591 ExprList *pDistinct /* The result set that needs to be DISTINCT */ 000592 ){ 000593 Table *pTab; 000594 Index *pIdx; 000595 int i; 000596 int iBase; 000597 000598 /* If there is more than one table or sub-select in the FROM clause of 000599 ** this query, then it will not be possible to show that the DISTINCT 000600 ** clause is redundant. */ 000601 if( pTabList->nSrc!=1 ) return 0; 000602 iBase = pTabList->a[0].iCursor; 000603 pTab = pTabList->a[0].pTab; 000604 000605 /* If any of the expressions is an IPK column on table iBase, then return 000606 ** true. Note: The (p->iTable==iBase) part of this test may be false if the 000607 ** current SELECT is a correlated sub-query. 000608 */ 000609 for(i=0; i<pDistinct->nExpr; i++){ 000610 Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr); 000611 if( NEVER(p==0) ) continue; 000612 if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue; 000613 if( p->iTable==iBase && p->iColumn<0 ) return 1; 000614 } 000615 000616 /* Loop through all indices on the table, checking each to see if it makes 000617 ** the DISTINCT qualifier redundant. It does so if: 000618 ** 000619 ** 1. The index is itself UNIQUE, and 000620 ** 000621 ** 2. All of the columns in the index are either part of the pDistinct 000622 ** list, or else the WHERE clause contains a term of the form "col=X", 000623 ** where X is a constant value. The collation sequences of the 000624 ** comparison and select-list expressions must match those of the index. 000625 ** 000626 ** 3. All of those index columns for which the WHERE clause does not 000627 ** contain a "col=X" term are subject to a NOT NULL constraint. 000628 */ 000629 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ 000630 if( !IsUniqueIndex(pIdx) ) continue; 000631 if( pIdx->pPartIdxWhere ) continue; 000632 for(i=0; i<pIdx->nKeyCol; i++){ 000633 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ 000634 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; 000635 if( indexColumnNotNull(pIdx, i)==0 ) break; 000636 } 000637 } 000638 if( i==pIdx->nKeyCol ){ 000639 /* This index implies that the DISTINCT qualifier is redundant. */ 000640 return 1; 000641 } 000642 } 000643 000644 return 0; 000645 } 000646 000647 000648 /* 000649 ** Estimate the logarithm of the input value to base 2. 000650 */ 000651 static LogEst estLog(LogEst N){ 000652 return N<=10 ? 0 : sqlite3LogEst(N) - 33; 000653 } 000654 000655 /* 000656 ** Convert OP_Column opcodes to OP_Copy in previously generated code. 000657 ** 000658 ** This routine runs over generated VDBE code and translates OP_Column 000659 ** opcodes into OP_Copy when the table is being accessed via co-routine 000660 ** instead of via table lookup. 000661 ** 000662 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on 000663 ** cursor iTabCur are transformed into OP_Sequence opcode for the 000664 ** iAutoidxCur cursor, in order to generate unique rowids for the 000665 ** automatic index being generated. 000666 */ 000667 static void translateColumnToCopy( 000668 Parse *pParse, /* Parsing context */ 000669 int iStart, /* Translate from this opcode to the end */ 000670 int iTabCur, /* OP_Column/OP_Rowid references to this table */ 000671 int iRegister, /* The first column is in this register */ 000672 int iAutoidxCur /* If non-zero, cursor of autoindex being generated */ 000673 ){ 000674 Vdbe *v = pParse->pVdbe; 000675 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); 000676 int iEnd = sqlite3VdbeCurrentAddr(v); 000677 if( pParse->db->mallocFailed ) return; 000678 for(; iStart<iEnd; iStart++, pOp++){ 000679 if( pOp->p1!=iTabCur ) continue; 000680 if( pOp->opcode==OP_Column ){ 000681 #ifdef SQLITE_DEBUG 000682 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ 000683 printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart); 000684 } 000685 #endif 000686 pOp->opcode = OP_Copy; 000687 pOp->p1 = pOp->p2 + iRegister; 000688 pOp->p2 = pOp->p3; 000689 pOp->p3 = 0; 000690 pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */ 000691 }else if( pOp->opcode==OP_Rowid ){ 000692 #ifdef SQLITE_DEBUG 000693 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ 000694 printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart); 000695 } 000696 #endif 000697 pOp->opcode = OP_Sequence; 000698 pOp->p1 = iAutoidxCur; 000699 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW 000700 if( iAutoidxCur==0 ){ 000701 pOp->opcode = OP_Null; 000702 pOp->p3 = 0; 000703 } 000704 #endif 000705 } 000706 } 000707 } 000708 000709 /* 000710 ** Two routines for printing the content of an sqlite3_index_info 000711 ** structure. Used for testing and debugging only. If neither 000712 ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines 000713 ** are no-ops. 000714 */ 000715 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) 000716 static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ 000717 int i; 000718 if( (sqlite3WhereTrace & 0x10)==0 ) return; 000719 for(i=0; i<p->nConstraint; i++){ 000720 sqlite3DebugPrintf( 000721 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", 000722 i, 000723 p->aConstraint[i].iColumn, 000724 p->aConstraint[i].iTermOffset, 000725 p->aConstraint[i].op, 000726 p->aConstraint[i].usable, 000727 sqlite3_vtab_collation(p,i)); 000728 } 000729 for(i=0; i<p->nOrderBy; i++){ 000730 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", 000731 i, 000732 p->aOrderBy[i].iColumn, 000733 p->aOrderBy[i].desc); 000734 } 000735 } 000736 static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ 000737 int i; 000738 if( (sqlite3WhereTrace & 0x10)==0 ) return; 000739 for(i=0; i<p->nConstraint; i++){ 000740 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", 000741 i, 000742 p->aConstraintUsage[i].argvIndex, 000743 p->aConstraintUsage[i].omit); 000744 } 000745 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); 000746 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); 000747 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); 000748 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); 000749 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); 000750 } 000751 #else 000752 #define whereTraceIndexInfoInputs(A) 000753 #define whereTraceIndexInfoOutputs(A) 000754 #endif 000755 000756 /* 000757 ** We know that pSrc is an operand of an outer join. Return true if 000758 ** pTerm is a constraint that is compatible with that join. 000759 ** 000760 ** pTerm must be EP_OuterON if pSrc is the right operand of an 000761 ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc 000762 ** is the left operand of a RIGHT join. 000763 ** 000764 ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f 000765 ** for an example of a WHERE clause constraints that may not be used on 000766 ** the right table of a RIGHT JOIN because the constraint implies a 000767 ** not-NULL condition on the left table of the RIGHT JOIN. 000768 */ 000769 static int constraintCompatibleWithOuterJoin( 000770 const WhereTerm *pTerm, /* WHERE clause term to check */ 000771 const SrcItem *pSrc /* Table we are trying to access */ 000772 ){ 000773 assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */ 000774 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); 000775 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); 000776 testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) 000777 testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); 000778 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) 000779 || pTerm->pExpr->w.iJoin != pSrc->iCursor 000780 ){ 000781 return 0; 000782 } 000783 if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 000784 && ExprHasProperty(pTerm->pExpr, EP_InnerON) 000785 ){ 000786 return 0; 000787 } 000788 return 1; 000789 } 000790 000791 000792 000793 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX 000794 /* 000795 ** Return TRUE if the WHERE clause term pTerm is of a form where it 000796 ** could be used with an index to access pSrc, assuming an appropriate 000797 ** index existed. 000798 */ 000799 static int termCanDriveIndex( 000800 const WhereTerm *pTerm, /* WHERE clause term to check */ 000801 const SrcItem *pSrc, /* Table we are trying to access */ 000802 const Bitmask notReady /* Tables in outer loops of the join */ 000803 ){ 000804 char aff; 000805 if( pTerm->leftCursor!=pSrc->iCursor ) return 0; 000806 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; 000807 assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); 000808 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 000809 && !constraintCompatibleWithOuterJoin(pTerm,pSrc) 000810 ){ 000811 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */ 000812 } 000813 if( (pTerm->prereqRight & notReady)!=0 ) return 0; 000814 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); 000815 if( pTerm->u.x.leftColumn<0 ) return 0; 000816 aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity; 000817 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; 000818 testcase( pTerm->pExpr->op==TK_IS ); 000819 return 1; 000820 } 000821 #endif 000822 000823 000824 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX 000825 000826 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS 000827 /* 000828 ** Argument pIdx represents an automatic index that the current statement 000829 ** will create and populate. Add an OP_Explain with text of the form: 000830 ** 000831 ** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>] 000832 ** 000833 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to 000834 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP 000835 ** values with. In order to avoid breaking legacy code and test cases, 000836 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command. 000837 */ 000838 static void explainAutomaticIndex( 000839 Parse *pParse, 000840 Index *pIdx, /* Automatic index to explain */ 000841 int bPartial, /* True if pIdx is a partial index */ 000842 int *pAddrExplain /* OUT: Address of OP_Explain */ 000843 ){ 000844 if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){ 000845 Table *pTab = pIdx->pTable; 000846 const char *zSep = ""; 000847 char *zText = 0; 000848 int ii = 0; 000849 sqlite3_str *pStr = sqlite3_str_new(pParse->db); 000850 sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName); 000851 assert( pIdx->nColumn>1 ); 000852 assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); 000853 for(ii=0; ii<(pIdx->nColumn-1); ii++){ 000854 const char *zName = 0; 000855 int iCol = pIdx->aiColumn[ii]; 000856 000857 zName = pTab->aCol[iCol].zCnName; 000858 sqlite3_str_appendf(pStr, "%s%s", zSep, zName); 000859 zSep = ", "; 000860 } 000861 zText = sqlite3_str_finish(pStr); 000862 if( zText==0 ){ 000863 sqlite3OomFault(pParse->db); 000864 }else{ 000865 *pAddrExplain = sqlite3VdbeExplain( 000866 pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "") 000867 ); 000868 sqlite3_free(zText); 000869 } 000870 } 000871 } 000872 #else 000873 # define explainAutomaticIndex(a,b,c,d) 000874 #endif 000875 000876 /* 000877 ** Generate code to construct the Index object for an automatic index 000878 ** and to set up the WhereLevel object pLevel so that the code generator 000879 ** makes use of the automatic index. 000880 */ 000881 static SQLITE_NOINLINE void constructAutomaticIndex( 000882 Parse *pParse, /* The parsing context */ 000883 WhereClause *pWC, /* The WHERE clause */ 000884 const Bitmask notReady, /* Mask of cursors that are not available */ 000885 WhereLevel *pLevel /* Write new index here */ 000886 ){ 000887 int nKeyCol; /* Number of columns in the constructed index */ 000888 WhereTerm *pTerm; /* A single term of the WHERE clause */ 000889 WhereTerm *pWCEnd; /* End of pWC->a[] */ 000890 Index *pIdx; /* Object describing the transient index */ 000891 Vdbe *v; /* Prepared statement under construction */ 000892 int addrInit; /* Address of the initialization bypass jump */ 000893 Table *pTable; /* The table being indexed */ 000894 int addrTop; /* Top of the index fill loop */ 000895 int regRecord; /* Register holding an index record */ 000896 int n; /* Column counter */ 000897 int i; /* Loop counter */ 000898 int mxBitCol; /* Maximum column in pSrc->colUsed */ 000899 CollSeq *pColl; /* Collating sequence to on a column */ 000900 WhereLoop *pLoop; /* The Loop object */ 000901 char *zNotUsed; /* Extra space on the end of pIdx */ 000902 Bitmask idxCols; /* Bitmap of columns used for indexing */ 000903 Bitmask extraCols; /* Bitmap of additional columns */ 000904 u8 sentWarning = 0; /* True if a warning has been issued */ 000905 u8 useBloomFilter = 0; /* True to also add a Bloom filter */ 000906 Expr *pPartial = 0; /* Partial Index Expression */ 000907 int iContinue = 0; /* Jump here to skip excluded rows */ 000908 SrcList *pTabList; /* The complete FROM clause */ 000909 SrcItem *pSrc; /* The FROM clause term to get the next index */ 000910 int addrCounter = 0; /* Address where integer counter is initialized */ 000911 int regBase; /* Array of registers where record is assembled */ 000912 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS 000913 int addrExp = 0; /* Address of OP_Explain */ 000914 #endif 000915 000916 /* Generate code to skip over the creation and initialization of the 000917 ** transient index on 2nd and subsequent iterations of the loop. */ 000918 v = pParse->pVdbe; 000919 assert( v!=0 ); 000920 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); 000921 000922 /* Count the number of columns that will be added to the index 000923 ** and used to match WHERE clause constraints */ 000924 nKeyCol = 0; 000925 pTabList = pWC->pWInfo->pTabList; 000926 pSrc = &pTabList->a[pLevel->iFrom]; 000927 pTable = pSrc->pTab; 000928 pWCEnd = &pWC->a[pWC->nTerm]; 000929 pLoop = pLevel->pWLoop; 000930 idxCols = 0; 000931 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){ 000932 Expr *pExpr = pTerm->pExpr; 000933 /* Make the automatic index a partial index if there are terms in the 000934 ** WHERE clause (or the ON clause of a LEFT join) that constrain which 000935 ** rows of the target table (pSrc) that can be used. */ 000936 if( (pTerm->wtFlags & TERM_VIRTUAL)==0 000937 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) 000938 ){ 000939 pPartial = sqlite3ExprAnd(pParse, pPartial, 000940 sqlite3ExprDup(pParse->db, pExpr, 0)); 000941 } 000942 if( termCanDriveIndex(pTerm, pSrc, notReady) ){ 000943 int iCol; 000944 Bitmask cMask; 000945 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); 000946 iCol = pTerm->u.x.leftColumn; 000947 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); 000948 testcase( iCol==BMS ); 000949 testcase( iCol==BMS-1 ); 000950 if( !sentWarning ){ 000951 sqlite3_log(SQLITE_WARNING_AUTOINDEX, 000952 "automatic index on %s(%s)", pTable->zName, 000953 pTable->aCol[iCol].zCnName); 000954 sentWarning = 1; 000955 } 000956 if( (idxCols & cMask)==0 ){ 000957 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ 000958 goto end_auto_index_create; 000959 } 000960 pLoop->aLTerm[nKeyCol++] = pTerm; 000961 idxCols |= cMask; 000962 } 000963 } 000964 } 000965 assert( nKeyCol>0 || pParse->db->mallocFailed ); 000966 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; 000967 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED 000968 | WHERE_AUTO_INDEX; 000969 000970 /* Count the number of additional columns needed to create a 000971 ** covering index. A "covering index" is an index that contains all 000972 ** columns that are needed by the query. With a covering index, the 000973 ** original table never needs to be accessed. Automatic indices must 000974 ** be a covering index because the index will not be updated if the 000975 ** original table changes and the index and table cannot both be used 000976 ** if they go out of sync. 000977 */ 000978 if( IsView(pTable) ){ 000979 extraCols = ALLBITS; 000980 }else{ 000981 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); 000982 } 000983 mxBitCol = MIN(BMS-1,pTable->nCol); 000984 testcase( pTable->nCol==BMS-1 ); 000985 testcase( pTable->nCol==BMS-2 ); 000986 for(i=0; i<mxBitCol; i++){ 000987 if( extraCols & MASKBIT(i) ) nKeyCol++; 000988 } 000989 if( pSrc->colUsed & MASKBIT(BMS-1) ){ 000990 nKeyCol += pTable->nCol - BMS + 1; 000991 } 000992 000993 /* Construct the Index object to describe this index */ 000994 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); 000995 if( pIdx==0 ) goto end_auto_index_create; 000996 pLoop->u.btree.pIndex = pIdx; 000997 pIdx->zName = "auto-index"; 000998 pIdx->pTable = pTable; 000999 n = 0; 001000 idxCols = 0; 001001 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){ 001002 if( termCanDriveIndex(pTerm, pSrc, notReady) ){ 001003 int iCol; 001004 Bitmask cMask; 001005 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); 001006 iCol = pTerm->u.x.leftColumn; 001007 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); 001008 testcase( iCol==BMS-1 ); 001009 testcase( iCol==BMS ); 001010 if( (idxCols & cMask)==0 ){ 001011 Expr *pX = pTerm->pExpr; 001012 idxCols |= cMask; 001013 pIdx->aiColumn[n] = pTerm->u.x.leftColumn; 001014 pColl = sqlite3ExprCompareCollSeq(pParse, pX); 001015 assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ 001016 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; 001017 n++; 001018 if( ALWAYS(pX->pLeft!=0) 001019 && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT 001020 ){ 001021 /* TUNING: only use a Bloom filter on an automatic index 001022 ** if one or more key columns has the ability to hold numeric 001023 ** values, since strings all have the same hash in the Bloom 001024 ** filter implementation and hence a Bloom filter on a text column 001025 ** is not usually helpful. */ 001026 useBloomFilter = 1; 001027 } 001028 } 001029 } 001030 } 001031 assert( (u32)n==pLoop->u.btree.nEq ); 001032 001033 /* Add additional columns needed to make the automatic index into 001034 ** a covering index */ 001035 for(i=0; i<mxBitCol; i++){ 001036 if( extraCols & MASKBIT(i) ){ 001037 pIdx->aiColumn[n] = i; 001038 pIdx->azColl[n] = sqlite3StrBINARY; 001039 n++; 001040 } 001041 } 001042 if( pSrc->colUsed & MASKBIT(BMS-1) ){ 001043 for(i=BMS-1; i<pTable->nCol; i++){ 001044 pIdx->aiColumn[n] = i; 001045 pIdx->azColl[n] = sqlite3StrBINARY; 001046 n++; 001047 } 001048 } 001049 assert( n==nKeyCol ); 001050 pIdx->aiColumn[n] = XN_ROWID; 001051 pIdx->azColl[n] = sqlite3StrBINARY; 001052 001053 /* Create the automatic index */ 001054 explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); 001055 assert( pLevel->iIdxCur>=0 ); 001056 pLevel->iIdxCur = pParse->nTab++; 001057 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); 001058 sqlite3VdbeSetP4KeyInfo(pParse, pIdx); 001059 VdbeComment((v, "for %s", pTable->zName)); 001060 if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){ 001061 sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); 001062 pLevel->regFilter = ++pParse->nMem; 001063 sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); 001064 } 001065 001066 /* Fill the automatic index with content */ 001067 assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); 001068 if( pSrc->fg.viaCoroutine ){ 001069 int regYield = pSrc->regReturn; 001070 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); 001071 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub); 001072 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); 001073 VdbeCoverage(v); 001074 VdbeComment((v, "next row of %s", pSrc->pTab->zName)); 001075 }else{ 001076 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); 001077 } 001078 if( pPartial ){ 001079 iContinue = sqlite3VdbeMakeLabel(pParse); 001080 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); 001081 pLoop->wsFlags |= WHERE_PARTIALIDX; 001082 } 001083 regRecord = sqlite3GetTempReg(pParse); 001084 regBase = sqlite3GenerateIndexKey( 001085 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 001086 ); 001087 if( pLevel->regFilter ){ 001088 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, 001089 regBase, pLoop->u.btree.nEq); 001090 } 001091 sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v)); 001092 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); 001093 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); 001094 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); 001095 if( pSrc->fg.viaCoroutine ){ 001096 sqlite3VdbeChangeP2(v, addrCounter, regBase+n); 001097 testcase( pParse->db->mallocFailed ); 001098 assert( pLevel->iIdxCur>0 ); 001099 translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, 001100 pSrc->regResult, pLevel->iIdxCur); 001101 sqlite3VdbeGoto(v, addrTop); 001102 pSrc->fg.viaCoroutine = 0; 001103 }else{ 001104 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); 001105 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); 001106 } 001107 sqlite3VdbeJumpHere(v, addrTop); 001108 sqlite3ReleaseTempReg(pParse, regRecord); 001109 001110 /* Jump here when skipping the initialization */ 001111 sqlite3VdbeJumpHere(v, addrInit); 001112 sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1); 001113 001114 end_auto_index_create: 001115 sqlite3ExprDelete(pParse->db, pPartial); 001116 } 001117 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ 001118 001119 /* 001120 ** Generate bytecode that will initialize a Bloom filter that is appropriate 001121 ** for pLevel. 001122 ** 001123 ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER 001124 ** flag set, initialize a Bloomfilter for them as well. Except don't do 001125 ** this recursive initialization if the SQLITE_BloomPulldown optimization has 001126 ** been turned off. 001127 ** 001128 ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared 001129 ** from the loop, but the regFilter value is set to a register that implements 001130 ** the Bloom filter. When regFilter is positive, the 001131 ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter 001132 ** and skip the subsequence B-Tree seek if the Bloom filter indicates that 001133 ** no matching rows exist. 001134 ** 001135 ** This routine may only be called if it has previously been determined that 001136 ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit 001137 ** is set. 001138 */ 001139 static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( 001140 WhereInfo *pWInfo, /* The WHERE clause */ 001141 int iLevel, /* Index in pWInfo->a[] that is pLevel */ 001142 WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */ 001143 Bitmask notReady /* Loops that are not ready */ 001144 ){ 001145 int addrOnce; /* Address of opening OP_Once */ 001146 int addrTop; /* Address of OP_Rewind */ 001147 int addrCont; /* Jump here to skip a row */ 001148 const WhereTerm *pTerm; /* For looping over WHERE clause terms */ 001149 const WhereTerm *pWCEnd; /* Last WHERE clause term */ 001150 Parse *pParse = pWInfo->pParse; /* Parsing context */ 001151 Vdbe *v = pParse->pVdbe; /* VDBE under construction */ 001152 WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ 001153 int iCur; /* Cursor for table getting the filter */ 001154 IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ 001155 IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */ 001156 001157 saved_pIdxEpr = pParse->pIdxEpr; 001158 saved_pIdxPartExpr = pParse->pIdxPartExpr; 001159 pParse->pIdxEpr = 0; 001160 pParse->pIdxPartExpr = 0; 001161 001162 assert( pLoop!=0 ); 001163 assert( v!=0 ); 001164 assert( pLoop->wsFlags & WHERE_BLOOMFILTER ); 001165 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ); 001166 001167 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); 001168 do{ 001169 const SrcList *pTabList; 001170 const SrcItem *pItem; 001171 const Table *pTab; 001172 u64 sz; 001173 int iSrc; 001174 sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel); 001175 addrCont = sqlite3VdbeMakeLabel(pParse); 001176 iCur = pLevel->iTabCur; 001177 pLevel->regFilter = ++pParse->nMem; 001178 001179 /* The Bloom filter is a Blob held in a register. Initialize it 001180 ** to zero-filled blob of at least 80K bits, but maybe more if the 001181 ** estimated size of the table is larger. We could actually 001182 ** measure the size of the table at run-time using OP_Count with 001183 ** P3==1 and use that value to initialize the blob. But that makes 001184 ** testing complicated. By basing the blob size on the value in the 001185 ** sqlite_stat1 table, testing is much easier. 001186 */ 001187 pTabList = pWInfo->pTabList; 001188 iSrc = pLevel->iFrom; 001189 pItem = &pTabList->a[iSrc]; 001190 assert( pItem!=0 ); 001191 pTab = pItem->pTab; 001192 assert( pTab!=0 ); 001193 sz = sqlite3LogEstToInt(pTab->nRowLogEst); 001194 if( sz<10000 ){ 001195 sz = 10000; 001196 }else if( sz>10000000 ){ 001197 sz = 10000000; 001198 } 001199 sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter); 001200 001201 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); 001202 pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm]; 001203 for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){ 001204 Expr *pExpr = pTerm->pExpr; 001205 if( (pTerm->wtFlags & TERM_VIRTUAL)==0 001206 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) 001207 ){ 001208 sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); 001209 } 001210 } 001211 if( pLoop->wsFlags & WHERE_IPK ){ 001212 int r1 = sqlite3GetTempReg(pParse); 001213 sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1); 001214 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1); 001215 sqlite3ReleaseTempReg(pParse, r1); 001216 }else{ 001217 Index *pIdx = pLoop->u.btree.pIndex; 001218 int n = pLoop->u.btree.nEq; 001219 int r1 = sqlite3GetTempRange(pParse, n); 001220 int jj; 001221 for(jj=0; jj<n; jj++){ 001222 assert( pIdx->pTable==pItem->pTab ); 001223 sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); 001224 } 001225 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); 001226 sqlite3ReleaseTempRange(pParse, r1, n); 001227 } 001228 sqlite3VdbeResolveLabel(v, addrCont); 001229 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); 001230 VdbeCoverage(v); 001231 sqlite3VdbeJumpHere(v, addrTop); 001232 pLoop->wsFlags &= ~WHERE_BLOOMFILTER; 001233 if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break; 001234 while( ++iLevel < pWInfo->nLevel ){ 001235 const SrcItem *pTabItem; 001236 pLevel = &pWInfo->a[iLevel]; 001237 pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; 001238 if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue; 001239 pLoop = pLevel->pWLoop; 001240 if( NEVER(pLoop==0) ) continue; 001241 if( pLoop->prereq & notReady ) continue; 001242 if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN)) 001243 ==WHERE_BLOOMFILTER 001244 ){ 001245 /* This is a candidate for bloom-filter pull-down (early evaluation). 001246 ** The test that WHERE_COLUMN_IN is omitted is important, as we are 001247 ** not able to do early evaluation of bloom filters that make use of 001248 ** the IN operator */ 001249 break; 001250 } 001251 } 001252 }while( iLevel < pWInfo->nLevel ); 001253 sqlite3VdbeJumpHere(v, addrOnce); 001254 pParse->pIdxEpr = saved_pIdxEpr; 001255 pParse->pIdxPartExpr = saved_pIdxPartExpr; 001256 } 001257 001258 001259 #ifndef SQLITE_OMIT_VIRTUALTABLE 001260 /* 001261 ** Allocate and populate an sqlite3_index_info structure. It is the 001262 ** responsibility of the caller to eventually release the structure 001263 ** by passing the pointer returned by this function to freeIndexInfo(). 001264 */ 001265 static sqlite3_index_info *allocateIndexInfo( 001266 WhereInfo *pWInfo, /* The WHERE clause */ 001267 WhereClause *pWC, /* The WHERE clause being analyzed */ 001268 Bitmask mUnusable, /* Ignore terms with these prereqs */ 001269 SrcItem *pSrc, /* The FROM clause term that is the vtab */ 001270 u16 *pmNoOmit /* Mask of terms not to omit */ 001271 ){ 001272 int i, j; 001273 int nTerm; 001274 Parse *pParse = pWInfo->pParse; 001275 struct sqlite3_index_constraint *pIdxCons; 001276 struct sqlite3_index_orderby *pIdxOrderBy; 001277 struct sqlite3_index_constraint_usage *pUsage; 001278 struct HiddenIndexInfo *pHidden; 001279 WhereTerm *pTerm; 001280 int nOrderBy; 001281 sqlite3_index_info *pIdxInfo; 001282 u16 mNoOmit = 0; 001283 const Table *pTab; 001284 int eDistinct = 0; 001285 ExprList *pOrderBy = pWInfo->pOrderBy; 001286 001287 assert( pSrc!=0 ); 001288 pTab = pSrc->pTab; 001289 assert( pTab!=0 ); 001290 assert( IsVirtual(pTab) ); 001291 001292 /* Find all WHERE clause constraints referring to this virtual table. 001293 ** Mark each term with the TERM_OK flag. Set nTerm to the number of 001294 ** terms found. 001295 */ 001296 for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ 001297 pTerm->wtFlags &= ~TERM_OK; 001298 if( pTerm->leftCursor != pSrc->iCursor ) continue; 001299 if( pTerm->prereqRight & mUnusable ) continue; 001300 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); 001301 testcase( pTerm->eOperator & WO_IN ); 001302 testcase( pTerm->eOperator & WO_ISNULL ); 001303 testcase( pTerm->eOperator & WO_IS ); 001304 testcase( pTerm->eOperator & WO_ALL ); 001305 if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; 001306 if( pTerm->wtFlags & TERM_VNULL ) continue; 001307 001308 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); 001309 assert( pTerm->u.x.leftColumn>=XN_ROWID ); 001310 assert( pTerm->u.x.leftColumn<pTab->nCol ); 001311 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 001312 && !constraintCompatibleWithOuterJoin(pTerm,pSrc) 001313 ){ 001314 continue; 001315 } 001316 nTerm++; 001317 pTerm->wtFlags |= TERM_OK; 001318 } 001319 001320 /* If the ORDER BY clause contains only columns in the current 001321 ** virtual table then allocate space for the aOrderBy part of 001322 ** the sqlite3_index_info structure. 001323 */ 001324 nOrderBy = 0; 001325 if( pOrderBy ){ 001326 int n = pOrderBy->nExpr; 001327 for(i=0; i<n; i++){ 001328 Expr *pExpr = pOrderBy->a[i].pExpr; 001329 Expr *pE2; 001330 001331 /* Skip over constant terms in the ORDER BY clause */ 001332 if( sqlite3ExprIsConstant(pExpr) ){ 001333 continue; 001334 } 001335 001336 /* Virtual tables are unable to deal with NULLS FIRST */ 001337 if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break; 001338 001339 /* First case - a direct column references without a COLLATE operator */ 001340 if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){ 001341 assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol ); 001342 continue; 001343 } 001344 001345 /* 2nd case - a column reference with a COLLATE operator. Only match 001346 ** of the COLLATE operator matches the collation of the column. */ 001347 if( pExpr->op==TK_COLLATE 001348 && (pE2 = pExpr->pLeft)->op==TK_COLUMN 001349 && pE2->iTable==pSrc->iCursor 001350 ){ 001351 const char *zColl; /* The collating sequence name */ 001352 assert( !ExprHasProperty(pExpr, EP_IntValue) ); 001353 assert( pExpr->u.zToken!=0 ); 001354 assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol ); 001355 pExpr->iColumn = pE2->iColumn; 001356 if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */ 001357 zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]); 001358 if( zColl==0 ) zColl = sqlite3StrBINARY; 001359 if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue; 001360 } 001361 001362 /* No matches cause a break out of the loop */ 001363 break; 001364 } 001365 if( i==n ){ 001366 nOrderBy = n; 001367 if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ 001368 eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); 001369 }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ 001370 eDistinct = 1; 001371 } 001372 } 001373 } 001374 001375 /* Allocate the sqlite3_index_info structure 001376 */ 001377 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) 001378 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm 001379 + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) 001380 + sizeof(sqlite3_value*)*nTerm ); 001381 if( pIdxInfo==0 ){ 001382 sqlite3ErrorMsg(pParse, "out of memory"); 001383 return 0; 001384 } 001385 pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1]; 001386 pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm]; 001387 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; 001388 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; 001389 pIdxInfo->aConstraint = pIdxCons; 001390 pIdxInfo->aOrderBy = pIdxOrderBy; 001391 pIdxInfo->aConstraintUsage = pUsage; 001392 pHidden->pWC = pWC; 001393 pHidden->pParse = pParse; 001394 pHidden->eDistinct = eDistinct; 001395 pHidden->mIn = 0; 001396 for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ 001397 u16 op; 001398 if( (pTerm->wtFlags & TERM_OK)==0 ) continue; 001399 pIdxCons[j].iColumn = pTerm->u.x.leftColumn; 001400 pIdxCons[j].iTermOffset = i; 001401 op = pTerm->eOperator & WO_ALL; 001402 if( op==WO_IN ){ 001403 if( (pTerm->wtFlags & TERM_SLICE)==0 ){ 001404 pHidden->mIn |= SMASKBIT32(j); 001405 } 001406 op = WO_EQ; 001407 } 001408 if( op==WO_AUX ){ 001409 pIdxCons[j].op = pTerm->eMatchOp; 001410 }else if( op & (WO_ISNULL|WO_IS) ){ 001411 if( op==WO_ISNULL ){ 001412 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; 001413 }else{ 001414 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; 001415 } 001416 }else{ 001417 pIdxCons[j].op = (u8)op; 001418 /* The direct assignment in the previous line is possible only because 001419 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The 001420 ** following asserts verify this fact. */ 001421 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); 001422 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); 001423 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); 001424 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); 001425 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); 001426 assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); 001427 001428 if( op & (WO_LT|WO_LE|WO_GT|WO_GE) 001429 && sqlite3ExprIsVector(pTerm->pExpr->pRight) 001430 ){ 001431 testcase( j!=i ); 001432 if( j<16 ) mNoOmit |= (1 << j); 001433 if( op==WO_LT ) pIdxCons[j].op = WO_LE; 001434 if( op==WO_GT ) pIdxCons[j].op = WO_GE; 001435 } 001436 } 001437 001438 j++; 001439 } 001440 assert( j==nTerm ); 001441 pIdxInfo->nConstraint = j; 001442 for(i=j=0; i<nOrderBy; i++){ 001443 Expr *pExpr = pOrderBy->a[i].pExpr; 001444 if( sqlite3ExprIsConstant(pExpr) ) continue; 001445 assert( pExpr->op==TK_COLUMN 001446 || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN 001447 && pExpr->iColumn==pExpr->pLeft->iColumn) ); 001448 pIdxOrderBy[j].iColumn = pExpr->iColumn; 001449 pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC; 001450 j++; 001451 } 001452 pIdxInfo->nOrderBy = j; 001453 001454 *pmNoOmit = mNoOmit; 001455 return pIdxInfo; 001456 } 001457 001458 /* 001459 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo() 001460 ** and possibly modified by xBestIndex methods. 001461 */ 001462 static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){ 001463 HiddenIndexInfo *pHidden; 001464 int i; 001465 assert( pIdxInfo!=0 ); 001466 pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; 001467 assert( pHidden->pParse!=0 ); 001468 assert( pHidden->pParse->db==db ); 001469 for(i=0; i<pIdxInfo->nConstraint; i++){ 001470 sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */ 001471 pHidden->aRhs[i] = 0; 001472 } 001473 sqlite3DbFree(db, pIdxInfo); 001474 } 001475 001476 /* 001477 ** The table object reference passed as the second argument to this function 001478 ** must represent a virtual table. This function invokes the xBestIndex() 001479 ** method of the virtual table with the sqlite3_index_info object that 001480 ** comes in as the 3rd argument to this function. 001481 ** 001482 ** If an error occurs, pParse is populated with an error message and an 001483 ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from 001484 ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that 001485 ** the current configuration of "unusable" flags in sqlite3_index_info can 001486 ** not result in a valid plan. 001487 ** 001488 ** Whether or not an error is returned, it is the responsibility of the 001489 ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates 001490 ** that this is required. 001491 */ 001492 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ 001493 sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; 001494 int rc; 001495 001496 whereTraceIndexInfoInputs(p); 001497 pParse->db->nSchemaLock++; 001498 rc = pVtab->pModule->xBestIndex(pVtab, p); 001499 pParse->db->nSchemaLock--; 001500 whereTraceIndexInfoOutputs(p); 001501 001502 if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ 001503 if( rc==SQLITE_NOMEM ){ 001504 sqlite3OomFault(pParse->db); 001505 }else if( !pVtab->zErrMsg ){ 001506 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); 001507 }else{ 001508 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); 001509 } 001510 } 001511 if( pTab->u.vtab.p->bAllSchemas ){ 001512 sqlite3VtabUsesAllSchemas(pParse); 001513 } 001514 sqlite3_free(pVtab->zErrMsg); 001515 pVtab->zErrMsg = 0; 001516 return rc; 001517 } 001518 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ 001519 001520 #ifdef SQLITE_ENABLE_STAT4 001521 /* 001522 ** Estimate the location of a particular key among all keys in an 001523 ** index. Store the results in aStat as follows: 001524 ** 001525 ** aStat[0] Est. number of rows less than pRec 001526 ** aStat[1] Est. number of rows equal to pRec 001527 ** 001528 ** Return the index of the sample that is the smallest sample that 001529 ** is greater than or equal to pRec. Note that this index is not an index 001530 ** into the aSample[] array - it is an index into a virtual set of samples 001531 ** based on the contents of aSample[] and the number of fields in record 001532 ** pRec. 001533 */ 001534 static int whereKeyStats( 001535 Parse *pParse, /* Database connection */ 001536 Index *pIdx, /* Index to consider domain of */ 001537 UnpackedRecord *pRec, /* Vector of values to consider */ 001538 int roundUp, /* Round up if true. Round down if false */ 001539 tRowcnt *aStat /* OUT: stats written here */ 001540 ){ 001541 IndexSample *aSample = pIdx->aSample; 001542 int iCol; /* Index of required stats in anEq[] etc. */ 001543 int i; /* Index of first sample >= pRec */ 001544 int iSample; /* Smallest sample larger than or equal to pRec */ 001545 int iMin = 0; /* Smallest sample not yet tested */ 001546 int iTest; /* Next sample to test */ 001547 int res; /* Result of comparison operation */ 001548 int nField; /* Number of fields in pRec */ 001549 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ 001550 001551 #ifndef SQLITE_DEBUG 001552 UNUSED_PARAMETER( pParse ); 001553 #endif 001554 assert( pRec!=0 ); 001555 assert( pIdx->nSample>0 ); 001556 assert( pRec->nField>0 ); 001557 001558 001559 /* Do a binary search to find the first sample greater than or equal 001560 ** to pRec. If pRec contains a single field, the set of samples to search 001561 ** is simply the aSample[] array. If the samples in aSample[] contain more 001562 ** than one fields, all fields following the first are ignored. 001563 ** 001564 ** If pRec contains N fields, where N is more than one, then as well as the 001565 ** samples in aSample[] (truncated to N fields), the search also has to 001566 ** consider prefixes of those samples. For example, if the set of samples 001567 ** in aSample is: 001568 ** 001569 ** aSample[0] = (a, 5) 001570 ** aSample[1] = (a, 10) 001571 ** aSample[2] = (b, 5) 001572 ** aSample[3] = (c, 100) 001573 ** aSample[4] = (c, 105) 001574 ** 001575 ** Then the search space should ideally be the samples above and the 001576 ** unique prefixes [a], [b] and [c]. But since that is hard to organize, 001577 ** the code actually searches this set: 001578 ** 001579 ** 0: (a) 001580 ** 1: (a, 5) 001581 ** 2: (a, 10) 001582 ** 3: (a, 10) 001583 ** 4: (b) 001584 ** 5: (b, 5) 001585 ** 6: (c) 001586 ** 7: (c, 100) 001587 ** 8: (c, 105) 001588 ** 9: (c, 105) 001589 ** 001590 ** For each sample in the aSample[] array, N samples are present in the 001591 ** effective sample array. In the above, samples 0 and 1 are based on 001592 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. 001593 ** 001594 ** Often, sample i of each block of N effective samples has (i+1) fields. 001595 ** Except, each sample may be extended to ensure that it is greater than or 001596 ** equal to the previous sample in the array. For example, in the above, 001597 ** sample 2 is the first sample of a block of N samples, so at first it 001598 ** appears that it should be 1 field in size. However, that would make it 001599 ** smaller than sample 1, so the binary search would not work. As a result, 001600 ** it is extended to two fields. The duplicates that this creates do not 001601 ** cause any problems. 001602 */ 001603 if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ 001604 nField = pIdx->nKeyCol; 001605 }else{ 001606 nField = pIdx->nColumn; 001607 } 001608 nField = MIN(pRec->nField, nField); 001609 iCol = 0; 001610 iSample = pIdx->nSample * nField; 001611 do{ 001612 int iSamp; /* Index in aSample[] of test sample */ 001613 int n; /* Number of fields in test sample */ 001614 001615 iTest = (iMin+iSample)/2; 001616 iSamp = iTest / nField; 001617 if( iSamp>0 ){ 001618 /* The proposed effective sample is a prefix of sample aSample[iSamp]. 001619 ** Specifically, the shortest prefix of at least (1 + iTest%nField) 001620 ** fields that is greater than the previous effective sample. */ 001621 for(n=(iTest % nField) + 1; n<nField; n++){ 001622 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break; 001623 } 001624 }else{ 001625 n = iTest + 1; 001626 } 001627 001628 pRec->nField = n; 001629 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); 001630 if( res<0 ){ 001631 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; 001632 iMin = iTest+1; 001633 }else if( res==0 && n<nField ){ 001634 iLower = aSample[iSamp].anLt[n-1]; 001635 iMin = iTest+1; 001636 res = -1; 001637 }else{ 001638 iSample = iTest; 001639 iCol = n-1; 001640 } 001641 }while( res && iMin<iSample ); 001642 i = iSample / nField; 001643 001644 #ifdef SQLITE_DEBUG 001645 /* The following assert statements check that the binary search code 001646 ** above found the right answer. This block serves no purpose other 001647 ** than to invoke the asserts. */ 001648 if( pParse->db->mallocFailed==0 ){ 001649 if( res==0 ){ 001650 /* If (res==0) is true, then pRec must be equal to sample i. */ 001651 assert( i<pIdx->nSample ); 001652 assert( iCol==nField-1 ); 001653 pRec->nField = nField; 001654 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) 001655 || pParse->db->mallocFailed 001656 ); 001657 }else{ 001658 /* Unless i==pIdx->nSample, indicating that pRec is larger than 001659 ** all samples in the aSample[] array, pRec must be smaller than the 001660 ** (iCol+1) field prefix of sample i. */ 001661 assert( i<=pIdx->nSample && i>=0 ); 001662 pRec->nField = iCol+1; 001663 assert( i==pIdx->nSample 001664 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 001665 || pParse->db->mallocFailed ); 001666 001667 /* if i==0 and iCol==0, then record pRec is smaller than all samples 001668 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must 001669 ** be greater than or equal to the (iCol) field prefix of sample i. 001670 ** If (i>0), then pRec must also be greater than sample (i-1). */ 001671 if( iCol>0 ){ 001672 pRec->nField = iCol; 001673 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 001674 || pParse->db->mallocFailed || CORRUPT_DB ); 001675 } 001676 if( i>0 ){ 001677 pRec->nField = nField; 001678 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 001679 || pParse->db->mallocFailed || CORRUPT_DB ); 001680 } 001681 } 001682 } 001683 #endif /* ifdef SQLITE_DEBUG */ 001684 001685 if( res==0 ){ 001686 /* Record pRec is equal to sample i */ 001687 assert( iCol==nField-1 ); 001688 aStat[0] = aSample[i].anLt[iCol]; 001689 aStat[1] = aSample[i].anEq[iCol]; 001690 }else{ 001691 /* At this point, the (iCol+1) field prefix of aSample[i] is the first 001692 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec 001693 ** is larger than all samples in the array. */ 001694 tRowcnt iUpper, iGap; 001695 if( i>=pIdx->nSample ){ 001696 iUpper = pIdx->nRowEst0; 001697 }else{ 001698 iUpper = aSample[i].anLt[iCol]; 001699 } 001700 001701 if( iLower>=iUpper ){ 001702 iGap = 0; 001703 }else{ 001704 iGap = iUpper - iLower; 001705 } 001706 if( roundUp ){ 001707 iGap = (iGap*2)/3; 001708 }else{ 001709 iGap = iGap/3; 001710 } 001711 aStat[0] = iLower + iGap; 001712 aStat[1] = pIdx->aAvgEq[nField-1]; 001713 } 001714 001715 /* Restore the pRec->nField value before returning. */ 001716 pRec->nField = nField; 001717 return i; 001718 } 001719 #endif /* SQLITE_ENABLE_STAT4 */ 001720 001721 /* 001722 ** If it is not NULL, pTerm is a term that provides an upper or lower 001723 ** bound on a range scan. Without considering pTerm, it is estimated 001724 ** that the scan will visit nNew rows. This function returns the number 001725 ** estimated to be visited after taking pTerm into account. 001726 ** 001727 ** If the user explicitly specified a likelihood() value for this term, 001728 ** then the return value is the likelihood multiplied by the number of 001729 ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term 001730 ** has a likelihood of 0.50, and any other term a likelihood of 0.25. 001731 */ 001732 static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ 001733 LogEst nRet = nNew; 001734 if( pTerm ){ 001735 if( pTerm->truthProb<=0 ){ 001736 nRet += pTerm->truthProb; 001737 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ 001738 nRet -= 20; assert( 20==sqlite3LogEst(4) ); 001739 } 001740 } 001741 return nRet; 001742 } 001743 001744 001745 #ifdef SQLITE_ENABLE_STAT4 001746 /* 001747 ** Return the affinity for a single column of an index. 001748 */ 001749 char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ 001750 assert( iCol>=0 && iCol<pIdx->nColumn ); 001751 if( !pIdx->zColAff ){ 001752 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; 001753 } 001754 assert( pIdx->zColAff[iCol]!=0 ); 001755 return pIdx->zColAff[iCol]; 001756 } 001757 #endif 001758 001759 001760 #ifdef SQLITE_ENABLE_STAT4 001761 /* 001762 ** This function is called to estimate the number of rows visited by a 001763 ** range-scan on a skip-scan index. For example: 001764 ** 001765 ** CREATE INDEX i1 ON t1(a, b, c); 001766 ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; 001767 ** 001768 ** Value pLoop->nOut is currently set to the estimated number of rows 001769 ** visited for scanning (a=? AND b=?). This function reduces that estimate 001770 ** by some factor to account for the (c BETWEEN ? AND ?) expression based 001771 ** on the stat4 data for the index. this scan will be performed multiple 001772 ** times (once for each (a,b) combination that matches a=?) is dealt with 001773 ** by the caller. 001774 ** 001775 ** It does this by scanning through all stat4 samples, comparing values 001776 ** extracted from pLower and pUpper with the corresponding column in each 001777 ** sample. If L and U are the number of samples found to be less than or 001778 ** equal to the values extracted from pLower and pUpper respectively, and 001779 ** N is the total number of samples, the pLoop->nOut value is adjusted 001780 ** as follows: 001781 ** 001782 ** nOut = nOut * ( min(U - L, 1) / N ) 001783 ** 001784 ** If pLower is NULL, or a value cannot be extracted from the term, L is 001785 ** set to zero. If pUpper is NULL, or a value cannot be extracted from it, 001786 ** U is set to N. 001787 ** 001788 ** Normally, this function sets *pbDone to 1 before returning. However, 001789 ** if no value can be extracted from either pLower or pUpper (and so the 001790 ** estimate of the number of rows delivered remains unchanged), *pbDone 001791 ** is left as is. 001792 ** 001793 ** If an error occurs, an SQLite error code is returned. Otherwise, 001794 ** SQLITE_OK. 001795 */ 001796 static int whereRangeSkipScanEst( 001797 Parse *pParse, /* Parsing & code generating context */ 001798 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ 001799 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ 001800 WhereLoop *pLoop, /* Update the .nOut value of this loop */ 001801 int *pbDone /* Set to true if at least one expr. value extracted */ 001802 ){ 001803 Index *p = pLoop->u.btree.pIndex; 001804 int nEq = pLoop->u.btree.nEq; 001805 sqlite3 *db = pParse->db; 001806 int nLower = -1; 001807 int nUpper = p->nSample+1; 001808 int rc = SQLITE_OK; 001809 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); 001810 CollSeq *pColl; 001811 001812 sqlite3_value *p1 = 0; /* Value extracted from pLower */ 001813 sqlite3_value *p2 = 0; /* Value extracted from pUpper */ 001814 sqlite3_value *pVal = 0; /* Value extracted from record */ 001815 001816 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); 001817 if( pLower ){ 001818 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); 001819 nLower = 0; 001820 } 001821 if( pUpper && rc==SQLITE_OK ){ 001822 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); 001823 nUpper = p2 ? 0 : p->nSample; 001824 } 001825 001826 if( p1 || p2 ){ 001827 int i; 001828 int nDiff; 001829 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){ 001830 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); 001831 if( rc==SQLITE_OK && p1 ){ 001832 int res = sqlite3MemCompare(p1, pVal, pColl); 001833 if( res>=0 ) nLower++; 001834 } 001835 if( rc==SQLITE_OK && p2 ){ 001836 int res = sqlite3MemCompare(p2, pVal, pColl); 001837 if( res>=0 ) nUpper++; 001838 } 001839 } 001840 nDiff = (nUpper - nLower); 001841 if( nDiff<=0 ) nDiff = 1; 001842 001843 /* If there is both an upper and lower bound specified, and the 001844 ** comparisons indicate that they are close together, use the fallback 001845 ** method (assume that the scan visits 1/64 of the rows) for estimating 001846 ** the number of rows visited. Otherwise, estimate the number of rows 001847 ** using the method described in the header comment for this function. */ 001848 if( nDiff!=1 || pUpper==0 || pLower==0 ){ 001849 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); 001850 pLoop->nOut -= nAdjust; 001851 *pbDone = 1; 001852 WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", 001853 nLower, nUpper, nAdjust*-1, pLoop->nOut)); 001854 } 001855 001856 }else{ 001857 assert( *pbDone==0 ); 001858 } 001859 001860 sqlite3ValueFree(p1); 001861 sqlite3ValueFree(p2); 001862 sqlite3ValueFree(pVal); 001863 001864 return rc; 001865 } 001866 #endif /* SQLITE_ENABLE_STAT4 */ 001867 001868 /* 001869 ** This function is used to estimate the number of rows that will be visited 001870 ** by scanning an index for a range of values. The range may have an upper 001871 ** bound, a lower bound, or both. The WHERE clause terms that set the upper 001872 ** and lower bounds are represented by pLower and pUpper respectively. For 001873 ** example, assuming that index p is on t1(a): 001874 ** 001875 ** ... FROM t1 WHERE a > ? AND a < ? ... 001876 ** |_____| |_____| 001877 ** | | 001878 ** pLower pUpper 001879 ** 001880 ** If either of the upper or lower bound is not present, then NULL is passed in 001881 ** place of the corresponding WhereTerm. 001882 ** 001883 ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index 001884 ** column subject to the range constraint. Or, equivalently, the number of 001885 ** equality constraints optimized by the proposed index scan. For example, 001886 ** assuming index p is on t1(a, b), and the SQL query is: 001887 ** 001888 ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... 001889 ** 001890 ** then nEq is set to 1 (as the range restricted column, b, is the second 001891 ** left-most column of the index). Or, if the query is: 001892 ** 001893 ** ... FROM t1 WHERE a > ? AND a < ? ... 001894 ** 001895 ** then nEq is set to 0. 001896 ** 001897 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the 001898 ** number of rows that the index scan is expected to visit without 001899 ** considering the range constraints. If nEq is 0, then *pnOut is the number of 001900 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) 001901 ** to account for the range constraints pLower and pUpper. 001902 ** 001903 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be 001904 ** used, a single range inequality reduces the search space by a factor of 4. 001905 ** and a pair of constraints (x>? AND x<?) reduces the expected number of 001906 ** rows visited by a factor of 64. 001907 */ 001908 static int whereRangeScanEst( 001909 Parse *pParse, /* Parsing & code generating context */ 001910 WhereLoopBuilder *pBuilder, 001911 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ 001912 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ 001913 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ 001914 ){ 001915 int rc = SQLITE_OK; 001916 int nOut = pLoop->nOut; 001917 LogEst nNew; 001918 001919 #ifdef SQLITE_ENABLE_STAT4 001920 Index *p = pLoop->u.btree.pIndex; 001921 int nEq = pLoop->u.btree.nEq; 001922 001923 if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol) 001924 && OptimizationEnabled(pParse->db, SQLITE_Stat4) 001925 ){ 001926 if( nEq==pBuilder->nRecValid ){ 001927 UnpackedRecord *pRec = pBuilder->pRec; 001928 tRowcnt a[2]; 001929 int nBtm = pLoop->u.btree.nBtm; 001930 int nTop = pLoop->u.btree.nTop; 001931 001932 /* Variable iLower will be set to the estimate of the number of rows in 001933 ** the index that are less than the lower bound of the range query. The 001934 ** lower bound being the concatenation of $P and $L, where $P is the 001935 ** key-prefix formed by the nEq values matched against the nEq left-most 001936 ** columns of the index, and $L is the value in pLower. 001937 ** 001938 ** Or, if pLower is NULL or $L cannot be extracted from it (because it 001939 ** is not a simple variable or literal value), the lower bound of the 001940 ** range is $P. Due to a quirk in the way whereKeyStats() works, even 001941 ** if $L is available, whereKeyStats() is called for both ($P) and 001942 ** ($P:$L) and the larger of the two returned values is used. 001943 ** 001944 ** Similarly, iUpper is to be set to the estimate of the number of rows 001945 ** less than the upper bound of the range query. Where the upper bound 001946 ** is either ($P) or ($P:$U). Again, even if $U is available, both values 001947 ** of iUpper are requested of whereKeyStats() and the smaller used. 001948 ** 001949 ** The number of rows between the two bounds is then just iUpper-iLower. 001950 */ 001951 tRowcnt iLower; /* Rows less than the lower bound */ 001952 tRowcnt iUpper; /* Rows less than the upper bound */ 001953 int iLwrIdx = -2; /* aSample[] for the lower bound */ 001954 int iUprIdx = -1; /* aSample[] for the upper bound */ 001955 001956 if( pRec ){ 001957 testcase( pRec->nField!=pBuilder->nRecValid ); 001958 pRec->nField = pBuilder->nRecValid; 001959 } 001960 /* Determine iLower and iUpper using ($P) only. */ 001961 if( nEq==0 ){ 001962 iLower = 0; 001963 iUpper = p->nRowEst0; 001964 }else{ 001965 /* Note: this call could be optimized away - since the same values must 001966 ** have been requested when testing key $P in whereEqualScanEst(). */ 001967 whereKeyStats(pParse, p, pRec, 0, a); 001968 iLower = a[0]; 001969 iUpper = a[0] + a[1]; 001970 } 001971 001972 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); 001973 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); 001974 assert( p->aSortOrder!=0 ); 001975 if( p->aSortOrder[nEq] ){ 001976 /* The roles of pLower and pUpper are swapped for a DESC index */ 001977 SWAP(WhereTerm*, pLower, pUpper); 001978 SWAP(int, nBtm, nTop); 001979 } 001980 001981 /* If possible, improve on the iLower estimate using ($P:$L). */ 001982 if( pLower ){ 001983 int n; /* Values extracted from pExpr */ 001984 Expr *pExpr = pLower->pExpr->pRight; 001985 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n); 001986 if( rc==SQLITE_OK && n ){ 001987 tRowcnt iNew; 001988 u16 mask = WO_GT|WO_LE; 001989 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); 001990 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); 001991 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0); 001992 if( iNew>iLower ) iLower = iNew; 001993 nOut--; 001994 pLower = 0; 001995 } 001996 } 001997 001998 /* If possible, improve on the iUpper estimate using ($P:$U). */ 001999 if( pUpper ){ 002000 int n; /* Values extracted from pExpr */ 002001 Expr *pExpr = pUpper->pExpr->pRight; 002002 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n); 002003 if( rc==SQLITE_OK && n ){ 002004 tRowcnt iNew; 002005 u16 mask = WO_GT|WO_LE; 002006 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); 002007 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); 002008 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0); 002009 if( iNew<iUpper ) iUpper = iNew; 002010 nOut--; 002011 pUpper = 0; 002012 } 002013 } 002014 002015 pBuilder->pRec = pRec; 002016 if( rc==SQLITE_OK ){ 002017 if( iUpper>iLower ){ 002018 nNew = sqlite3LogEst(iUpper - iLower); 002019 /* TUNING: If both iUpper and iLower are derived from the same 002020 ** sample, then assume they are 4x more selective. This brings 002021 ** the estimated selectivity more in line with what it would be 002022 ** if estimated without the use of STAT4 tables. */ 002023 if( iLwrIdx==iUprIdx ){ nNew -= 20; } 002024 assert( 20==sqlite3LogEst(4) ); 002025 }else{ 002026 nNew = 10; assert( 10==sqlite3LogEst(2) ); 002027 } 002028 if( nNew<nOut ){ 002029 nOut = nNew; 002030 } 002031 WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n", 002032 (u32)iLower, (u32)iUpper, nOut)); 002033 } 002034 }else{ 002035 int bDone = 0; 002036 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); 002037 if( bDone ) return rc; 002038 } 002039 } 002040 #else 002041 UNUSED_PARAMETER(pParse); 002042 UNUSED_PARAMETER(pBuilder); 002043 assert( pLower || pUpper ); 002044 #endif 002045 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 ); 002046 nNew = whereRangeAdjust(pLower, nOut); 002047 nNew = whereRangeAdjust(pUpper, nNew); 002048 002049 /* TUNING: If there is both an upper and lower limit and neither limit 002050 ** has an application-defined likelihood(), assume the range is 002051 ** reduced by an additional 75%. This means that, by default, an open-ended 002052 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the 002053 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to 002054 ** match 1/64 of the index. */ 002055 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ 002056 nNew -= 20; 002057 } 002058 002059 nOut -= (pLower!=0) + (pUpper!=0); 002060 if( nNew<10 ) nNew = 10; 002061 if( nNew<nOut ) nOut = nNew; 002062 #if defined(WHERETRACE_ENABLED) 002063 if( pLoop->nOut>nOut ){ 002064 WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n", 002065 pLoop->nOut, nOut)); 002066 } 002067 #endif 002068 pLoop->nOut = (LogEst)nOut; 002069 return rc; 002070 } 002071 002072 #ifdef SQLITE_ENABLE_STAT4 002073 /* 002074 ** Estimate the number of rows that will be returned based on 002075 ** an equality constraint x=VALUE and where that VALUE occurs in 002076 ** the histogram data. This only works when x is the left-most 002077 ** column of an index and sqlite_stat4 histogram data is available 002078 ** for that index. When pExpr==NULL that means the constraint is 002079 ** "x IS NULL" instead of "x=VALUE". 002080 ** 002081 ** Write the estimated row count into *pnRow and return SQLITE_OK. 002082 ** If unable to make an estimate, leave *pnRow unchanged and return 002083 ** non-zero. 002084 ** 002085 ** This routine can fail if it is unable to load a collating sequence 002086 ** required for string comparison, or if unable to allocate memory 002087 ** for a UTF conversion required for comparison. The error is stored 002088 ** in the pParse structure. 002089 */ 002090 static int whereEqualScanEst( 002091 Parse *pParse, /* Parsing & code generating context */ 002092 WhereLoopBuilder *pBuilder, 002093 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ 002094 tRowcnt *pnRow /* Write the revised row estimate here */ 002095 ){ 002096 Index *p = pBuilder->pNew->u.btree.pIndex; 002097 int nEq = pBuilder->pNew->u.btree.nEq; 002098 UnpackedRecord *pRec = pBuilder->pRec; 002099 int rc; /* Subfunction return code */ 002100 tRowcnt a[2]; /* Statistics */ 002101 int bOk; 002102 002103 assert( nEq>=1 ); 002104 assert( nEq<=p->nColumn ); 002105 assert( p->aSample!=0 ); 002106 assert( p->nSample>0 ); 002107 assert( pBuilder->nRecValid<nEq ); 002108 002109 /* If values are not available for all fields of the index to the left 002110 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */ 002111 if( pBuilder->nRecValid<(nEq-1) ){ 002112 return SQLITE_NOTFOUND; 002113 } 002114 002115 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() 002116 ** below would return the same value. */ 002117 if( nEq>=p->nColumn ){ 002118 *pnRow = 1; 002119 return SQLITE_OK; 002120 } 002121 002122 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk); 002123 pBuilder->pRec = pRec; 002124 if( rc!=SQLITE_OK ) return rc; 002125 if( bOk==0 ) return SQLITE_NOTFOUND; 002126 pBuilder->nRecValid = nEq; 002127 002128 whereKeyStats(pParse, p, pRec, 0, a); 002129 WHERETRACE(0x20,("equality scan regions %s(%d): %d\n", 002130 p->zName, nEq-1, (int)a[1])); 002131 *pnRow = a[1]; 002132 002133 return rc; 002134 } 002135 #endif /* SQLITE_ENABLE_STAT4 */ 002136 002137 #ifdef SQLITE_ENABLE_STAT4 002138 /* 002139 ** Estimate the number of rows that will be returned based on 002140 ** an IN constraint where the right-hand side of the IN operator 002141 ** is a list of values. Example: 002142 ** 002143 ** WHERE x IN (1,2,3,4) 002144 ** 002145 ** Write the estimated row count into *pnRow and return SQLITE_OK. 002146 ** If unable to make an estimate, leave *pnRow unchanged and return 002147 ** non-zero. 002148 ** 002149 ** This routine can fail if it is unable to load a collating sequence 002150 ** required for string comparison, or if unable to allocate memory 002151 ** for a UTF conversion required for comparison. The error is stored 002152 ** in the pParse structure. 002153 */ 002154 static int whereInScanEst( 002155 Parse *pParse, /* Parsing & code generating context */ 002156 WhereLoopBuilder *pBuilder, 002157 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ 002158 tRowcnt *pnRow /* Write the revised row estimate here */ 002159 ){ 002160 Index *p = pBuilder->pNew->u.btree.pIndex; 002161 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); 002162 int nRecValid = pBuilder->nRecValid; 002163 int rc = SQLITE_OK; /* Subfunction return code */ 002164 tRowcnt nEst; /* Number of rows for a single term */ 002165 tRowcnt nRowEst = 0; /* New estimate of the number of rows */ 002166 int i; /* Loop counter */ 002167 002168 assert( p->aSample!=0 ); 002169 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){ 002170 nEst = nRow0; 002171 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); 002172 nRowEst += nEst; 002173 pBuilder->nRecValid = nRecValid; 002174 } 002175 002176 if( rc==SQLITE_OK ){ 002177 if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0; 002178 *pnRow = nRowEst; 002179 WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst)); 002180 } 002181 assert( pBuilder->nRecValid==nRecValid ); 002182 return rc; 002183 } 002184 #endif /* SQLITE_ENABLE_STAT4 */ 002185 002186 002187 #ifdef WHERETRACE_ENABLED 002188 /* 002189 ** Print the content of a WhereTerm object 002190 */ 002191 void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ 002192 if( pTerm==0 ){ 002193 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); 002194 }else{ 002195 char zType[8]; 002196 char zLeft[50]; 002197 memcpy(zType, "....", 5); 002198 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; 002199 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; 002200 if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L'; 002201 if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C'; 002202 if( pTerm->eOperator & WO_SINGLE ){ 002203 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); 002204 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}", 002205 pTerm->leftCursor, pTerm->u.x.leftColumn); 002206 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){ 002207 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx", 002208 pTerm->u.pOrInfo->indexable); 002209 }else{ 002210 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); 002211 } 002212 sqlite3DebugPrintf( 002213 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x", 002214 iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); 002215 /* The 0x10000 .wheretrace flag causes extra information to be 002216 ** shown about each Term */ 002217 if( sqlite3WhereTrace & 0x10000 ){ 002218 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx", 002219 pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight); 002220 } 002221 if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){ 002222 sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField); 002223 } 002224 if( pTerm->iParent>=0 ){ 002225 sqlite3DebugPrintf(" iParent=%d", pTerm->iParent); 002226 } 002227 sqlite3DebugPrintf("\n"); 002228 sqlite3TreeViewExpr(0, pTerm->pExpr, 0); 002229 } 002230 } 002231 #endif 002232 002233 #ifdef WHERETRACE_ENABLED 002234 /* 002235 ** Show the complete content of a WhereClause 002236 */ 002237 void sqlite3WhereClausePrint(WhereClause *pWC){ 002238 int i; 002239 for(i=0; i<pWC->nTerm; i++){ 002240 sqlite3WhereTermPrint(&pWC->a[i], i); 002241 } 002242 } 002243 #endif 002244 002245 #ifdef WHERETRACE_ENABLED 002246 /* 002247 ** Print a WhereLoop object for debugging purposes 002248 ** 002249 ** Format example: 002250 ** 002251 ** .--- Position in WHERE clause rSetup, rRun, nOut ---. 002252 ** | | 002253 ** | .--- selfMask nTerm ------. | 002254 ** | | | | 002255 ** | | .-- prereq Idx wsFlags----. | | 002256 ** | | | Name | | | 002257 ** | | | __|__ nEq ---. ___|__ | __|__ 002258 ** | / \ / \ / \ | / \ / \ / \ 002259 ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 002260 */ 002261 void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ 002262 if( pWC ){ 002263 WhereInfo *pWInfo = pWC->pWInfo; 002264 int nb = 1+(pWInfo->pTabList->nSrc+3)/4; 002265 SrcItem *pItem = pWInfo->pTabList->a + p->iTab; 002266 Table *pTab = pItem->pTab; 002267 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; 002268 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, 002269 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); 002270 sqlite3DebugPrintf(" %12s", 002271 pItem->zAlias ? pItem->zAlias : pTab->zName); 002272 }else{ 002273 sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d", 002274 p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab); 002275 } 002276 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ 002277 const char *zName; 002278 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ 002279 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ 002280 int i = sqlite3Strlen30(zName) - 1; 002281 while( zName[i]!='_' ) i--; 002282 zName += i; 002283 } 002284 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); 002285 }else{ 002286 sqlite3DebugPrintf("%20s",""); 002287 } 002288 }else{ 002289 char *z; 002290 if( p->u.vtab.idxStr ){ 002291 z = sqlite3_mprintf("(%d,\"%s\",%#x)", 002292 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); 002293 }else{ 002294 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); 002295 } 002296 sqlite3DebugPrintf(" %-19s", z); 002297 sqlite3_free(z); 002298 } 002299 if( p->wsFlags & WHERE_SKIPSCAN ){ 002300 sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); 002301 }else{ 002302 sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm); 002303 } 002304 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); 002305 if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){ 002306 int i; 002307 for(i=0; i<p->nLTerm; i++){ 002308 sqlite3WhereTermPrint(p->aLTerm[i], i); 002309 } 002310 } 002311 } 002312 void sqlite3ShowWhereLoop(const WhereLoop *p){ 002313 if( p ) sqlite3WhereLoopPrint(p, 0); 002314 } 002315 void sqlite3ShowWhereLoopList(const WhereLoop *p){ 002316 while( p ){ 002317 sqlite3ShowWhereLoop(p); 002318 p = p->pNextLoop; 002319 } 002320 } 002321 #endif 002322 002323 /* 002324 ** Convert bulk memory into a valid WhereLoop that can be passed 002325 ** to whereLoopClear harmlessly. 002326 */ 002327 static void whereLoopInit(WhereLoop *p){ 002328 p->aLTerm = p->aLTermSpace; 002329 p->nLTerm = 0; 002330 p->nLSlot = ArraySize(p->aLTermSpace); 002331 p->wsFlags = 0; 002332 } 002333 002334 /* 002335 ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. 002336 */ 002337 static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ 002338 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ 002339 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ 002340 sqlite3_free(p->u.vtab.idxStr); 002341 p->u.vtab.needFree = 0; 002342 p->u.vtab.idxStr = 0; 002343 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ 002344 sqlite3DbFree(db, p->u.btree.pIndex->zColAff); 002345 sqlite3DbFreeNN(db, p->u.btree.pIndex); 002346 p->u.btree.pIndex = 0; 002347 } 002348 } 002349 } 002350 002351 /* 002352 ** Deallocate internal memory used by a WhereLoop object. Leave the 002353 ** object in an initialized state, as if it had been newly allocated. 002354 */ 002355 static void whereLoopClear(sqlite3 *db, WhereLoop *p){ 002356 if( p->aLTerm!=p->aLTermSpace ){ 002357 sqlite3DbFreeNN(db, p->aLTerm); 002358 p->aLTerm = p->aLTermSpace; 002359 p->nLSlot = ArraySize(p->aLTermSpace); 002360 } 002361 whereLoopClearUnion(db, p); 002362 p->nLTerm = 0; 002363 p->wsFlags = 0; 002364 } 002365 002366 /* 002367 ** Increase the memory allocation for pLoop->aLTerm[] to be at least n. 002368 */ 002369 static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ 002370 WhereTerm **paNew; 002371 if( p->nLSlot>=n ) return SQLITE_OK; 002372 n = (n+7)&~7; 002373 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); 002374 if( paNew==0 ) return SQLITE_NOMEM_BKPT; 002375 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); 002376 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); 002377 p->aLTerm = paNew; 002378 p->nLSlot = n; 002379 return SQLITE_OK; 002380 } 002381 002382 /* 002383 ** Transfer content from the second pLoop into the first. 002384 */ 002385 static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ 002386 whereLoopClearUnion(db, pTo); 002387 if( pFrom->nLTerm > pTo->nLSlot 002388 && whereLoopResize(db, pTo, pFrom->nLTerm) 002389 ){ 002390 memset(pTo, 0, WHERE_LOOP_XFER_SZ); 002391 return SQLITE_NOMEM_BKPT; 002392 } 002393 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); 002394 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); 002395 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ 002396 pFrom->u.vtab.needFree = 0; 002397 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ 002398 pFrom->u.btree.pIndex = 0; 002399 } 002400 return SQLITE_OK; 002401 } 002402 002403 /* 002404 ** Delete a WhereLoop object 002405 */ 002406 static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ 002407 assert( db!=0 ); 002408 whereLoopClear(db, p); 002409 sqlite3DbNNFreeNN(db, p); 002410 } 002411 002412 /* 002413 ** Free a WhereInfo structure 002414 */ 002415 static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ 002416 assert( pWInfo!=0 ); 002417 assert( db!=0 ); 002418 sqlite3WhereClauseClear(&pWInfo->sWC); 002419 while( pWInfo->pLoops ){ 002420 WhereLoop *p = pWInfo->pLoops; 002421 pWInfo->pLoops = p->pNextLoop; 002422 whereLoopDelete(db, p); 002423 } 002424 while( pWInfo->pMemToFree ){ 002425 WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; 002426 sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); 002427 pWInfo->pMemToFree = pNext; 002428 } 002429 sqlite3DbNNFreeNN(db, pWInfo); 002430 } 002431 002432 /* 002433 ** Return TRUE if X is a proper subset of Y but is of equal or less cost. 002434 ** In other words, return true if all constraints of X are also part of Y 002435 ** and Y has additional constraints that might speed the search that X lacks 002436 ** but the cost of running X is not more than the cost of running Y. 002437 ** 002438 ** In other words, return true if the cost relationwship between X and Y 002439 ** is inverted and needs to be adjusted. 002440 ** 002441 ** Case 1: 002442 ** 002443 ** (1a) X and Y use the same index. 002444 ** (1b) X has fewer == terms than Y 002445 ** (1c) Neither X nor Y use skip-scan 002446 ** (1d) X does not have a a greater cost than Y 002447 ** 002448 ** Case 2: 002449 ** 002450 ** (2a) X has the same or lower cost, or returns the same or fewer rows, 002451 ** than Y. 002452 ** (2b) X uses fewer WHERE clause terms than Y 002453 ** (2c) Every WHERE clause term used by X is also used by Y 002454 ** (2d) X skips at least as many columns as Y 002455 ** (2e) If X is a covering index, than Y is too 002456 */ 002457 static int whereLoopCheaperProperSubset( 002458 const WhereLoop *pX, /* First WhereLoop to compare */ 002459 const WhereLoop *pY /* Compare against this WhereLoop */ 002460 ){ 002461 int i, j; 002462 if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */ 002463 assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 ); 002464 assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 ); 002465 if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */ 002466 && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */ 002467 && pX->nSkip==0 && pY->nSkip==0 /* (1c) */ 002468 ){ 002469 return 1; /* Case 1 is true */ 002470 } 002471 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ 002472 return 0; /* (2b) */ 002473 } 002474 if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */ 002475 for(i=pX->nLTerm-1; i>=0; i--){ 002476 if( pX->aLTerm[i]==0 ) continue; 002477 for(j=pY->nLTerm-1; j>=0; j--){ 002478 if( pY->aLTerm[j]==pX->aLTerm[i] ) break; 002479 } 002480 if( j<0 ) return 0; /* (2c) */ 002481 } 002482 if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 002483 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ 002484 return 0; /* (2e) */ 002485 } 002486 return 1; /* Case 2 is true */ 002487 } 002488 002489 /* 002490 ** Try to adjust the cost and number of output rows of WhereLoop pTemplate 002491 ** upwards or downwards so that: 002492 ** 002493 ** (1) pTemplate costs less than any other WhereLoops that are a proper 002494 ** subset of pTemplate 002495 ** 002496 ** (2) pTemplate costs more than any other WhereLoops for which pTemplate 002497 ** is a proper subset. 002498 ** 002499 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer 002500 ** WHERE clause terms than Y and that every WHERE clause term used by X is 002501 ** also used by Y. 002502 */ 002503 static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ 002504 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; 002505 for(; p; p=p->pNextLoop){ 002506 if( p->iTab!=pTemplate->iTab ) continue; 002507 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; 002508 if( whereLoopCheaperProperSubset(p, pTemplate) ){ 002509 /* Adjust pTemplate cost downward so that it is cheaper than its 002510 ** subset p. */ 002511 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", 002512 pTemplate->rRun, pTemplate->nOut, 002513 MIN(p->rRun, pTemplate->rRun), 002514 MIN(p->nOut - 1, pTemplate->nOut))); 002515 pTemplate->rRun = MIN(p->rRun, pTemplate->rRun); 002516 pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut); 002517 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ 002518 /* Adjust pTemplate cost upward so that it is costlier than p since 002519 ** pTemplate is a proper subset of p */ 002520 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", 002521 pTemplate->rRun, pTemplate->nOut, 002522 MAX(p->rRun, pTemplate->rRun), 002523 MAX(p->nOut + 1, pTemplate->nOut))); 002524 pTemplate->rRun = MAX(p->rRun, pTemplate->rRun); 002525 pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut); 002526 } 002527 } 002528 } 002529 002530 /* 002531 ** Search the list of WhereLoops in *ppPrev looking for one that can be 002532 ** replaced by pTemplate. 002533 ** 002534 ** Return NULL if pTemplate does not belong on the WhereLoop list. 002535 ** In other words if pTemplate ought to be dropped from further consideration. 002536 ** 002537 ** If pX is a WhereLoop that pTemplate can replace, then return the 002538 ** link that points to pX. 002539 ** 002540 ** If pTemplate cannot replace any existing element of the list but needs 002541 ** to be added to the list as a new entry, then return a pointer to the 002542 ** tail of the list. 002543 */ 002544 static WhereLoop **whereLoopFindLesser( 002545 WhereLoop **ppPrev, 002546 const WhereLoop *pTemplate 002547 ){ 002548 WhereLoop *p; 002549 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ 002550 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ 002551 /* If either the iTab or iSortIdx values for two WhereLoop are different 002552 ** then those WhereLoops need to be considered separately. Neither is 002553 ** a candidate to replace the other. */ 002554 continue; 002555 } 002556 /* In the current implementation, the rSetup value is either zero 002557 ** or the cost of building an automatic index (NlogN) and the NlogN 002558 ** is the same for compatible WhereLoops. */ 002559 assert( p->rSetup==0 || pTemplate->rSetup==0 002560 || p->rSetup==pTemplate->rSetup ); 002561 002562 /* whereLoopAddBtree() always generates and inserts the automatic index 002563 ** case first. Hence compatible candidate WhereLoops never have a larger 002564 ** rSetup. Call this SETUP-INVARIANT */ 002565 assert( p->rSetup>=pTemplate->rSetup ); 002566 002567 /* Any loop using an application-defined index (or PRIMARY KEY or 002568 ** UNIQUE constraint) with one or more == constraints is better 002569 ** than an automatic index. Unless it is a skip-scan. */ 002570 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 002571 && (pTemplate->nSkip)==0 002572 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 002573 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 002574 && (p->prereq & pTemplate->prereq)==pTemplate->prereq 002575 ){ 002576 break; 002577 } 002578 002579 /* If existing WhereLoop p is better than pTemplate, pTemplate can be 002580 ** discarded. WhereLoop p is better if: 002581 ** (1) p has no more dependencies than pTemplate, and 002582 ** (2) p has an equal or lower cost than pTemplate 002583 */ 002584 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ 002585 && p->rSetup<=pTemplate->rSetup /* (2a) */ 002586 && p->rRun<=pTemplate->rRun /* (2b) */ 002587 && p->nOut<=pTemplate->nOut /* (2c) */ 002588 ){ 002589 return 0; /* Discard pTemplate */ 002590 } 002591 002592 /* If pTemplate is always better than p, then cause p to be overwritten 002593 ** with pTemplate. pTemplate is better than p if: 002594 ** (1) pTemplate has no more dependencies than p, and 002595 ** (2) pTemplate has an equal or lower cost than p. 002596 */ 002597 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ 002598 && p->rRun>=pTemplate->rRun /* (2a) */ 002599 && p->nOut>=pTemplate->nOut /* (2b) */ 002600 ){ 002601 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ 002602 break; /* Cause p to be overwritten by pTemplate */ 002603 } 002604 } 002605 return ppPrev; 002606 } 002607 002608 /* 002609 ** Insert or replace a WhereLoop entry using the template supplied. 002610 ** 002611 ** An existing WhereLoop entry might be overwritten if the new template 002612 ** is better and has fewer dependencies. Or the template will be ignored 002613 ** and no insert will occur if an existing WhereLoop is faster and has 002614 ** fewer dependencies than the template. Otherwise a new WhereLoop is 002615 ** added based on the template. 002616 ** 002617 ** If pBuilder->pOrSet is not NULL then we care about only the 002618 ** prerequisites and rRun and nOut costs of the N best loops. That 002619 ** information is gathered in the pBuilder->pOrSet object. This special 002620 ** processing mode is used only for OR clause processing. 002621 ** 002622 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we 002623 ** still might overwrite similar loops with the new template if the 002624 ** new template is better. Loops may be overwritten if the following 002625 ** conditions are met: 002626 ** 002627 ** (1) They have the same iTab. 002628 ** (2) They have the same iSortIdx. 002629 ** (3) The template has same or fewer dependencies than the current loop 002630 ** (4) The template has the same or lower cost than the current loop 002631 */ 002632 static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ 002633 WhereLoop **ppPrev, *p; 002634 WhereInfo *pWInfo = pBuilder->pWInfo; 002635 sqlite3 *db = pWInfo->pParse->db; 002636 int rc; 002637 002638 /* Stop the search once we hit the query planner search limit */ 002639 if( pBuilder->iPlanLimit==0 ){ 002640 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n")); 002641 if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0; 002642 return SQLITE_DONE; 002643 } 002644 pBuilder->iPlanLimit--; 002645 002646 whereLoopAdjustCost(pWInfo->pLoops, pTemplate); 002647 002648 /* If pBuilder->pOrSet is defined, then only keep track of the costs 002649 ** and prereqs. 002650 */ 002651 if( pBuilder->pOrSet!=0 ){ 002652 if( pTemplate->nLTerm ){ 002653 #if WHERETRACE_ENABLED 002654 u16 n = pBuilder->pOrSet->n; 002655 int x = 002656 #endif 002657 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, 002658 pTemplate->nOut); 002659 #if WHERETRACE_ENABLED /* 0x8 */ 002660 if( sqlite3WhereTrace & 0x8 ){ 002661 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); 002662 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); 002663 } 002664 #endif 002665 } 002666 return SQLITE_OK; 002667 } 002668 002669 /* Look for an existing WhereLoop to replace with pTemplate 002670 */ 002671 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); 002672 002673 if( ppPrev==0 ){ 002674 /* There already exists a WhereLoop on the list that is better 002675 ** than pTemplate, so just ignore pTemplate */ 002676 #if WHERETRACE_ENABLED /* 0x8 */ 002677 if( sqlite3WhereTrace & 0x8 ){ 002678 sqlite3DebugPrintf(" skip: "); 002679 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); 002680 } 002681 #endif 002682 return SQLITE_OK; 002683 }else{ 002684 p = *ppPrev; 002685 } 002686 002687 /* If we reach this point it means that either p[] should be overwritten 002688 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new 002689 ** WhereLoop and insert it. 002690 */ 002691 #if WHERETRACE_ENABLED /* 0x8 */ 002692 if( sqlite3WhereTrace & 0x8 ){ 002693 if( p!=0 ){ 002694 sqlite3DebugPrintf("replace: "); 002695 sqlite3WhereLoopPrint(p, pBuilder->pWC); 002696 sqlite3DebugPrintf(" with: "); 002697 }else{ 002698 sqlite3DebugPrintf(" add: "); 002699 } 002700 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); 002701 } 002702 #endif 002703 if( p==0 ){ 002704 /* Allocate a new WhereLoop to add to the end of the list */ 002705 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop)); 002706 if( p==0 ) return SQLITE_NOMEM_BKPT; 002707 whereLoopInit(p); 002708 p->pNextLoop = 0; 002709 }else{ 002710 /* We will be overwriting WhereLoop p[]. But before we do, first 002711 ** go through the rest of the list and delete any other entries besides 002712 ** p[] that are also supplanted by pTemplate */ 002713 WhereLoop **ppTail = &p->pNextLoop; 002714 WhereLoop *pToDel; 002715 while( *ppTail ){ 002716 ppTail = whereLoopFindLesser(ppTail, pTemplate); 002717 if( ppTail==0 ) break; 002718 pToDel = *ppTail; 002719 if( pToDel==0 ) break; 002720 *ppTail = pToDel->pNextLoop; 002721 #if WHERETRACE_ENABLED /* 0x8 */ 002722 if( sqlite3WhereTrace & 0x8 ){ 002723 sqlite3DebugPrintf(" delete: "); 002724 sqlite3WhereLoopPrint(pToDel, pBuilder->pWC); 002725 } 002726 #endif 002727 whereLoopDelete(db, pToDel); 002728 } 002729 } 002730 rc = whereLoopXfer(db, p, pTemplate); 002731 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ 002732 Index *pIndex = p->u.btree.pIndex; 002733 if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){ 002734 p->u.btree.pIndex = 0; 002735 } 002736 } 002737 return rc; 002738 } 002739 002740 /* 002741 ** Adjust the WhereLoop.nOut value downward to account for terms of the 002742 ** WHERE clause that reference the loop but which are not used by an 002743 ** index. 002744 * 002745 ** For every WHERE clause term that is not used by the index 002746 ** and which has a truth probability assigned by one of the likelihood(), 002747 ** likely(), or unlikely() SQL functions, reduce the estimated number 002748 ** of output rows by the probability specified. 002749 ** 002750 ** TUNING: For every WHERE clause term that is not used by the index 002751 ** and which does not have an assigned truth probability, heuristics 002752 ** described below are used to try to estimate the truth probability. 002753 ** TODO --> Perhaps this is something that could be improved by better 002754 ** table statistics. 002755 ** 002756 ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% 002757 ** value corresponds to -1 in LogEst notation, so this means decrement 002758 ** the WhereLoop.nOut field for every such WHERE clause term. 002759 ** 002760 ** Heuristic 2: If there exists one or more WHERE clause terms of the 002761 ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the 002762 ** final output row estimate is no greater than 1/4 of the total number 002763 ** of rows in the table. In other words, assume that x==EXPR will filter 002764 ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the 002765 ** "x" column is boolean or else -1 or 0 or 1 is a common default value 002766 ** on the "x" column and so in that case only cap the output row estimate 002767 ** at 1/2 instead of 1/4. 002768 */ 002769 static void whereLoopOutputAdjust( 002770 WhereClause *pWC, /* The WHERE clause */ 002771 WhereLoop *pLoop, /* The loop to adjust downward */ 002772 LogEst nRow /* Number of rows in the entire table */ 002773 ){ 002774 WhereTerm *pTerm, *pX; 002775 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); 002776 int i, j; 002777 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ 002778 002779 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); 002780 for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ 002781 assert( pTerm!=0 ); 002782 if( (pTerm->prereqAll & notAllowed)!=0 ) continue; 002783 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; 002784 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue; 002785 for(j=pLoop->nLTerm-1; j>=0; j--){ 002786 pX = pLoop->aLTerm[j]; 002787 if( pX==0 ) continue; 002788 if( pX==pTerm ) break; 002789 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; 002790 } 002791 if( j<0 ){ 002792 sqlite3ProgressCheck(pWC->pWInfo->pParse); 002793 if( pLoop->maskSelf==pTerm->prereqAll ){ 002794 /* If there are extra terms in the WHERE clause not used by an index 002795 ** that depend only on the table being scanned, and that will tend to 002796 ** cause many rows to be omitted, then mark that table as 002797 ** "self-culling". 002798 ** 002799 ** 2022-03-24: Self-culling only applies if either the extra terms 002800 ** are straight comparison operators that are non-true with NULL 002801 ** operand, or if the loop is not an OUTER JOIN. 002802 */ 002803 if( (pTerm->eOperator & 0x3f)!=0 002804 || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype 002805 & (JT_LEFT|JT_LTORJ))==0 002806 ){ 002807 pLoop->wsFlags |= WHERE_SELFCULL; 002808 } 002809 } 002810 if( pTerm->truthProb<=0 ){ 002811 /* If a truth probability is specified using the likelihood() hints, 002812 ** then use the probability provided by the application. */ 002813 pLoop->nOut += pTerm->truthProb; 002814 }else{ 002815 /* In the absence of explicit truth probabilities, use heuristics to 002816 ** guess a reasonable truth probability. */ 002817 pLoop->nOut--; 002818 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 002819 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ 002820 ){ 002821 Expr *pRight = pTerm->pExpr->pRight; 002822 int k = 0; 002823 testcase( pTerm->pExpr->op==TK_IS ); 002824 if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ 002825 k = 10; 002826 }else{ 002827 k = 20; 002828 } 002829 if( iReduce<k ){ 002830 pTerm->wtFlags |= TERM_HEURTRUTH; 002831 iReduce = k; 002832 } 002833 } 002834 } 002835 } 002836 } 002837 if( pLoop->nOut > nRow-iReduce ){ 002838 pLoop->nOut = nRow - iReduce; 002839 } 002840 } 002841 002842 /* 002843 ** Term pTerm is a vector range comparison operation. The first comparison 002844 ** in the vector can be optimized using column nEq of the index. This 002845 ** function returns the total number of vector elements that can be used 002846 ** as part of the range comparison. 002847 ** 002848 ** For example, if the query is: 002849 ** 002850 ** WHERE a = ? AND (b, c, d) > (?, ?, ?) 002851 ** 002852 ** and the index: 002853 ** 002854 ** CREATE INDEX ... ON (a, b, c, d, e) 002855 ** 002856 ** then this function would be invoked with nEq=1. The value returned in 002857 ** this case is 3. 002858 */ 002859 static int whereRangeVectorLen( 002860 Parse *pParse, /* Parsing context */ 002861 int iCur, /* Cursor open on pIdx */ 002862 Index *pIdx, /* The index to be used for a inequality constraint */ 002863 int nEq, /* Number of prior equality constraints on same index */ 002864 WhereTerm *pTerm /* The vector inequality constraint */ 002865 ){ 002866 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft); 002867 int i; 002868 002869 nCmp = MIN(nCmp, (pIdx->nColumn - nEq)); 002870 for(i=1; i<nCmp; i++){ 002871 /* Test if comparison i of pTerm is compatible with column (i+nEq) 002872 ** of the index. If not, exit the loop. */ 002873 char aff; /* Comparison affinity */ 002874 char idxaff = 0; /* Indexed columns affinity */ 002875 CollSeq *pColl; /* Comparison collation sequence */ 002876 Expr *pLhs, *pRhs; 002877 002878 assert( ExprUseXList(pTerm->pExpr->pLeft) ); 002879 pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr; 002880 pRhs = pTerm->pExpr->pRight; 002881 if( ExprUseXSelect(pRhs) ){ 002882 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr; 002883 }else{ 002884 pRhs = pRhs->x.pList->a[i].pExpr; 002885 } 002886 002887 /* Check that the LHS of the comparison is a column reference to 002888 ** the right column of the right source table. And that the sort 002889 ** order of the index column is the same as the sort order of the 002890 ** leftmost index column. */ 002891 if( pLhs->op!=TK_COLUMN 002892 || pLhs->iTable!=iCur 002893 || pLhs->iColumn!=pIdx->aiColumn[i+nEq] 002894 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq] 002895 ){ 002896 break; 002897 } 002898 002899 testcase( pLhs->iColumn==XN_ROWID ); 002900 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs)); 002901 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); 002902 if( aff!=idxaff ) break; 002903 002904 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); 002905 if( pColl==0 ) break; 002906 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; 002907 } 002908 return i; 002909 } 002910 002911 /* 002912 ** Adjust the cost C by the costMult factor T. This only occurs if 002913 ** compiled with -DSQLITE_ENABLE_COSTMULT 002914 */ 002915 #ifdef SQLITE_ENABLE_COSTMULT 002916 # define ApplyCostMultiplier(C,T) C += T 002917 #else 002918 # define ApplyCostMultiplier(C,T) 002919 #endif 002920 002921 /* 002922 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the 002923 ** index pIndex. Try to match one more. 002924 ** 002925 ** When this function is called, pBuilder->pNew->nOut contains the 002926 ** number of rows expected to be visited by filtering using the nEq 002927 ** terms only. If it is modified, this value is restored before this 002928 ** function returns. 002929 ** 002930 ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is 002931 ** a fake index used for the INTEGER PRIMARY KEY. 002932 */ 002933 static int whereLoopAddBtreeIndex( 002934 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ 002935 SrcItem *pSrc, /* FROM clause term being analyzed */ 002936 Index *pProbe, /* An index on pSrc */ 002937 LogEst nInMul /* log(Number of iterations due to IN) */ 002938 ){ 002939 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */ 002940 Parse *pParse = pWInfo->pParse; /* Parsing context */ 002941 sqlite3 *db = pParse->db; /* Database connection malloc context */ 002942 WhereLoop *pNew; /* Template WhereLoop under construction */ 002943 WhereTerm *pTerm; /* A WhereTerm under consideration */ 002944 int opMask; /* Valid operators for constraints */ 002945 WhereScan scan; /* Iterator for WHERE terms */ 002946 Bitmask saved_prereq; /* Original value of pNew->prereq */ 002947 u16 saved_nLTerm; /* Original value of pNew->nLTerm */ 002948 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ 002949 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */ 002950 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */ 002951 u16 saved_nSkip; /* Original value of pNew->nSkip */ 002952 u32 saved_wsFlags; /* Original value of pNew->wsFlags */ 002953 LogEst saved_nOut; /* Original value of pNew->nOut */ 002954 int rc = SQLITE_OK; /* Return code */ 002955 LogEst rSize; /* Number of rows in the table */ 002956 LogEst rLogSize; /* Logarithm of table size */ 002957 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ 002958 002959 pNew = pBuilder->pNew; 002960 assert( db->mallocFailed==0 || pParse->nErr>0 ); 002961 if( pParse->nErr ){ 002962 return pParse->rc; 002963 } 002964 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n", 002965 pProbe->pTable->zName,pProbe->zName, 002966 pNew->u.btree.nEq, pNew->nSkip, pNew->rRun)); 002967 002968 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); 002969 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); 002970 if( pNew->wsFlags & WHERE_BTM_LIMIT ){ 002971 opMask = WO_LT|WO_LE; 002972 }else{ 002973 assert( pNew->u.btree.nBtm==0 ); 002974 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; 002975 } 002976 if( pProbe->bUnordered || pProbe->bLowQual ){ 002977 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); 002978 if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS); 002979 } 002980 002981 assert( pNew->u.btree.nEq<pProbe->nColumn ); 002982 assert( pNew->u.btree.nEq<pProbe->nKeyCol 002983 || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY ); 002984 002985 saved_nEq = pNew->u.btree.nEq; 002986 saved_nBtm = pNew->u.btree.nBtm; 002987 saved_nTop = pNew->u.btree.nTop; 002988 saved_nSkip = pNew->nSkip; 002989 saved_nLTerm = pNew->nLTerm; 002990 saved_wsFlags = pNew->wsFlags; 002991 saved_prereq = pNew->prereq; 002992 saved_nOut = pNew->nOut; 002993 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, 002994 opMask, pProbe); 002995 pNew->rSetup = 0; 002996 rSize = pProbe->aiRowLogEst[0]; 002997 rLogSize = estLog(rSize); 002998 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ 002999 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ 003000 LogEst rCostIdx; 003001 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ 003002 int nIn = 0; 003003 #ifdef SQLITE_ENABLE_STAT4 003004 int nRecValid = pBuilder->nRecValid; 003005 #endif 003006 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) 003007 && indexColumnNotNull(pProbe, saved_nEq) 003008 ){ 003009 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ 003010 } 003011 if( pTerm->prereqRight & pNew->maskSelf ) continue; 003012 003013 /* Do not allow the upper bound of a LIKE optimization range constraint 003014 ** to mix with a lower range bound from some other source */ 003015 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; 003016 003017 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 003018 && !constraintCompatibleWithOuterJoin(pTerm,pSrc) 003019 ){ 003020 continue; 003021 } 003022 if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ 003023 pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; 003024 }else{ 003025 pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED; 003026 } 003027 pNew->wsFlags = saved_wsFlags; 003028 pNew->u.btree.nEq = saved_nEq; 003029 pNew->u.btree.nBtm = saved_nBtm; 003030 pNew->u.btree.nTop = saved_nTop; 003031 pNew->nLTerm = saved_nLTerm; 003032 if( pNew->nLTerm>=pNew->nLSlot 003033 && whereLoopResize(db, pNew, pNew->nLTerm+1) 003034 ){ 003035 break; /* OOM while trying to enlarge the pNew->aLTerm array */ 003036 } 003037 pNew->aLTerm[pNew->nLTerm++] = pTerm; 003038 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; 003039 003040 assert( nInMul==0 003041 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 003042 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 003043 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 003044 ); 003045 003046 if( eOp & WO_IN ){ 003047 Expr *pExpr = pTerm->pExpr; 003048 if( ExprUseXSelect(pExpr) ){ 003049 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ 003050 int i; 003051 nIn = 46; assert( 46==sqlite3LogEst(25) ); 003052 003053 /* The expression may actually be of the form (x, y) IN (SELECT...). 003054 ** In this case there is a separate term for each of (x) and (y). 003055 ** However, the nIn multiplier should only be applied once, not once 003056 ** for each such term. The following loop checks that pTerm is the 003057 ** first such term in use, and sets nIn back to 0 if it is not. */ 003058 for(i=0; i<pNew->nLTerm-1; i++){ 003059 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; 003060 } 003061 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ 003062 /* "x IN (value, value, ...)" */ 003063 nIn = sqlite3LogEst(pExpr->x.pList->nExpr); 003064 } 003065 if( pProbe->hasStat1 && rLogSize>=10 ){ 003066 LogEst M, logK, x; 003067 /* Let: 003068 ** N = the total number of rows in the table 003069 ** K = the number of entries on the RHS of the IN operator 003070 ** M = the number of rows in the table that match terms to the 003071 ** to the left in the same index. If the IN operator is on 003072 ** the left-most index column, M==N. 003073 ** 003074 ** Given the definitions above, it is better to omit the IN operator 003075 ** from the index lookup and instead do a scan of the M elements, 003076 ** testing each scanned row against the IN operator separately, if: 003077 ** 003078 ** M*log(K) < K*log(N) 003079 ** 003080 ** Our estimates for M, K, and N might be inaccurate, so we build in 003081 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator 003082 ** with the index, as using an index has better worst-case behavior. 003083 ** If we do not have real sqlite_stat1 data, always prefer to use 003084 ** the index. Do not bother with this optimization on very small 003085 ** tables (less than 2 rows) as it is pointless in that case. 003086 */ 003087 M = pProbe->aiRowLogEst[saved_nEq]; 003088 logK = estLog(nIn); 003089 /* TUNING v----- 10 to bias toward indexed IN */ 003090 x = M + logK + 10 - (nIn + rLogSize); 003091 if( x>=0 ){ 003092 WHERETRACE(0x40, 003093 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) " 003094 "prefers indexed lookup\n", 003095 saved_nEq, M, logK, nIn, rLogSize, x)); 003096 }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){ 003097 WHERETRACE(0x40, 003098 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" 003099 " nInMul=%d) prefers skip-scan\n", 003100 saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); 003101 pNew->wsFlags |= WHERE_IN_SEEKSCAN; 003102 }else{ 003103 WHERETRACE(0x40, 003104 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" 003105 " nInMul=%d) prefers normal scan\n", 003106 saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); 003107 continue; 003108 } 003109 } 003110 pNew->wsFlags |= WHERE_COLUMN_IN; 003111 }else if( eOp & (WO_EQ|WO_IS) ){ 003112 int iCol = pProbe->aiColumn[saved_nEq]; 003113 pNew->wsFlags |= WHERE_COLUMN_EQ; 003114 assert( saved_nEq==pNew->u.btree.nEq ); 003115 if( iCol==XN_ROWID 003116 || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) 003117 ){ 003118 if( iCol==XN_ROWID || pProbe->uniqNotNull 003119 || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) 003120 ){ 003121 pNew->wsFlags |= WHERE_ONEROW; 003122 }else{ 003123 pNew->wsFlags |= WHERE_UNQ_WANTED; 003124 } 003125 } 003126 if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; 003127 }else if( eOp & WO_ISNULL ){ 003128 pNew->wsFlags |= WHERE_COLUMN_NULL; 003129 }else{ 003130 int nVecLen = whereRangeVectorLen( 003131 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm 003132 ); 003133 if( eOp & (WO_GT|WO_GE) ){ 003134 testcase( eOp & WO_GT ); 003135 testcase( eOp & WO_GE ); 003136 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; 003137 pNew->u.btree.nBtm = nVecLen; 003138 pBtm = pTerm; 003139 pTop = 0; 003140 if( pTerm->wtFlags & TERM_LIKEOPT ){ 003141 /* Range constraints that come from the LIKE optimization are 003142 ** always used in pairs. */ 003143 pTop = &pTerm[1]; 003144 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm ); 003145 assert( pTop->wtFlags & TERM_LIKEOPT ); 003146 assert( pTop->eOperator==WO_LT ); 003147 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ 003148 pNew->aLTerm[pNew->nLTerm++] = pTop; 003149 pNew->wsFlags |= WHERE_TOP_LIMIT; 003150 pNew->u.btree.nTop = 1; 003151 } 003152 }else{ 003153 assert( eOp & (WO_LT|WO_LE) ); 003154 testcase( eOp & WO_LT ); 003155 testcase( eOp & WO_LE ); 003156 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; 003157 pNew->u.btree.nTop = nVecLen; 003158 pTop = pTerm; 003159 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? 003160 pNew->aLTerm[pNew->nLTerm-2] : 0; 003161 } 003162 } 003163 003164 /* At this point pNew->nOut is set to the number of rows expected to 003165 ** be visited by the index scan before considering term pTerm, or the 003166 ** values of nIn and nInMul. In other words, assuming that all 003167 ** "x IN(...)" terms are replaced with "x = ?". This block updates 003168 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ 003169 assert( pNew->nOut==saved_nOut ); 003170 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ 003171 /* Adjust nOut using stat4 data. Or, if there is no stat4 003172 ** data, using some other estimate. */ 003173 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); 003174 }else{ 003175 int nEq = ++pNew->u.btree.nEq; 003176 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); 003177 003178 assert( pNew->nOut==saved_nOut ); 003179 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ 003180 assert( (eOp & WO_IN) || nIn==0 ); 003181 testcase( eOp & WO_IN ); 003182 pNew->nOut += pTerm->truthProb; 003183 pNew->nOut -= nIn; 003184 }else{ 003185 #ifdef SQLITE_ENABLE_STAT4 003186 tRowcnt nOut = 0; 003187 if( nInMul==0 003188 && pProbe->nSample 003189 && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol) 003190 && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr)) 003191 && OptimizationEnabled(db, SQLITE_Stat4) 003192 ){ 003193 Expr *pExpr = pTerm->pExpr; 003194 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ 003195 testcase( eOp & WO_EQ ); 003196 testcase( eOp & WO_IS ); 003197 testcase( eOp & WO_ISNULL ); 003198 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); 003199 }else{ 003200 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); 003201 } 003202 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; 003203 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ 003204 if( nOut ){ 003205 pNew->nOut = sqlite3LogEst(nOut); 003206 if( nEq==1 003207 /* TUNING: Mark terms as "low selectivity" if they seem likely 003208 ** to be true for half or more of the rows in the table. 003209 ** See tag-202002240-1 */ 003210 && pNew->nOut+10 > pProbe->aiRowLogEst[0] 003211 ){ 003212 #if WHERETRACE_ENABLED /* 0x01 */ 003213 if( sqlite3WhereTrace & 0x20 ){ 003214 sqlite3DebugPrintf( 003215 "STAT4 determines term has low selectivity:\n"); 003216 sqlite3WhereTermPrint(pTerm, 999); 003217 } 003218 #endif 003219 pTerm->wtFlags |= TERM_HIGHTRUTH; 003220 if( pTerm->wtFlags & TERM_HEURTRUTH ){ 003221 /* If the term has previously been used with an assumption of 003222 ** higher selectivity, then set the flag to rerun the 003223 ** loop computations. */ 003224 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS; 003225 } 003226 } 003227 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; 003228 pNew->nOut -= nIn; 003229 } 003230 } 003231 if( nOut==0 ) 003232 #endif 003233 { 003234 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); 003235 if( eOp & WO_ISNULL ){ 003236 /* TUNING: If there is no likelihood() value, assume that a 003237 ** "col IS NULL" expression matches twice as many rows 003238 ** as (col=?). */ 003239 pNew->nOut += 10; 003240 } 003241 } 003242 } 003243 } 003244 003245 /* Set rCostIdx to the cost of visiting selected rows in index. Add 003246 ** it to pNew->rRun, which is currently set to the cost of the index 003247 ** seek only. Then, if this is a non-covering index, add the cost of 003248 ** visiting the rows in the main table. */ 003249 assert( pSrc->pTab->szTabRow>0 ); 003250 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ 003251 /* The pProbe->szIdxRow is low for an IPK table since the interior 003252 ** pages are small. Thus szIdxRow gives a good estimate of seek cost. 003253 ** But the leaf pages are full-size, so pProbe->szIdxRow would badly 003254 ** under-estimate the scanning cost. */ 003255 rCostIdx = pNew->nOut + 16; 003256 }else{ 003257 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; 003258 } 003259 pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); 003260 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ 003261 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); 003262 } 003263 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); 003264 003265 nOutUnadjusted = pNew->nOut; 003266 pNew->rRun += nInMul + nIn; 003267 pNew->nOut += nInMul + nIn; 003268 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); 003269 rc = whereLoopInsert(pBuilder, pNew); 003270 003271 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ 003272 pNew->nOut = saved_nOut; 003273 }else{ 003274 pNew->nOut = nOutUnadjusted; 003275 } 003276 003277 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 003278 && pNew->u.btree.nEq<pProbe->nColumn 003279 && (pNew->u.btree.nEq<pProbe->nKeyCol || 003280 pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) 003281 ){ 003282 if( pNew->u.btree.nEq>3 ){ 003283 sqlite3ProgressCheck(pParse); 003284 } 003285 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); 003286 } 003287 pNew->nOut = saved_nOut; 003288 #ifdef SQLITE_ENABLE_STAT4 003289 pBuilder->nRecValid = nRecValid; 003290 #endif 003291 } 003292 pNew->prereq = saved_prereq; 003293 pNew->u.btree.nEq = saved_nEq; 003294 pNew->u.btree.nBtm = saved_nBtm; 003295 pNew->u.btree.nTop = saved_nTop; 003296 pNew->nSkip = saved_nSkip; 003297 pNew->wsFlags = saved_wsFlags; 003298 pNew->nOut = saved_nOut; 003299 pNew->nLTerm = saved_nLTerm; 003300 003301 /* Consider using a skip-scan if there are no WHERE clause constraints 003302 ** available for the left-most terms of the index, and if the average 003303 ** number of repeats in the left-most terms is at least 18. 003304 ** 003305 ** The magic number 18 is selected on the basis that scanning 17 rows 003306 ** is almost always quicker than an index seek (even though if the index 003307 ** contains fewer than 2^17 rows we assume otherwise in other parts of 003308 ** the code). And, even if it is not, it should not be too much slower. 003309 ** On the other hand, the extra seeks could end up being significantly 003310 ** more expensive. */ 003311 assert( 42==sqlite3LogEst(18) ); 003312 if( saved_nEq==saved_nSkip 003313 && saved_nEq+1<pProbe->nKeyCol 003314 && saved_nEq==pNew->nLTerm 003315 && pProbe->noSkipScan==0 003316 && pProbe->hasStat1!=0 003317 && OptimizationEnabled(db, SQLITE_SkipScan) 003318 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ 003319 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK 003320 ){ 003321 LogEst nIter; 003322 pNew->u.btree.nEq++; 003323 pNew->nSkip++; 003324 pNew->aLTerm[pNew->nLTerm++] = 0; 003325 pNew->wsFlags |= WHERE_SKIPSCAN; 003326 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; 003327 pNew->nOut -= nIter; 003328 /* TUNING: Because uncertainties in the estimates for skip-scan queries, 003329 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ 003330 nIter += 5; 003331 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); 003332 pNew->nOut = saved_nOut; 003333 pNew->u.btree.nEq = saved_nEq; 003334 pNew->nSkip = saved_nSkip; 003335 pNew->wsFlags = saved_wsFlags; 003336 } 003337 003338 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n", 003339 pProbe->pTable->zName, pProbe->zName, saved_nEq, rc)); 003340 return rc; 003341 } 003342 003343 /* 003344 ** Return True if it is possible that pIndex might be useful in 003345 ** implementing the ORDER BY clause in pBuilder. 003346 ** 003347 ** Return False if pBuilder does not contain an ORDER BY clause or 003348 ** if there is no way for pIndex to be useful in implementing that 003349 ** ORDER BY clause. 003350 */ 003351 static int indexMightHelpWithOrderBy( 003352 WhereLoopBuilder *pBuilder, 003353 Index *pIndex, 003354 int iCursor 003355 ){ 003356 ExprList *pOB; 003357 ExprList *aColExpr; 003358 int ii, jj; 003359 003360 if( pIndex->bUnordered ) return 0; 003361 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; 003362 for(ii=0; ii<pOB->nExpr; ii++){ 003363 Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); 003364 if( NEVER(pExpr==0) ) continue; 003365 if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ 003366 if( pExpr->iColumn<0 ) return 1; 003367 for(jj=0; jj<pIndex->nKeyCol; jj++){ 003368 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; 003369 } 003370 }else if( (aColExpr = pIndex->aColExpr)!=0 ){ 003371 for(jj=0; jj<pIndex->nKeyCol; jj++){ 003372 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; 003373 if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ 003374 return 1; 003375 } 003376 } 003377 } 003378 } 003379 return 0; 003380 } 003381 003382 /* Check to see if a partial index with pPartIndexWhere can be used 003383 ** in the current query. Return true if it can be and false if not. 003384 */ 003385 static int whereUsablePartialIndex( 003386 int iTab, /* The table for which we want an index */ 003387 u8 jointype, /* The JT_* flags on the join */ 003388 WhereClause *pWC, /* The WHERE clause of the query */ 003389 Expr *pWhere /* The WHERE clause from the partial index */ 003390 ){ 003391 int i; 003392 WhereTerm *pTerm; 003393 Parse *pParse; 003394 003395 if( jointype & JT_LTORJ ) return 0; 003396 pParse = pWC->pWInfo->pParse; 003397 while( pWhere->op==TK_AND ){ 003398 if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0; 003399 pWhere = pWhere->pRight; 003400 } 003401 if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; 003402 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ 003403 Expr *pExpr; 003404 pExpr = pTerm->pExpr; 003405 if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab) 003406 && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON)) 003407 && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) 003408 && (pTerm->wtFlags & TERM_VNULL)==0 003409 ){ 003410 return 1; 003411 } 003412 } 003413 return 0; 003414 } 003415 003416 /* 003417 ** pIdx is an index containing expressions. Check it see if any of the 003418 ** expressions in the index match the pExpr expression. 003419 */ 003420 static int exprIsCoveredByIndex( 003421 const Expr *pExpr, 003422 const Index *pIdx, 003423 int iTabCur 003424 ){ 003425 int i; 003426 for(i=0; i<pIdx->nColumn; i++){ 003427 if( pIdx->aiColumn[i]==XN_EXPR 003428 && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 003429 ){ 003430 return 1; 003431 } 003432 } 003433 return 0; 003434 } 003435 003436 /* 003437 ** Structure passed to the whereIsCoveringIndex Walker callback. 003438 */ 003439 typedef struct CoveringIndexCheck CoveringIndexCheck; 003440 struct CoveringIndexCheck { 003441 Index *pIdx; /* The index */ 003442 int iTabCur; /* Cursor number for the corresponding table */ 003443 u8 bExpr; /* Uses an indexed expression */ 003444 u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ 003445 }; 003446 003447 /* 003448 ** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. 003449 ** 003450 ** If the Expr node references the table with cursor pCk->iTabCur, then 003451 ** make sure that column is covered by the index pCk->pIdx. We know that 003452 ** all columns less than 63 (really BMS-1) are covered, so we don't need 003453 ** to check them. But we do need to check any column at 63 or greater. 003454 ** 003455 ** If the index does not cover the column, then set pWalk->eCode to 003456 ** non-zero and return WRC_Abort to stop the search. 003457 ** 003458 ** If this node does not disprove that the index can be a covering index, 003459 ** then just return WRC_Continue, to continue the search. 003460 ** 003461 ** If pCk->pIdx contains indexed expressions and one of those expressions 003462 ** matches pExpr, then prune the search. 003463 */ 003464 static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ 003465 int i; /* Loop counter */ 003466 const Index *pIdx; /* The index of interest */ 003467 const i16 *aiColumn; /* Columns contained in the index */ 003468 u16 nColumn; /* Number of columns in the index */ 003469 CoveringIndexCheck *pCk; /* Info about this search */ 003470 003471 pCk = pWalk->u.pCovIdxCk; 003472 pIdx = pCk->pIdx; 003473 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ 003474 /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ 003475 if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; 003476 pIdx = pWalk->u.pCovIdxCk->pIdx; 003477 aiColumn = pIdx->aiColumn; 003478 nColumn = pIdx->nColumn; 003479 for(i=0; i<nColumn; i++){ 003480 if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue; 003481 } 003482 pCk->bUnidx = 1; 003483 return WRC_Abort; 003484 }else if( pIdx->bHasExpr 003485 && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ 003486 pCk->bExpr = 1; 003487 return WRC_Prune; 003488 } 003489 return WRC_Continue; 003490 } 003491 003492 003493 /* 003494 ** pIdx is an index that covers all of the low-number columns used by 003495 ** pWInfo->pSelect (columns from 0 through 62) or an index that has 003496 ** expressions terms. Hence, we cannot determine whether or not it is 003497 ** a covering index by using the colUsed bitmasks. We have to do a search 003498 ** to see if the index is covering. This routine does that search. 003499 ** 003500 ** The return value is one of these: 003501 ** 003502 ** 0 The index is definitely not a covering index 003503 ** 003504 ** WHERE_IDX_ONLY The index is definitely a covering index 003505 ** 003506 ** WHERE_EXPRIDX The index is likely a covering index, but it is 003507 ** difficult to determine precisely because of the 003508 ** expressions that are indexed. Score it as a 003509 ** covering index, but still keep the main table open 003510 ** just in case we need it. 003511 ** 003512 ** This routine is an optimization. It is always safe to return zero. 003513 ** But returning one of the other two values when zero should have been 003514 ** returned can lead to incorrect bytecode and assertion faults. 003515 */ 003516 static SQLITE_NOINLINE u32 whereIsCoveringIndex( 003517 WhereInfo *pWInfo, /* The WHERE clause context */ 003518 Index *pIdx, /* Index that is being tested */ 003519 int iTabCur /* Cursor for the table being indexed */ 003520 ){ 003521 int i, rc; 003522 struct CoveringIndexCheck ck; 003523 Walker w; 003524 if( pWInfo->pSelect==0 ){ 003525 /* We don't have access to the full query, so we cannot check to see 003526 ** if pIdx is covering. Assume it is not. */ 003527 return 0; 003528 } 003529 if( pIdx->bHasExpr==0 ){ 003530 for(i=0; i<pIdx->nColumn; i++){ 003531 if( pIdx->aiColumn[i]>=BMS-1 ) break; 003532 } 003533 if( i>=pIdx->nColumn ){ 003534 /* pIdx does not index any columns greater than 62, but we know from 003535 ** colMask that columns greater than 62 are used, so this is not a 003536 ** covering index */ 003537 return 0; 003538 } 003539 } 003540 ck.pIdx = pIdx; 003541 ck.iTabCur = iTabCur; 003542 ck.bExpr = 0; 003543 ck.bUnidx = 0; 003544 memset(&w, 0, sizeof(w)); 003545 w.xExprCallback = whereIsCoveringIndexWalkCallback; 003546 w.xSelectCallback = sqlite3SelectWalkNoop; 003547 w.u.pCovIdxCk = &ck; 003548 sqlite3WalkSelect(&w, pWInfo->pSelect); 003549 if( ck.bUnidx ){ 003550 rc = 0; 003551 }else if( ck.bExpr ){ 003552 rc = WHERE_EXPRIDX; 003553 }else{ 003554 rc = WHERE_IDX_ONLY; 003555 } 003556 return rc; 003557 } 003558 003559 /* 003560 ** This is an sqlite3ParserAddCleanup() callback that is invoked to 003561 ** free the Parse->pIdxEpr list when the Parse object is destroyed. 003562 */ 003563 static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ 003564 IndexedExpr **pp = (IndexedExpr**)pObject; 003565 while( *pp!=0 ){ 003566 IndexedExpr *p = *pp; 003567 *pp = p->pIENext; 003568 sqlite3ExprDelete(db, p->pExpr); 003569 sqlite3DbFreeNN(db, p); 003570 } 003571 } 003572 003573 /* 003574 ** This function is called for a partial index - one with a WHERE clause - in 003575 ** two scenarios. In both cases, it determines whether or not the WHERE 003576 ** clause on the index implies that a column of the table may be safely 003577 ** replaced by a constant expression. For example, in the following 003578 ** SELECT: 003579 ** 003580 ** CREATE INDEX i1 ON t1(b, c) WHERE a=<expr>; 003581 ** SELECT a, b, c FROM t1 WHERE a=<expr> AND b=?; 003582 ** 003583 ** The "a" in the select-list may be replaced by <expr>, iff: 003584 ** 003585 ** (a) <expr> is a constant expression, and 003586 ** (b) The (a=<expr>) comparison uses the BINARY collation sequence, and 003587 ** (c) Column "a" has an affinity other than NONE or BLOB. 003588 ** 003589 ** If argument pItem is NULL, then pMask must not be NULL. In this case this 003590 ** function is being called as part of determining whether or not pIdx 003591 ** is a covering index. This function clears any bits in (*pMask) 003592 ** corresponding to columns that may be replaced by constants as described 003593 ** above. 003594 ** 003595 ** Otherwise, if pItem is not NULL, then this function is being called 003596 ** as part of coding a loop that uses index pIdx. In this case, add entries 003597 ** to the Parse.pIdxPartExpr list for each column that can be replaced 003598 ** by a constant. 003599 */ 003600 static void wherePartIdxExpr( 003601 Parse *pParse, /* Parse context */ 003602 Index *pIdx, /* Partial index being processed */ 003603 Expr *pPart, /* WHERE clause being processed */ 003604 Bitmask *pMask, /* Mask to clear bits in */ 003605 int iIdxCur, /* Cursor number for index */ 003606 SrcItem *pItem /* The FROM clause entry for the table */ 003607 ){ 003608 assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 ); 003609 assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) ); 003610 003611 if( pPart->op==TK_AND ){ 003612 wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem); 003613 pPart = pPart->pLeft; 003614 } 003615 003616 if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){ 003617 Expr *pLeft = pPart->pLeft; 003618 Expr *pRight = pPart->pRight; 003619 u8 aff; 003620 003621 if( pLeft->op!=TK_COLUMN ) return; 003622 if( !sqlite3ExprIsConstant(pRight) ) return; 003623 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; 003624 if( pLeft->iColumn<0 ) return; 003625 aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; 003626 if( aff>=SQLITE_AFF_TEXT ){ 003627 if( pItem ){ 003628 sqlite3 *db = pParse->db; 003629 IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p)); 003630 if( p ){ 003631 int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0; 003632 p->pExpr = sqlite3ExprDup(db, pRight, 0); 003633 p->iDataCur = pItem->iCursor; 003634 p->iIdxCur = iIdxCur; 003635 p->iIdxCol = pLeft->iColumn; 003636 p->bMaybeNullRow = bNullRow; 003637 p->pIENext = pParse->pIdxPartExpr; 003638 p->aff = aff; 003639 pParse->pIdxPartExpr = p; 003640 if( p->pIENext==0 ){ 003641 void *pArg = (void*)&pParse->pIdxPartExpr; 003642 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); 003643 } 003644 } 003645 }else if( pLeft->iColumn<(BMS-1) ){ 003646 *pMask &= ~((Bitmask)1 << pLeft->iColumn); 003647 } 003648 } 003649 } 003650 } 003651 003652 003653 /* 003654 ** Add all WhereLoop objects for a single table of the join where the table 003655 ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be 003656 ** a b-tree table, not a virtual table. 003657 ** 003658 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function 003659 ** are calculated as follows: 003660 ** 003661 ** For a full scan, assuming the table (or index) contains nRow rows: 003662 ** 003663 ** cost = nRow * 3.0 // full-table scan 003664 ** cost = nRow * K // scan of covering index 003665 ** cost = nRow * (K+3.0) // scan of non-covering index 003666 ** 003667 ** where K is a value between 1.1 and 3.0 set based on the relative 003668 ** estimated average size of the index and table records. 003669 ** 003670 ** For an index scan, where nVisit is the number of index rows visited 003671 ** by the scan, and nSeek is the number of seek operations required on 003672 ** the index b-tree: 003673 ** 003674 ** cost = nSeek * (log(nRow) + K * nVisit) // covering index 003675 ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index 003676 ** 003677 ** Normally, nSeek is 1. nSeek values greater than 1 come about if the 003678 ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when 003679 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. 003680 ** 003681 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount 003682 ** of uncertainty. For this reason, scoring is designed to pick plans that 003683 ** "do the least harm" if the estimates are inaccurate. For example, a 003684 ** log(nRow) factor is omitted from a non-covering index scan in order to 003685 ** bias the scoring in favor of using an index, since the worst-case 003686 ** performance of using an index is far better than the worst-case performance 003687 ** of a full table scan. 003688 */ 003689 static int whereLoopAddBtree( 003690 WhereLoopBuilder *pBuilder, /* WHERE clause information */ 003691 Bitmask mPrereq /* Extra prerequisites for using this table */ 003692 ){ 003693 WhereInfo *pWInfo; /* WHERE analysis context */ 003694 Index *pProbe; /* An index we are evaluating */ 003695 Index sPk; /* A fake index object for the primary key */ 003696 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ 003697 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ 003698 SrcList *pTabList; /* The FROM clause */ 003699 SrcItem *pSrc; /* The FROM clause btree term to add */ 003700 WhereLoop *pNew; /* Template WhereLoop object */ 003701 int rc = SQLITE_OK; /* Return code */ 003702 int iSortIdx = 1; /* Index number */ 003703 int b; /* A boolean value */ 003704 LogEst rSize; /* number of rows in the table */ 003705 WhereClause *pWC; /* The parsed WHERE clause */ 003706 Table *pTab; /* Table being queried */ 003707 003708 pNew = pBuilder->pNew; 003709 pWInfo = pBuilder->pWInfo; 003710 pTabList = pWInfo->pTabList; 003711 pSrc = pTabList->a + pNew->iTab; 003712 pTab = pSrc->pTab; 003713 pWC = pBuilder->pWC; 003714 assert( !IsVirtual(pSrc->pTab) ); 003715 003716 if( pSrc->fg.isIndexedBy ){ 003717 assert( pSrc->fg.isCte==0 ); 003718 /* An INDEXED BY clause specifies a particular index to use */ 003719 pProbe = pSrc->u2.pIBIndex; 003720 }else if( !HasRowid(pTab) ){ 003721 pProbe = pTab->pIndex; 003722 }else{ 003723 /* There is no INDEXED BY clause. Create a fake Index object in local 003724 ** variable sPk to represent the rowid primary key index. Make this 003725 ** fake index the first in a chain of Index objects with all of the real 003726 ** indices to follow */ 003727 Index *pFirst; /* First of real indices on the table */ 003728 memset(&sPk, 0, sizeof(Index)); 003729 sPk.nKeyCol = 1; 003730 sPk.nColumn = 1; 003731 sPk.aiColumn = &aiColumnPk; 003732 sPk.aiRowLogEst = aiRowEstPk; 003733 sPk.onError = OE_Replace; 003734 sPk.pTable = pTab; 003735 sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */ 003736 sPk.idxType = SQLITE_IDXTYPE_IPK; 003737 aiRowEstPk[0] = pTab->nRowLogEst; 003738 aiRowEstPk[1] = 0; 003739 pFirst = pSrc->pTab->pIndex; 003740 if( pSrc->fg.notIndexed==0 ){ 003741 /* The real indices of the table are only considered if the 003742 ** NOT INDEXED qualifier is omitted from the FROM clause */ 003743 sPk.pNext = pFirst; 003744 } 003745 pProbe = &sPk; 003746 } 003747 rSize = pTab->nRowLogEst; 003748 003749 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX 003750 /* Automatic indexes */ 003751 if( !pBuilder->pOrSet /* Not part of an OR optimization */ 003752 && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0 003753 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 003754 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ 003755 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ 003756 && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ 003757 && !pSrc->fg.isCorrelated /* Not a correlated subquery */ 003758 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ 003759 && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */ 003760 ){ 003761 /* Generate auto-index WhereLoops */ 003762 LogEst rLogSize; /* Logarithm of the number of rows in the table */ 003763 WhereTerm *pTerm; 003764 WhereTerm *pWCEnd = pWC->a + pWC->nTerm; 003765 rLogSize = estLog(rSize); 003766 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){ 003767 if( pTerm->prereqRight & pNew->maskSelf ) continue; 003768 if( termCanDriveIndex(pTerm, pSrc, 0) ){ 003769 pNew->u.btree.nEq = 1; 003770 pNew->nSkip = 0; 003771 pNew->u.btree.pIndex = 0; 003772 pNew->nLTerm = 1; 003773 pNew->aLTerm[0] = pTerm; 003774 /* TUNING: One-time cost for computing the automatic index is 003775 ** estimated to be X*N*log2(N) where N is the number of rows in 003776 ** the table being indexed and where X is 7 (LogEst=28) for normal 003777 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value 003778 ** of X is smaller for views and subqueries so that the query planner 003779 ** will be more aggressive about generating automatic indexes for 003780 ** those objects, since there is no opportunity to add schema 003781 ** indexes on subqueries and views. */ 003782 pNew->rSetup = rLogSize + rSize; 003783 if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){ 003784 pNew->rSetup += 28; 003785 }else{ 003786 pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes 003787 ** on ephemeral materializations of views */ 003788 } 003789 ApplyCostMultiplier(pNew->rSetup, pTab->costMult); 003790 if( pNew->rSetup<0 ) pNew->rSetup = 0; 003791 /* TUNING: Each index lookup yields 20 rows in the table. This 003792 ** is more than the usual guess of 10 rows, since we have no way 003793 ** of knowing how selective the index will ultimately be. It would 003794 ** not be unreasonable to make this value much larger. */ 003795 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); 003796 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); 003797 pNew->wsFlags = WHERE_AUTO_INDEX; 003798 pNew->prereq = mPrereq | pTerm->prereqRight; 003799 rc = whereLoopInsert(pBuilder, pNew); 003800 } 003801 } 003802 } 003803 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ 003804 003805 /* Loop over all indices. If there was an INDEXED BY clause, then only 003806 ** consider index pProbe. */ 003807 for(; rc==SQLITE_OK && pProbe; 003808 pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ 003809 ){ 003810 if( pProbe->pPartIdxWhere!=0 003811 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC, 003812 pProbe->pPartIdxWhere) 003813 ){ 003814 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ 003815 continue; /* Partial index inappropriate for this query */ 003816 } 003817 if( pProbe->bNoQuery ) continue; 003818 rSize = pProbe->aiRowLogEst[0]; 003819 pNew->u.btree.nEq = 0; 003820 pNew->u.btree.nBtm = 0; 003821 pNew->u.btree.nTop = 0; 003822 pNew->nSkip = 0; 003823 pNew->nLTerm = 0; 003824 pNew->iSortIdx = 0; 003825 pNew->rSetup = 0; 003826 pNew->prereq = mPrereq; 003827 pNew->nOut = rSize; 003828 pNew->u.btree.pIndex = pProbe; 003829 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); 003830 003831 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ 003832 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); 003833 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ 003834 /* Integer primary key index */ 003835 pNew->wsFlags = WHERE_IPK; 003836 003837 /* Full table scan */ 003838 pNew->iSortIdx = b ? iSortIdx : 0; 003839 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an 003840 ** extra cost designed to discourage the use of full table scans, 003841 ** since index lookups have better worst-case performance if our 003842 ** stat guesses are wrong. Reduce the 3.0 penalty slightly 003843 ** (to 2.75) if we have valid STAT4 information for the table. 003844 ** At 2.75, a full table scan is preferred over using an index on 003845 ** a column with just two distinct values where each value has about 003846 ** an equal number of appearances. Without STAT4 data, we still want 003847 ** to use an index in that case, since the constraint might be for 003848 ** the scarcer of the two values, and in that case an index lookup is 003849 ** better. 003850 */ 003851 #ifdef SQLITE_ENABLE_STAT4 003852 pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0); 003853 #else 003854 pNew->rRun = rSize + 16; 003855 #endif 003856 ApplyCostMultiplier(pNew->rRun, pTab->costMult); 003857 whereLoopOutputAdjust(pWC, pNew, rSize); 003858 rc = whereLoopInsert(pBuilder, pNew); 003859 pNew->nOut = rSize; 003860 if( rc ) break; 003861 }else{ 003862 Bitmask m; 003863 if( pProbe->isCovering ){ 003864 m = 0; 003865 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; 003866 }else{ 003867 m = pSrc->colUsed & pProbe->colNotIdxed; 003868 if( pProbe->pPartIdxWhere ){ 003869 wherePartIdxExpr( 003870 pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0 003871 ); 003872 } 003873 pNew->wsFlags = WHERE_INDEXED; 003874 if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ 003875 u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); 003876 if( isCov==0 ){ 003877 WHERETRACE(0x200, 003878 ("-> %s is not a covering index" 003879 " according to whereIsCoveringIndex()\n", pProbe->zName)); 003880 assert( m!=0 ); 003881 }else{ 003882 m = 0; 003883 pNew->wsFlags |= isCov; 003884 if( isCov & WHERE_IDX_ONLY ){ 003885 WHERETRACE(0x200, 003886 ("-> %s is a covering expression index" 003887 " according to whereIsCoveringIndex()\n", pProbe->zName)); 003888 }else{ 003889 assert( isCov==WHERE_EXPRIDX ); 003890 WHERETRACE(0x200, 003891 ("-> %s might be a covering expression index" 003892 " according to whereIsCoveringIndex()\n", pProbe->zName)); 003893 } 003894 } 003895 }else if( m==0 ){ 003896 WHERETRACE(0x200, 003897 ("-> %s a covering index according to bitmasks\n", 003898 pProbe->zName, m==0 ? "is" : "is not")); 003899 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; 003900 } 003901 } 003902 003903 /* Full scan via index */ 003904 if( b 003905 || !HasRowid(pTab) 003906 || pProbe->pPartIdxWhere!=0 003907 || pSrc->fg.isIndexedBy 003908 || ( m==0 003909 && pProbe->bUnordered==0 003910 && (pProbe->szIdxRow<pTab->szTabRow) 003911 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 003912 && sqlite3GlobalConfig.bUseCis 003913 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) 003914 ) 003915 ){ 003916 pNew->iSortIdx = b ? iSortIdx : 0; 003917 003918 /* The cost of visiting the index rows is N*K, where K is 003919 ** between 1.1 and 3.0, depending on the relative sizes of the 003920 ** index and table rows. */ 003921 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; 003922 if( m!=0 ){ 003923 /* If this is a non-covering index scan, add in the cost of 003924 ** doing table lookups. The cost will be 3x the number of 003925 ** lookups. Take into account WHERE clause terms that can be 003926 ** satisfied using just the index, and that do not require a 003927 ** table lookup. */ 003928 LogEst nLookup = rSize + 16; /* Base cost: N*3 */ 003929 int ii; 003930 int iCur = pSrc->iCursor; 003931 WhereClause *pWC2 = &pWInfo->sWC; 003932 for(ii=0; ii<pWC2->nTerm; ii++){ 003933 WhereTerm *pTerm = &pWC2->a[ii]; 003934 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){ 003935 break; 003936 } 003937 /* pTerm can be evaluated using just the index. So reduce 003938 ** the expected number of table lookups accordingly */ 003939 if( pTerm->truthProb<=0 ){ 003940 nLookup += pTerm->truthProb; 003941 }else{ 003942 nLookup--; 003943 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19; 003944 } 003945 } 003946 003947 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup); 003948 } 003949 ApplyCostMultiplier(pNew->rRun, pTab->costMult); 003950 whereLoopOutputAdjust(pWC, pNew, rSize); 003951 if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){ 003952 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN 003953 ** because the cursor used to access the index might not be 003954 ** positioned to the correct row during the right-join no-match 003955 ** loop. */ 003956 }else{ 003957 rc = whereLoopInsert(pBuilder, pNew); 003958 } 003959 pNew->nOut = rSize; 003960 if( rc ) break; 003961 } 003962 } 003963 003964 pBuilder->bldFlags1 = 0; 003965 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); 003966 if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){ 003967 /* If a non-unique index is used, or if a prefix of the key for 003968 ** unique index is used (making the index functionally non-unique) 003969 ** then the sqlite_stat1 data becomes important for scoring the 003970 ** plan */ 003971 pTab->tabFlags |= TF_StatsUsed; 003972 } 003973 #ifdef SQLITE_ENABLE_STAT4 003974 sqlite3Stat4ProbeFree(pBuilder->pRec); 003975 pBuilder->nRecValid = 0; 003976 pBuilder->pRec = 0; 003977 #endif 003978 } 003979 return rc; 003980 } 003981 003982 #ifndef SQLITE_OMIT_VIRTUALTABLE 003983 003984 /* 003985 ** Return true if pTerm is a virtual table LIMIT or OFFSET term. 003986 */ 003987 static int isLimitTerm(WhereTerm *pTerm){ 003988 assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 ); 003989 return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT 003990 && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET; 003991 } 003992 003993 /* 003994 ** Argument pIdxInfo is already populated with all constraints that may 003995 ** be used by the virtual table identified by pBuilder->pNew->iTab. This 003996 ** function marks a subset of those constraints usable, invokes the 003997 ** xBestIndex method and adds the returned plan to pBuilder. 003998 ** 003999 ** A constraint is marked usable if: 004000 ** 004001 ** * Argument mUsable indicates that its prerequisites are available, and 004002 ** 004003 ** * It is not one of the operators specified in the mExclude mask passed 004004 ** as the fourth argument (which in practice is either WO_IN or 0). 004005 ** 004006 ** Argument mPrereq is a mask of tables that must be scanned before the 004007 ** virtual table in question. These are added to the plans prerequisites 004008 ** before it is added to pBuilder. 004009 ** 004010 ** Output parameter *pbIn is set to true if the plan added to pBuilder 004011 ** uses one or more WO_IN terms, or false otherwise. 004012 */ 004013 static int whereLoopAddVirtualOne( 004014 WhereLoopBuilder *pBuilder, 004015 Bitmask mPrereq, /* Mask of tables that must be used. */ 004016 Bitmask mUsable, /* Mask of usable tables */ 004017 u16 mExclude, /* Exclude terms using these operators */ 004018 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ 004019 u16 mNoOmit, /* Do not omit these constraints */ 004020 int *pbIn, /* OUT: True if plan uses an IN(...) op */ 004021 int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */ 004022 ){ 004023 WhereClause *pWC = pBuilder->pWC; 004024 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; 004025 struct sqlite3_index_constraint *pIdxCons; 004026 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; 004027 int i; 004028 int mxTerm; 004029 int rc = SQLITE_OK; 004030 WhereLoop *pNew = pBuilder->pNew; 004031 Parse *pParse = pBuilder->pWInfo->pParse; 004032 SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; 004033 int nConstraint = pIdxInfo->nConstraint; 004034 004035 assert( (mUsable & mPrereq)==mPrereq ); 004036 *pbIn = 0; 004037 pNew->prereq = mPrereq; 004038 004039 /* Set the usable flag on the subset of constraints identified by 004040 ** arguments mUsable and mExclude. */ 004041 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; 004042 for(i=0; i<nConstraint; i++, pIdxCons++){ 004043 WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset]; 004044 pIdxCons->usable = 0; 004045 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight 004046 && (pTerm->eOperator & mExclude)==0 004047 && (pbRetryLimit || !isLimitTerm(pTerm)) 004048 ){ 004049 pIdxCons->usable = 1; 004050 } 004051 } 004052 004053 /* Initialize the output fields of the sqlite3_index_info structure */ 004054 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint); 004055 assert( pIdxInfo->needToFreeIdxStr==0 ); 004056 pIdxInfo->idxStr = 0; 004057 pIdxInfo->idxNum = 0; 004058 pIdxInfo->orderByConsumed = 0; 004059 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; 004060 pIdxInfo->estimatedRows = 25; 004061 pIdxInfo->idxFlags = 0; 004062 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; 004063 pHidden->mHandleIn = 0; 004064 004065 /* Invoke the virtual table xBestIndex() method */ 004066 rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); 004067 if( rc ){ 004068 if( rc==SQLITE_CONSTRAINT ){ 004069 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means 004070 ** that the particular combination of parameters provided is unusable. 004071 ** Make no entries in the loop table. 004072 */ 004073 WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); 004074 return SQLITE_OK; 004075 } 004076 return rc; 004077 } 004078 004079 mxTerm = -1; 004080 assert( pNew->nLSlot>=nConstraint ); 004081 memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint ); 004082 memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab)); 004083 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; 004084 for(i=0; i<nConstraint; i++, pIdxCons++){ 004085 int iTerm; 004086 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ 004087 WhereTerm *pTerm; 004088 int j = pIdxCons->iTermOffset; 004089 if( iTerm>=nConstraint 004090 || j<0 004091 || j>=pWC->nTerm 004092 || pNew->aLTerm[iTerm]!=0 004093 || pIdxCons->usable==0 004094 ){ 004095 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); 004096 testcase( pIdxInfo->needToFreeIdxStr ); 004097 return SQLITE_ERROR; 004098 } 004099 testcase( iTerm==nConstraint-1 ); 004100 testcase( j==0 ); 004101 testcase( j==pWC->nTerm-1 ); 004102 pTerm = &pWC->a[j]; 004103 pNew->prereq |= pTerm->prereqRight; 004104 assert( iTerm<pNew->nLSlot ); 004105 pNew->aLTerm[iTerm] = pTerm; 004106 if( iTerm>mxTerm ) mxTerm = iTerm; 004107 testcase( iTerm==15 ); 004108 testcase( iTerm==16 ); 004109 if( pUsage[i].omit ){ 004110 if( i<16 && ((1<<i)&mNoOmit)==0 ){ 004111 testcase( i!=iTerm ); 004112 pNew->u.vtab.omitMask |= 1<<iTerm; 004113 }else{ 004114 testcase( i!=iTerm ); 004115 } 004116 if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){ 004117 pNew->u.vtab.bOmitOffset = 1; 004118 } 004119 } 004120 if( SMASKBIT32(i) & pHidden->mHandleIn ){ 004121 pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm); 004122 }else if( (pTerm->eOperator & WO_IN)!=0 ){ 004123 /* A virtual table that is constrained by an IN clause may not 004124 ** consume the ORDER BY clause because (1) the order of IN terms 004125 ** is not necessarily related to the order of output terms and 004126 ** (2) Multiple outputs from a single IN value will not merge 004127 ** together. */ 004128 pIdxInfo->orderByConsumed = 0; 004129 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; 004130 *pbIn = 1; assert( (mExclude & WO_IN)==0 ); 004131 } 004132 004133 assert( pbRetryLimit || !isLimitTerm(pTerm) ); 004134 if( isLimitTerm(pTerm) && *pbIn ){ 004135 /* If there is an IN(...) term handled as an == (separate call to 004136 ** xFilter for each value on the RHS of the IN) and a LIMIT or 004137 ** OFFSET term handled as well, the plan is unusable. Set output 004138 ** variable *pbRetryLimit to true to tell the caller to retry with 004139 ** LIMIT and OFFSET disabled. */ 004140 if( pIdxInfo->needToFreeIdxStr ){ 004141 sqlite3_free(pIdxInfo->idxStr); 004142 pIdxInfo->idxStr = 0; 004143 pIdxInfo->needToFreeIdxStr = 0; 004144 } 004145 *pbRetryLimit = 1; 004146 return SQLITE_OK; 004147 } 004148 } 004149 } 004150 004151 pNew->nLTerm = mxTerm+1; 004152 for(i=0; i<=mxTerm; i++){ 004153 if( pNew->aLTerm[i]==0 ){ 004154 /* The non-zero argvIdx values must be contiguous. Raise an 004155 ** error if they are not */ 004156 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); 004157 testcase( pIdxInfo->needToFreeIdxStr ); 004158 return SQLITE_ERROR; 004159 } 004160 } 004161 assert( pNew->nLTerm<=pNew->nLSlot ); 004162 pNew->u.vtab.idxNum = pIdxInfo->idxNum; 004163 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; 004164 pIdxInfo->needToFreeIdxStr = 0; 004165 pNew->u.vtab.idxStr = pIdxInfo->idxStr; 004166 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? 004167 pIdxInfo->nOrderBy : 0); 004168 pNew->rSetup = 0; 004169 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); 004170 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); 004171 004172 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated 004173 ** that the scan will visit at most one row. Clear it otherwise. */ 004174 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ 004175 pNew->wsFlags |= WHERE_ONEROW; 004176 }else{ 004177 pNew->wsFlags &= ~WHERE_ONEROW; 004178 } 004179 rc = whereLoopInsert(pBuilder, pNew); 004180 if( pNew->u.vtab.needFree ){ 004181 sqlite3_free(pNew->u.vtab.idxStr); 004182 pNew->u.vtab.needFree = 0; 004183 } 004184 WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", 004185 *pbIn, (sqlite3_uint64)mPrereq, 004186 (sqlite3_uint64)(pNew->prereq & ~mPrereq))); 004187 004188 return rc; 004189 } 004190 004191 /* 004192 ** Return the collating sequence for a constraint passed into xBestIndex. 004193 ** 004194 ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex. 004195 ** This routine depends on there being a HiddenIndexInfo structure immediately 004196 ** following the sqlite3_index_info structure. 004197 ** 004198 ** Return a pointer to the collation name: 004199 ** 004200 ** 1. If there is an explicit COLLATE operator on the constraint, return it. 004201 ** 004202 ** 2. Else, if the column has an alternative collation, return that. 004203 ** 004204 ** 3. Otherwise, return "BINARY". 004205 */ 004206 const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){ 004207 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; 004208 const char *zRet = 0; 004209 if( iCons>=0 && iCons<pIdxInfo->nConstraint ){ 004210 CollSeq *pC = 0; 004211 int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset; 004212 Expr *pX = pHidden->pWC->a[iTerm].pExpr; 004213 if( pX->pLeft ){ 004214 pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX); 004215 } 004216 zRet = (pC ? pC->zName : sqlite3StrBINARY); 004217 } 004218 return zRet; 004219 } 004220 004221 /* 004222 ** Return true if constraint iCons is really an IN(...) constraint, or 004223 ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0) 004224 ** or clear (if bHandle==0) the flag to handle it using an iterator. 004225 */ 004226 int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){ 004227 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; 004228 u32 m = SMASKBIT32(iCons); 004229 if( m & pHidden->mIn ){ 004230 if( bHandle==0 ){ 004231 pHidden->mHandleIn &= ~m; 004232 }else if( bHandle>0 ){ 004233 pHidden->mHandleIn |= m; 004234 } 004235 return 1; 004236 } 004237 return 0; 004238 } 004239 004240 /* 004241 ** This interface is callable from within the xBestIndex callback only. 004242 ** 004243 ** If possible, set (*ppVal) to point to an object containing the value 004244 ** on the right-hand-side of constraint iCons. 004245 */ 004246 int sqlite3_vtab_rhs_value( 004247 sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */ 004248 int iCons, /* Constraint for which RHS is wanted */ 004249 sqlite3_value **ppVal /* Write value extracted here */ 004250 ){ 004251 HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1]; 004252 sqlite3_value *pVal = 0; 004253 int rc = SQLITE_OK; 004254 if( iCons<0 || iCons>=pIdxInfo->nConstraint ){ 004255 rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */ 004256 }else{ 004257 if( pH->aRhs[iCons]==0 ){ 004258 WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset]; 004259 rc = sqlite3ValueFromExpr( 004260 pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db), 004261 SQLITE_AFF_BLOB, &pH->aRhs[iCons] 004262 ); 004263 testcase( rc!=SQLITE_OK ); 004264 } 004265 pVal = pH->aRhs[iCons]; 004266 } 004267 *ppVal = pVal; 004268 004269 if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */ 004270 rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */ 004271 } 004272 004273 return rc; 004274 } 004275 004276 /* 004277 ** Return true if ORDER BY clause may be handled as DISTINCT. 004278 */ 004279 int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ 004280 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; 004281 assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 ); 004282 return pHidden->eDistinct; 004283 } 004284 004285 /* 004286 ** Cause the prepared statement that is associated with a call to 004287 ** xBestIndex to potentially use all schemas. If the statement being 004288 ** prepared is read-only, then just start read transactions on all 004289 ** schemas. But if this is a write operation, start writes on all 004290 ** schemas. 004291 ** 004292 ** This is used by the (built-in) sqlite_dbpage virtual table. 004293 */ 004294 void sqlite3VtabUsesAllSchemas(Parse *pParse){ 004295 int nDb = pParse->db->nDb; 004296 int i; 004297 for(i=0; i<nDb; i++){ 004298 sqlite3CodeVerifySchema(pParse, i); 004299 } 004300 if( DbMaskNonZero(pParse->writeMask) ){ 004301 for(i=0; i<nDb; i++){ 004302 sqlite3BeginWriteOperation(pParse, 0, i); 004303 } 004304 } 004305 } 004306 004307 /* 004308 ** Add all WhereLoop objects for a table of the join identified by 004309 ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. 004310 ** 004311 ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and 004312 ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause 004313 ** entries that occur before the virtual table in the FROM clause and are 004314 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the 004315 ** mUnusable mask contains all FROM clause entries that occur after the 004316 ** virtual table and are separated from it by at least one LEFT or 004317 ** CROSS JOIN. 004318 ** 004319 ** For example, if the query were: 004320 ** 004321 ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; 004322 ** 004323 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6). 004324 ** 004325 ** All the tables in mPrereq must be scanned before the current virtual 004326 ** table. So any terms for which all prerequisites are satisfied by 004327 ** mPrereq may be specified as "usable" in all calls to xBestIndex. 004328 ** Conversely, all tables in mUnusable must be scanned after the current 004329 ** virtual table, so any terms for which the prerequisites overlap with 004330 ** mUnusable should always be configured as "not-usable" for xBestIndex. 004331 */ 004332 static int whereLoopAddVirtual( 004333 WhereLoopBuilder *pBuilder, /* WHERE clause information */ 004334 Bitmask mPrereq, /* Tables that must be scanned before this one */ 004335 Bitmask mUnusable /* Tables that must be scanned after this one */ 004336 ){ 004337 int rc = SQLITE_OK; /* Return code */ 004338 WhereInfo *pWInfo; /* WHERE analysis context */ 004339 Parse *pParse; /* The parsing context */ 004340 WhereClause *pWC; /* The WHERE clause */ 004341 SrcItem *pSrc; /* The FROM clause term to search */ 004342 sqlite3_index_info *p; /* Object to pass to xBestIndex() */ 004343 int nConstraint; /* Number of constraints in p */ 004344 int bIn; /* True if plan uses IN(...) operator */ 004345 WhereLoop *pNew; 004346 Bitmask mBest; /* Tables used by best possible plan */ 004347 u16 mNoOmit; 004348 int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */ 004349 004350 assert( (mPrereq & mUnusable)==0 ); 004351 pWInfo = pBuilder->pWInfo; 004352 pParse = pWInfo->pParse; 004353 pWC = pBuilder->pWC; 004354 pNew = pBuilder->pNew; 004355 pSrc = &pWInfo->pTabList->a[pNew->iTab]; 004356 assert( IsVirtual(pSrc->pTab) ); 004357 p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); 004358 if( p==0 ) return SQLITE_NOMEM_BKPT; 004359 pNew->rSetup = 0; 004360 pNew->wsFlags = WHERE_VIRTUALTABLE; 004361 pNew->nLTerm = 0; 004362 pNew->u.vtab.needFree = 0; 004363 nConstraint = p->nConstraint; 004364 if( whereLoopResize(pParse->db, pNew, nConstraint) ){ 004365 freeIndexInfo(pParse->db, p); 004366 return SQLITE_NOMEM_BKPT; 004367 } 004368 004369 /* First call xBestIndex() with all constraints usable. */ 004370 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); 004371 WHERETRACE(0x800, (" VirtualOne: all usable\n")); 004372 rc = whereLoopAddVirtualOne( 004373 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry 004374 ); 004375 if( bRetry ){ 004376 assert( rc==SQLITE_OK ); 004377 rc = whereLoopAddVirtualOne( 004378 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0 004379 ); 004380 } 004381 004382 /* If the call to xBestIndex() with all terms enabled produced a plan 004383 ** that does not require any source tables (IOW: a plan with mBest==0) 004384 ** and does not use an IN(...) operator, then there is no point in making 004385 ** any further calls to xBestIndex() since they will all return the same 004386 ** result (if the xBestIndex() implementation is sane). */ 004387 if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){ 004388 int seenZero = 0; /* True if a plan with no prereqs seen */ 004389 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ 004390 Bitmask mPrev = 0; 004391 Bitmask mBestNoIn = 0; 004392 004393 /* If the plan produced by the earlier call uses an IN(...) term, call 004394 ** xBestIndex again, this time with IN(...) terms disabled. */ 004395 if( bIn ){ 004396 WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n")); 004397 rc = whereLoopAddVirtualOne( 004398 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0); 004399 assert( bIn==0 ); 004400 mBestNoIn = pNew->prereq & ~mPrereq; 004401 if( mBestNoIn==0 ){ 004402 seenZero = 1; 004403 seenZeroNoIN = 1; 004404 } 004405 } 004406 004407 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) 004408 ** in the set of terms that apply to the current virtual table. */ 004409 while( rc==SQLITE_OK ){ 004410 int i; 004411 Bitmask mNext = ALLBITS; 004412 assert( mNext>0 ); 004413 for(i=0; i<nConstraint; i++){ 004414 Bitmask mThis = ( 004415 pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq 004416 ); 004417 if( mThis>mPrev && mThis<mNext ) mNext = mThis; 004418 } 004419 mPrev = mNext; 004420 if( mNext==ALLBITS ) break; 004421 if( mNext==mBest || mNext==mBestNoIn ) continue; 004422 WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", 004423 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext)); 004424 rc = whereLoopAddVirtualOne( 004425 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0); 004426 if( pNew->prereq==mPrereq ){ 004427 seenZero = 1; 004428 if( bIn==0 ) seenZeroNoIN = 1; 004429 } 004430 } 004431 004432 /* If the calls to xBestIndex() in the above loop did not find a plan 004433 ** that requires no source tables at all (i.e. one guaranteed to be 004434 ** usable), make a call here with all source tables disabled */ 004435 if( rc==SQLITE_OK && seenZero==0 ){ 004436 WHERETRACE(0x800, (" VirtualOne: all disabled\n")); 004437 rc = whereLoopAddVirtualOne( 004438 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0); 004439 if( bIn==0 ) seenZeroNoIN = 1; 004440 } 004441 004442 /* If the calls to xBestIndex() have so far failed to find a plan 004443 ** that requires no source tables at all and does not use an IN(...) 004444 ** operator, make a final call to obtain one here. */ 004445 if( rc==SQLITE_OK && seenZeroNoIN==0 ){ 004446 WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n")); 004447 rc = whereLoopAddVirtualOne( 004448 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0); 004449 } 004450 } 004451 004452 if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); 004453 freeIndexInfo(pParse->db, p); 004454 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc)); 004455 return rc; 004456 } 004457 #endif /* SQLITE_OMIT_VIRTUALTABLE */ 004458 004459 /* 004460 ** Add WhereLoop entries to handle OR terms. This works for either 004461 ** btrees or virtual tables. 004462 */ 004463 static int whereLoopAddOr( 004464 WhereLoopBuilder *pBuilder, 004465 Bitmask mPrereq, 004466 Bitmask mUnusable 004467 ){ 004468 WhereInfo *pWInfo = pBuilder->pWInfo; 004469 WhereClause *pWC; 004470 WhereLoop *pNew; 004471 WhereTerm *pTerm, *pWCEnd; 004472 int rc = SQLITE_OK; 004473 int iCur; 004474 WhereClause tempWC; 004475 WhereLoopBuilder sSubBuild; 004476 WhereOrSet sSum, sCur; 004477 SrcItem *pItem; 004478 004479 pWC = pBuilder->pWC; 004480 pWCEnd = pWC->a + pWC->nTerm; 004481 pNew = pBuilder->pNew; 004482 memset(&sSum, 0, sizeof(sSum)); 004483 pItem = pWInfo->pTabList->a + pNew->iTab; 004484 iCur = pItem->iCursor; 004485 004486 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */ 004487 if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK; 004488 004489 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){ 004490 if( (pTerm->eOperator & WO_OR)!=0 004491 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 004492 ){ 004493 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; 004494 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; 004495 WhereTerm *pOrTerm; 004496 int once = 1; 004497 int i, j; 004498 004499 sSubBuild = *pBuilder; 004500 sSubBuild.pOrSet = &sCur; 004501 004502 WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm)); 004503 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){ 004504 if( (pOrTerm->eOperator & WO_AND)!=0 ){ 004505 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; 004506 }else if( pOrTerm->leftCursor==iCur ){ 004507 tempWC.pWInfo = pWC->pWInfo; 004508 tempWC.pOuter = pWC; 004509 tempWC.op = TK_AND; 004510 tempWC.nTerm = 1; 004511 tempWC.nBase = 1; 004512 tempWC.a = pOrTerm; 004513 sSubBuild.pWC = &tempWC; 004514 }else{ 004515 continue; 004516 } 004517 sCur.n = 0; 004518 #ifdef WHERETRACE_ENABLED 004519 WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n", 004520 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); 004521 if( sqlite3WhereTrace & 0x20000 ){ 004522 sqlite3WhereClausePrint(sSubBuild.pWC); 004523 } 004524 #endif 004525 #ifndef SQLITE_OMIT_VIRTUALTABLE 004526 if( IsVirtual(pItem->pTab) ){ 004527 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); 004528 }else 004529 #endif 004530 { 004531 rc = whereLoopAddBtree(&sSubBuild, mPrereq); 004532 } 004533 if( rc==SQLITE_OK ){ 004534 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); 004535 } 004536 testcase( rc==SQLITE_NOMEM && sCur.n>0 ); 004537 testcase( rc==SQLITE_DONE ); 004538 if( sCur.n==0 ){ 004539 sSum.n = 0; 004540 break; 004541 }else if( once ){ 004542 whereOrMove(&sSum, &sCur); 004543 once = 0; 004544 }else{ 004545 WhereOrSet sPrev; 004546 whereOrMove(&sPrev, &sSum); 004547 sSum.n = 0; 004548 for(i=0; i<sPrev.n; i++){ 004549 for(j=0; j<sCur.n; j++){ 004550 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq, 004551 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun), 004552 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut)); 004553 } 004554 } 004555 } 004556 } 004557 pNew->nLTerm = 1; 004558 pNew->aLTerm[0] = pTerm; 004559 pNew->wsFlags = WHERE_MULTI_OR; 004560 pNew->rSetup = 0; 004561 pNew->iSortIdx = 0; 004562 memset(&pNew->u, 0, sizeof(pNew->u)); 004563 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){ 004564 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs 004565 ** of all sub-scans required by the OR-scan. However, due to rounding 004566 ** errors, it may be that the cost of the OR-scan is equal to its 004567 ** most expensive sub-scan. Add the smallest possible penalty 004568 ** (equivalent to multiplying the cost by 1.07) to ensure that 004569 ** this does not happen. Otherwise, for WHERE clauses such as the 004570 ** following where there is an index on "y": 004571 ** 004572 ** WHERE likelihood(x=?, 0.99) OR y=? 004573 ** 004574 ** the planner may elect to "OR" together a full-table scan and an 004575 ** index lookup. And other similarly odd results. */ 004576 pNew->rRun = sSum.a[i].rRun + 1; 004577 pNew->nOut = sSum.a[i].nOut; 004578 pNew->prereq = sSum.a[i].prereq; 004579 rc = whereLoopInsert(pBuilder, pNew); 004580 } 004581 WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm)); 004582 } 004583 } 004584 return rc; 004585 } 004586 004587 /* 004588 ** Add all WhereLoop objects for all tables 004589 */ 004590 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ 004591 WhereInfo *pWInfo = pBuilder->pWInfo; 004592 Bitmask mPrereq = 0; 004593 Bitmask mPrior = 0; 004594 int iTab; 004595 SrcList *pTabList = pWInfo->pTabList; 004596 SrcItem *pItem; 004597 SrcItem *pEnd = &pTabList->a[pWInfo->nLevel]; 004598 sqlite3 *db = pWInfo->pParse->db; 004599 int rc = SQLITE_OK; 004600 int bFirstPastRJ = 0; 004601 int hasRightJoin = 0; 004602 WhereLoop *pNew; 004603 004604 004605 /* Loop over the tables in the join, from left to right */ 004606 pNew = pBuilder->pNew; 004607 004608 /* Verify that pNew has already been initialized */ 004609 assert( pNew->nLTerm==0 ); 004610 assert( pNew->wsFlags==0 ); 004611 assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) ); 004612 assert( pNew->aLTerm!=0 ); 004613 004614 pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; 004615 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){ 004616 Bitmask mUnusable = 0; 004617 pNew->iTab = iTab; 004618 pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR; 004619 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); 004620 if( bFirstPastRJ 004621 || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0 004622 ){ 004623 /* Add prerequisites to prevent reordering of FROM clause terms 004624 ** across CROSS joins and outer joins. The bFirstPastRJ boolean 004625 ** prevents the right operand of a RIGHT JOIN from being swapped with 004626 ** other elements even further to the right. 004627 ** 004628 ** The JT_LTORJ case and the hasRightJoin flag work together to 004629 ** prevent FROM-clause terms from moving from the right side of 004630 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN 004631 ** is itself on the left side of a RIGHT JOIN. 004632 */ 004633 if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; 004634 mPrereq |= mPrior; 004635 bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; 004636 }else if( !hasRightJoin ){ 004637 mPrereq = 0; 004638 } 004639 #ifndef SQLITE_OMIT_VIRTUALTABLE 004640 if( IsVirtual(pItem->pTab) ){ 004641 SrcItem *p; 004642 for(p=&pItem[1]; p<pEnd; p++){ 004643 if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){ 004644 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); 004645 } 004646 } 004647 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable); 004648 }else 004649 #endif /* SQLITE_OMIT_VIRTUALTABLE */ 004650 { 004651 rc = whereLoopAddBtree(pBuilder, mPrereq); 004652 } 004653 if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){ 004654 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); 004655 } 004656 mPrior |= pNew->maskSelf; 004657 if( rc || db->mallocFailed ){ 004658 if( rc==SQLITE_DONE ){ 004659 /* We hit the query planner search limit set by iPlanLimit */ 004660 sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search"); 004661 rc = SQLITE_OK; 004662 }else{ 004663 break; 004664 } 004665 } 004666 } 004667 004668 whereLoopClear(db, pNew); 004669 return rc; 004670 } 004671 004672 /* 004673 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th 004674 ** parameters) to see if it outputs rows in the requested ORDER BY 004675 ** (or GROUP BY) without requiring a separate sort operation. Return N: 004676 ** 004677 ** N>0: N terms of the ORDER BY clause are satisfied 004678 ** N==0: No terms of the ORDER BY clause are satisfied 004679 ** N<0: Unknown yet how many terms of ORDER BY might be satisfied. 004680 ** 004681 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as 004682 ** strict. With GROUP BY and DISTINCT the only requirement is that 004683 ** equivalent rows appear immediately adjacent to one another. GROUP BY 004684 ** and DISTINCT do not require rows to appear in any particular order as long 004685 ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT 004686 ** the pOrderBy terms can be matched in any order. With ORDER BY, the 004687 ** pOrderBy terms must be matched in strict left-to-right order. 004688 */ 004689 static i8 wherePathSatisfiesOrderBy( 004690 WhereInfo *pWInfo, /* The WHERE clause */ 004691 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ 004692 WherePath *pPath, /* The WherePath to check */ 004693 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */ 004694 u16 nLoop, /* Number of entries in pPath->aLoop[] */ 004695 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ 004696 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ 004697 ){ 004698 u8 revSet; /* True if rev is known */ 004699 u8 rev; /* Composite sort order */ 004700 u8 revIdx; /* Index sort order */ 004701 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ 004702 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ 004703 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ 004704 u16 eqOpMask; /* Allowed equality operators */ 004705 u16 nKeyCol; /* Number of key columns in pIndex */ 004706 u16 nColumn; /* Total number of ordered columns in the index */ 004707 u16 nOrderBy; /* Number terms in the ORDER BY clause */ 004708 int iLoop; /* Index of WhereLoop in pPath being processed */ 004709 int i, j; /* Loop counters */ 004710 int iCur; /* Cursor number for current WhereLoop */ 004711 int iColumn; /* A column number within table iCur */ 004712 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ 004713 WhereTerm *pTerm; /* A single term of the WHERE clause */ 004714 Expr *pOBExpr; /* An expression from the ORDER BY clause */ 004715 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ 004716 Index *pIndex; /* The index associated with pLoop */ 004717 sqlite3 *db = pWInfo->pParse->db; /* Database connection */ 004718 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ 004719 Bitmask obDone; /* Mask of all ORDER BY terms */ 004720 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ 004721 Bitmask ready; /* Mask of inner loops */ 004722 004723 /* 004724 ** We say the WhereLoop is "one-row" if it generates no more than one 004725 ** row of output. A WhereLoop is one-row if all of the following are true: 004726 ** (a) All index columns match with WHERE_COLUMN_EQ. 004727 ** (b) The index is unique 004728 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. 004729 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. 004730 ** 004731 ** We say the WhereLoop is "order-distinct" if the set of columns from 004732 ** that WhereLoop that are in the ORDER BY clause are different for every 004733 ** row of the WhereLoop. Every one-row WhereLoop is automatically 004734 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause 004735 ** is not order-distinct. To be order-distinct is not quite the same as being 004736 ** UNIQUE since a UNIQUE column or index can have multiple rows that 004737 ** are NULL and NULL values are equivalent for the purpose of order-distinct. 004738 ** To be order-distinct, the columns must be UNIQUE and NOT NULL. 004739 ** 004740 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the 004741 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is 004742 ** automatically order-distinct. 004743 */ 004744 004745 assert( pOrderBy!=0 ); 004746 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; 004747 004748 nOrderBy = pOrderBy->nExpr; 004749 testcase( nOrderBy==BMS-1 ); 004750 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ 004751 isOrderDistinct = 1; 004752 obDone = MASKBIT(nOrderBy)-1; 004753 orderDistinctMask = 0; 004754 ready = 0; 004755 eqOpMask = WO_EQ | WO_IS | WO_ISNULL; 004756 if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){ 004757 eqOpMask |= WO_IN; 004758 } 004759 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){ 004760 if( iLoop>0 ) ready |= pLoop->maskSelf; 004761 if( iLoop<nLoop ){ 004762 pLoop = pPath->aLoop[iLoop]; 004763 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue; 004764 }else{ 004765 pLoop = pLast; 004766 } 004767 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ 004768 if( pLoop->u.vtab.isOrdered 004769 && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) 004770 ){ 004771 obSat = obDone; 004772 } 004773 break; 004774 }else if( wctrlFlags & WHERE_DISTINCTBY ){ 004775 pLoop->u.btree.nDistinctCol = 0; 004776 } 004777 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; 004778 004779 /* Mark off any ORDER BY term X that is a column in the table of 004780 ** the current loop for which there is term in the WHERE 004781 ** clause of the form X IS NULL or X=? that reference only outer 004782 ** loops. 004783 */ 004784 for(i=0; i<nOrderBy; i++){ 004785 if( MASKBIT(i) & obSat ) continue; 004786 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); 004787 if( NEVER(pOBExpr==0) ) continue; 004788 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; 004789 if( pOBExpr->iTable!=iCur ) continue; 004790 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, 004791 ~ready, eqOpMask, 0); 004792 if( pTerm==0 ) continue; 004793 if( pTerm->eOperator==WO_IN ){ 004794 /* IN terms are only valid for sorting in the ORDER BY LIMIT 004795 ** optimization, and then only if they are actually used 004796 ** by the query plan */ 004797 assert( wctrlFlags & 004798 (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ); 004799 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){} 004800 if( j>=pLoop->nLTerm ) continue; 004801 } 004802 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ 004803 Parse *pParse = pWInfo->pParse; 004804 CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr); 004805 CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); 004806 assert( pColl1 ); 004807 if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){ 004808 continue; 004809 } 004810 testcase( pTerm->pExpr->op==TK_IS ); 004811 } 004812 obSat |= MASKBIT(i); 004813 } 004814 004815 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ 004816 if( pLoop->wsFlags & WHERE_IPK ){ 004817 pIndex = 0; 004818 nKeyCol = 0; 004819 nColumn = 1; 004820 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ 004821 return 0; 004822 }else{ 004823 nKeyCol = pIndex->nKeyCol; 004824 nColumn = pIndex->nColumn; 004825 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); 004826 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID 004827 || !HasRowid(pIndex->pTable)); 004828 /* All relevant terms of the index must also be non-NULL in order 004829 ** for isOrderDistinct to be true. So the isOrderDistint value 004830 ** computed here might be a false positive. Corrections will be 004831 ** made at tag-20210426-1 below */ 004832 isOrderDistinct = IsUniqueIndex(pIndex) 004833 && (pLoop->wsFlags & WHERE_SKIPSCAN)==0; 004834 } 004835 004836 /* Loop through all columns of the index and deal with the ones 004837 ** that are not constrained by == or IN. 004838 */ 004839 rev = revSet = 0; 004840 distinctColumns = 0; 004841 for(j=0; j<nColumn; j++){ 004842 u8 bOnce = 1; /* True to run the ORDER BY search loop */ 004843 004844 assert( j>=pLoop->u.btree.nEq 004845 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip) 004846 ); 004847 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){ 004848 u16 eOp = pLoop->aLTerm[j]->eOperator; 004849 004850 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when 004851 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL 004852 ** terms imply that the index is not UNIQUE NOT NULL in which case 004853 ** the loop need to be marked as not order-distinct because it can 004854 ** have repeated NULL rows. 004855 ** 004856 ** If the current term is a column of an ((?,?) IN (SELECT...)) 004857 ** expression for which the SELECT returns more than one column, 004858 ** check that it is the only column used by this loop. Otherwise, 004859 ** if it is one of two or more, none of the columns can be 004860 ** considered to match an ORDER BY term. 004861 */ 004862 if( (eOp & eqOpMask)!=0 ){ 004863 if( eOp & (WO_ISNULL|WO_IS) ){ 004864 testcase( eOp & WO_ISNULL ); 004865 testcase( eOp & WO_IS ); 004866 testcase( isOrderDistinct ); 004867 isOrderDistinct = 0; 004868 } 004869 continue; 004870 }else if( ALWAYS(eOp & WO_IN) ){ 004871 /* ALWAYS() justification: eOp is an equality operator due to the 004872 ** j<pLoop->u.btree.nEq constraint above. Any equality other 004873 ** than WO_IN is captured by the previous "if". So this one 004874 ** always has to be WO_IN. */ 004875 Expr *pX = pLoop->aLTerm[j]->pExpr; 004876 for(i=j+1; i<pLoop->u.btree.nEq; i++){ 004877 if( pLoop->aLTerm[i]->pExpr==pX ){ 004878 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) ); 004879 bOnce = 0; 004880 break; 004881 } 004882 } 004883 } 004884 } 004885 004886 /* Get the column number in the table (iColumn) and sort order 004887 ** (revIdx) for the j-th column of the index. 004888 */ 004889 if( pIndex ){ 004890 iColumn = pIndex->aiColumn[j]; 004891 revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC; 004892 if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID; 004893 }else{ 004894 iColumn = XN_ROWID; 004895 revIdx = 0; 004896 } 004897 004898 /* An unconstrained column that might be NULL means that this 004899 ** WhereLoop is not well-ordered. tag-20210426-1 004900 */ 004901 if( isOrderDistinct ){ 004902 if( iColumn>=0 004903 && j>=pLoop->u.btree.nEq 004904 && pIndex->pTable->aCol[iColumn].notNull==0 004905 ){ 004906 isOrderDistinct = 0; 004907 } 004908 if( iColumn==XN_EXPR ){ 004909 isOrderDistinct = 0; 004910 } 004911 } 004912 004913 /* Find the ORDER BY term that corresponds to the j-th column 004914 ** of the index and mark that ORDER BY term off 004915 */ 004916 isMatch = 0; 004917 for(i=0; bOnce && i<nOrderBy; i++){ 004918 if( MASKBIT(i) & obSat ) continue; 004919 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); 004920 testcase( wctrlFlags & WHERE_GROUPBY ); 004921 testcase( wctrlFlags & WHERE_DISTINCTBY ); 004922 if( NEVER(pOBExpr==0) ) continue; 004923 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; 004924 if( iColumn>=XN_ROWID ){ 004925 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; 004926 if( pOBExpr->iTable!=iCur ) continue; 004927 if( pOBExpr->iColumn!=iColumn ) continue; 004928 }else{ 004929 Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr; 004930 if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){ 004931 continue; 004932 } 004933 } 004934 if( iColumn!=XN_ROWID ){ 004935 pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); 004936 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; 004937 } 004938 if( wctrlFlags & WHERE_DISTINCTBY ){ 004939 pLoop->u.btree.nDistinctCol = j+1; 004940 } 004941 isMatch = 1; 004942 break; 004943 } 004944 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ 004945 /* Make sure the sort order is compatible in an ORDER BY clause. 004946 ** Sort order is irrelevant for a GROUP BY clause. */ 004947 if( revSet ){ 004948 if( (rev ^ revIdx) 004949 != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) 004950 ){ 004951 isMatch = 0; 004952 } 004953 }else{ 004954 rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); 004955 if( rev ) *pRevMask |= MASKBIT(iLoop); 004956 revSet = 1; 004957 } 004958 } 004959 if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){ 004960 if( j==pLoop->u.btree.nEq ){ 004961 pLoop->wsFlags |= WHERE_BIGNULL_SORT; 004962 }else{ 004963 isMatch = 0; 004964 } 004965 } 004966 if( isMatch ){ 004967 if( iColumn==XN_ROWID ){ 004968 testcase( distinctColumns==0 ); 004969 distinctColumns = 1; 004970 } 004971 obSat |= MASKBIT(i); 004972 }else{ 004973 /* No match found */ 004974 if( j==0 || j<nKeyCol ){ 004975 testcase( isOrderDistinct!=0 ); 004976 isOrderDistinct = 0; 004977 } 004978 break; 004979 } 004980 } /* end Loop over all index columns */ 004981 if( distinctColumns ){ 004982 testcase( isOrderDistinct==0 ); 004983 isOrderDistinct = 1; 004984 } 004985 } /* end-if not one-row */ 004986 004987 /* Mark off any other ORDER BY terms that reference pLoop */ 004988 if( isOrderDistinct ){ 004989 orderDistinctMask |= pLoop->maskSelf; 004990 for(i=0; i<nOrderBy; i++){ 004991 Expr *p; 004992 Bitmask mTerm; 004993 if( MASKBIT(i) & obSat ) continue; 004994 p = pOrderBy->a[i].pExpr; 004995 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); 004996 if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; 004997 if( (mTerm&~orderDistinctMask)==0 ){ 004998 obSat |= MASKBIT(i); 004999 } 005000 } 005001 } 005002 } /* End the loop over all WhereLoops from outer-most down to inner-most */ 005003 if( obSat==obDone ) return (i8)nOrderBy; 005004 if( !isOrderDistinct ){ 005005 for(i=nOrderBy-1; i>0; i--){ 005006 Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0; 005007 if( (obSat&m)==m ) return i; 005008 } 005009 return 0; 005010 } 005011 return -1; 005012 } 005013 005014 005015 /* 005016 ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), 005017 ** the planner assumes that the specified pOrderBy list is actually a GROUP 005018 ** BY clause - and so any order that groups rows as required satisfies the 005019 ** request. 005020 ** 005021 ** Normally, in this case it is not possible for the caller to determine 005022 ** whether or not the rows are really being delivered in sorted order, or 005023 ** just in some other order that provides the required grouping. However, 005024 ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then 005025 ** this function may be called on the returned WhereInfo object. It returns 005026 ** true if the rows really will be sorted in the specified order, or false 005027 ** otherwise. 005028 ** 005029 ** For example, assuming: 005030 ** 005031 ** CREATE INDEX i1 ON t1(x, Y); 005032 ** 005033 ** then 005034 ** 005035 ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 005036 ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 005037 */ 005038 int sqlite3WhereIsSorted(WhereInfo *pWInfo){ 005039 assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) ); 005040 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); 005041 return pWInfo->sorted; 005042 } 005043 005044 #ifdef WHERETRACE_ENABLED 005045 /* For debugging use only: */ 005046 static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ 005047 static char zName[65]; 005048 int i; 005049 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; } 005050 if( pLast ) zName[i++] = pLast->cId; 005051 zName[i] = 0; 005052 return zName; 005053 } 005054 #endif 005055 005056 /* 005057 ** Return the cost of sorting nRow rows, assuming that the keys have 005058 ** nOrderby columns and that the first nSorted columns are already in 005059 ** order. 005060 */ 005061 static LogEst whereSortingCost( 005062 WhereInfo *pWInfo, /* Query planning context */ 005063 LogEst nRow, /* Estimated number of rows to sort */ 005064 int nOrderBy, /* Number of ORDER BY clause terms */ 005065 int nSorted /* Number of initial ORDER BY terms naturally in order */ 005066 ){ 005067 /* Estimated cost of a full external sort, where N is 005068 ** the number of rows to sort is: 005069 ** 005070 ** cost = (K * N * log(N)). 005071 ** 005072 ** Or, if the order-by clause has X terms but only the last Y 005073 ** terms are out of order, then block-sorting will reduce the 005074 ** sorting cost to: 005075 ** 005076 ** cost = (K * N * log(N)) * (Y/X) 005077 ** 005078 ** The constant K is at least 2.0 but will be larger if there are a 005079 ** large number of columns to be sorted, as the sorting time is 005080 ** proportional to the amount of content to be sorted. The algorithm 005081 ** does not currently distinguish between fat columns (BLOBs and TEXTs) 005082 ** and skinny columns (INTs). It just uses the number of columns as 005083 ** an approximation for the row width. 005084 ** 005085 ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort 005086 ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert. 005087 */ 005088 LogEst rSortCost, nCol; 005089 assert( pWInfo->pSelect!=0 ); 005090 assert( pWInfo->pSelect->pEList!=0 ); 005091 /* TUNING: sorting cost proportional to the number of output columns: */ 005092 nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30); 005093 rSortCost = nRow + nCol; 005094 if( nSorted>0 ){ 005095 /* Scale the result by (Y/X) */ 005096 rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; 005097 } 005098 005099 /* Multiple by log(M) where M is the number of output rows. 005100 ** Use the LIMIT for M if it is smaller. Or if this sort is for 005101 ** a DISTINCT operator, M will be the number of distinct output 005102 ** rows, so fudge it downwards a bit. 005103 */ 005104 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){ 005105 rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */ 005106 if( nSorted!=0 ){ 005107 rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */ 005108 } 005109 if( pWInfo->iLimit<nRow ){ 005110 nRow = pWInfo->iLimit; 005111 } 005112 }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ 005113 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT 005114 ** reduces the number of output rows by a factor of 2 */ 005115 if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); } 005116 } 005117 rSortCost += estLog(nRow); 005118 return rSortCost; 005119 } 005120 005121 /* 005122 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine 005123 ** attempts to find the lowest cost path that visits each WhereLoop 005124 ** once. This path is then loaded into the pWInfo->a[].pWLoop fields. 005125 ** 005126 ** Assume that the total number of output rows that will need to be sorted 005127 ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting 005128 ** costs if nRowEst==0. 005129 ** 005130 ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation 005131 ** error occurs. 005132 */ 005133 static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ 005134 int mxChoice; /* Maximum number of simultaneous paths tracked */ 005135 int nLoop; /* Number of terms in the join */ 005136 Parse *pParse; /* Parsing context */ 005137 int iLoop; /* Loop counter over the terms of the join */ 005138 int ii, jj; /* Loop counters */ 005139 int mxI = 0; /* Index of next entry to replace */ 005140 int nOrderBy; /* Number of ORDER BY clause terms */ 005141 LogEst mxCost = 0; /* Maximum cost of a set of paths */ 005142 LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ 005143 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ 005144 WherePath *aFrom; /* All nFrom paths at the previous level */ 005145 WherePath *aTo; /* The nTo best paths at the current level */ 005146 WherePath *pFrom; /* An element of aFrom[] that we are working on */ 005147 WherePath *pTo; /* An element of aTo[] that we are working on */ 005148 WhereLoop *pWLoop; /* One of the WhereLoop objects */ 005149 WhereLoop **pX; /* Used to divy up the pSpace memory */ 005150 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ 005151 char *pSpace; /* Temporary memory used by this routine */ 005152 int nSpace; /* Bytes of space allocated at pSpace */ 005153 005154 pParse = pWInfo->pParse; 005155 nLoop = pWInfo->nLevel; 005156 /* TUNING: For simple queries, only the best path is tracked. 005157 ** For 2-way joins, the 5 best paths are followed. 005158 ** For joins of 3 or more tables, track the 10 best paths */ 005159 mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); 005160 assert( nLoop<=pWInfo->pTabList->nSrc ); 005161 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n", 005162 nRowEst, pParse->nQueryLoop)); 005163 005164 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this 005165 ** case the purpose of this call is to estimate the number of rows returned 005166 ** by the overall query. Once this estimate has been obtained, the caller 005167 ** will invoke this function a second time, passing the estimate as the 005168 ** nRowEst parameter. */ 005169 if( pWInfo->pOrderBy==0 || nRowEst==0 ){ 005170 nOrderBy = 0; 005171 }else{ 005172 nOrderBy = pWInfo->pOrderBy->nExpr; 005173 } 005174 005175 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ 005176 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; 005177 nSpace += sizeof(LogEst) * nOrderBy; 005178 pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace); 005179 if( pSpace==0 ) return SQLITE_NOMEM_BKPT; 005180 aTo = (WherePath*)pSpace; 005181 aFrom = aTo+mxChoice; 005182 memset(aFrom, 0, sizeof(aFrom[0])); 005183 pX = (WhereLoop**)(aFrom+mxChoice); 005184 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ 005185 pFrom->aLoop = pX; 005186 } 005187 if( nOrderBy ){ 005188 /* If there is an ORDER BY clause and it is not being ignored, set up 005189 ** space for the aSortCost[] array. Each element of the aSortCost array 005190 ** is either zero - meaning it has not yet been initialized - or the 005191 ** cost of sorting nRowEst rows of data where the first X terms of 005192 ** the ORDER BY clause are already in order, where X is the array 005193 ** index. */ 005194 aSortCost = (LogEst*)pX; 005195 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); 005196 } 005197 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); 005198 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); 005199 005200 /* Seed the search with a single WherePath containing zero WhereLoops. 005201 ** 005202 ** TUNING: Do not let the number of iterations go above 28. If the cost 005203 ** of computing an automatic index is not paid back within the first 28 005204 ** rows, then do not use the automatic index. */ 005205 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) ); 005206 nFrom = 1; 005207 assert( aFrom[0].isOrdered==0 ); 005208 if( nOrderBy ){ 005209 /* If nLoop is zero, then there are no FROM terms in the query. Since 005210 ** in this case the query may return a maximum of one row, the results 005211 ** are already in the requested order. Set isOrdered to nOrderBy to 005212 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to 005213 ** -1, indicating that the result set may or may not be ordered, 005214 ** depending on the loops added to the current plan. */ 005215 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; 005216 } 005217 005218 /* Compute successively longer WherePaths using the previous generation 005219 ** of WherePaths as the basis for the next. Keep track of the mxChoice 005220 ** best paths at each generation */ 005221 for(iLoop=0; iLoop<nLoop; iLoop++){ 005222 nTo = 0; 005223 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){ 005224 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ 005225 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ 005226 LogEst rCost; /* Cost of path (pFrom+pWLoop) */ 005227 LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ 005228 i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ 005229 Bitmask maskNew; /* Mask of src visited by (..) */ 005230 Bitmask revMask; /* Mask of rev-order loops for (..) */ 005231 005232 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; 005233 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; 005234 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){ 005235 /* Do not use an automatic index if the this loop is expected 005236 ** to run less than 1.25 times. It is tempting to also exclude 005237 ** automatic index usage on an outer loop, but sometimes an automatic 005238 ** index is useful in the outer loop of a correlated subquery. */ 005239 assert( 10==sqlite3LogEst(2) ); 005240 continue; 005241 } 005242 005243 /* At this point, pWLoop is a candidate to be the next loop. 005244 ** Compute its cost */ 005245 rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); 005246 rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); 005247 nOut = pFrom->nRow + pWLoop->nOut; 005248 maskNew = pFrom->maskLoop | pWLoop->maskSelf; 005249 isOrdered = pFrom->isOrdered; 005250 if( isOrdered<0 ){ 005251 revMask = 0; 005252 isOrdered = wherePathSatisfiesOrderBy(pWInfo, 005253 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, 005254 iLoop, pWLoop, &revMask); 005255 }else{ 005256 revMask = pFrom->revLoop; 005257 } 005258 if( isOrdered>=0 && isOrdered<nOrderBy ){ 005259 if( aSortCost[isOrdered]==0 ){ 005260 aSortCost[isOrdered] = whereSortingCost( 005261 pWInfo, nRowEst, nOrderBy, isOrdered 005262 ); 005263 } 005264 /* TUNING: Add a small extra penalty (3) to sorting as an 005265 ** extra encouragement to the query planner to select a plan 005266 ** where the rows emerge in the correct order without any sorting 005267 ** required. */ 005268 rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; 005269 005270 WHERETRACE(0x002, 005271 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", 005272 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, 005273 rUnsorted, rCost)); 005274 }else{ 005275 rCost = rUnsorted; 005276 rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ 005277 } 005278 005279 /* Check to see if pWLoop should be added to the set of 005280 ** mxChoice best-so-far paths. 005281 ** 005282 ** First look for an existing path among best-so-far paths 005283 ** that covers the same set of loops and has the same isOrdered 005284 ** setting as the current path candidate. 005285 ** 005286 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent 005287 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range 005288 ** of legal values for isOrdered, -1..64. 005289 */ 005290 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){ 005291 if( pTo->maskLoop==maskNew 005292 && ((pTo->isOrdered^isOrdered)&0x80)==0 005293 ){ 005294 testcase( jj==nTo-1 ); 005295 break; 005296 } 005297 } 005298 if( jj>=nTo ){ 005299 /* None of the existing best-so-far paths match the candidate. */ 005300 if( nTo>=mxChoice 005301 && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) 005302 ){ 005303 /* The current candidate is no better than any of the mxChoice 005304 ** paths currently in the best-so-far buffer. So discard 005305 ** this candidate as not viable. */ 005306 #ifdef WHERETRACE_ENABLED /* 0x4 */ 005307 if( sqlite3WhereTrace&0x4 ){ 005308 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n", 005309 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, 005310 isOrdered>=0 ? isOrdered+'0' : '?'); 005311 } 005312 #endif 005313 continue; 005314 } 005315 /* If we reach this points it means that the new candidate path 005316 ** needs to be added to the set of best-so-far paths. */ 005317 if( nTo<mxChoice ){ 005318 /* Increase the size of the aTo set by one */ 005319 jj = nTo++; 005320 }else{ 005321 /* New path replaces the prior worst to keep count below mxChoice */ 005322 jj = mxI; 005323 } 005324 pTo = &aTo[jj]; 005325 #ifdef WHERETRACE_ENABLED /* 0x4 */ 005326 if( sqlite3WhereTrace&0x4 ){ 005327 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n", 005328 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, 005329 isOrdered>=0 ? isOrdered+'0' : '?'); 005330 } 005331 #endif 005332 }else{ 005333 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the 005334 ** same set of loops and has the same isOrdered setting as the 005335 ** candidate path. Check to see if the candidate should replace 005336 ** pTo or if the candidate should be skipped. 005337 ** 005338 ** The conditional is an expanded vector comparison equivalent to: 005339 ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted) 005340 */ 005341 if( pTo->rCost<rCost 005342 || (pTo->rCost==rCost 005343 && (pTo->nRow<nOut 005344 || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted) 005345 ) 005346 ) 005347 ){ 005348 #ifdef WHERETRACE_ENABLED /* 0x4 */ 005349 if( sqlite3WhereTrace&0x4 ){ 005350 sqlite3DebugPrintf( 005351 "Skip %s cost=%-3d,%3d,%3d order=%c", 005352 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, 005353 isOrdered>=0 ? isOrdered+'0' : '?'); 005354 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n", 005355 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, 005356 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); 005357 } 005358 #endif 005359 /* Discard the candidate path from further consideration */ 005360 testcase( pTo->rCost==rCost ); 005361 continue; 005362 } 005363 testcase( pTo->rCost==rCost+1 ); 005364 /* Control reaches here if the candidate path is better than the 005365 ** pTo path. Replace pTo with the candidate. */ 005366 #ifdef WHERETRACE_ENABLED /* 0x4 */ 005367 if( sqlite3WhereTrace&0x4 ){ 005368 sqlite3DebugPrintf( 005369 "Update %s cost=%-3d,%3d,%3d order=%c", 005370 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, 005371 isOrdered>=0 ? isOrdered+'0' : '?'); 005372 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n", 005373 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, 005374 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); 005375 } 005376 #endif 005377 } 005378 /* pWLoop is a winner. Add it to the set of best so far */ 005379 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; 005380 pTo->revLoop = revMask; 005381 pTo->nRow = nOut; 005382 pTo->rCost = rCost; 005383 pTo->rUnsorted = rUnsorted; 005384 pTo->isOrdered = isOrdered; 005385 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); 005386 pTo->aLoop[iLoop] = pWLoop; 005387 if( nTo>=mxChoice ){ 005388 mxI = 0; 005389 mxCost = aTo[0].rCost; 005390 mxUnsorted = aTo[0].nRow; 005391 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){ 005392 if( pTo->rCost>mxCost 005393 || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) 005394 ){ 005395 mxCost = pTo->rCost; 005396 mxUnsorted = pTo->rUnsorted; 005397 mxI = jj; 005398 } 005399 } 005400 } 005401 } 005402 } 005403 005404 #ifdef WHERETRACE_ENABLED /* >=2 */ 005405 if( sqlite3WhereTrace & 0x02 ){ 005406 sqlite3DebugPrintf("---- after round %d ----\n", iLoop); 005407 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){ 005408 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c", 005409 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, 005410 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); 005411 if( pTo->isOrdered>0 ){ 005412 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); 005413 }else{ 005414 sqlite3DebugPrintf("\n"); 005415 } 005416 } 005417 } 005418 #endif 005419 005420 /* Swap the roles of aFrom and aTo for the next generation */ 005421 pFrom = aTo; 005422 aTo = aFrom; 005423 aFrom = pFrom; 005424 nFrom = nTo; 005425 } 005426 005427 if( nFrom==0 ){ 005428 sqlite3ErrorMsg(pParse, "no query solution"); 005429 sqlite3StackFreeNN(pParse->db, pSpace); 005430 return SQLITE_ERROR; 005431 } 005432 005433 /* Find the lowest cost path. pFrom will be left pointing to that path */ 005434 pFrom = aFrom; 005435 for(ii=1; ii<nFrom; ii++){ 005436 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; 005437 } 005438 assert( pWInfo->nLevel==nLoop ); 005439 /* Load the lowest cost path into pWInfo */ 005440 for(iLoop=0; iLoop<nLoop; iLoop++){ 005441 WhereLevel *pLevel = pWInfo->a + iLoop; 005442 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; 005443 pLevel->iFrom = pWLoop->iTab; 005444 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; 005445 } 005446 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 005447 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 005448 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP 005449 && nRowEst 005450 ){ 005451 Bitmask notUsed; 005452 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, 005453 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); 005454 if( rc==pWInfo->pResultSet->nExpr ){ 005455 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; 005456 } 005457 } 005458 pWInfo->bOrderedInnerLoop = 0; 005459 if( pWInfo->pOrderBy ){ 005460 pWInfo->nOBSat = pFrom->isOrdered; 005461 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ 005462 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ 005463 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; 005464 } 005465 if( pWInfo->pSelect->pOrderBy 005466 && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ 005467 pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; 005468 } 005469 }else{ 005470 pWInfo->revMask = pFrom->revLoop; 005471 if( pWInfo->nOBSat<=0 ){ 005472 pWInfo->nOBSat = 0; 005473 if( nLoop>0 ){ 005474 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags; 005475 if( (wsFlags & WHERE_ONEROW)==0 005476 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN) 005477 ){ 005478 Bitmask m = 0; 005479 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, 005480 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m); 005481 testcase( wsFlags & WHERE_IPK ); 005482 testcase( wsFlags & WHERE_COLUMN_IN ); 005483 if( rc==pWInfo->pOrderBy->nExpr ){ 005484 pWInfo->bOrderedInnerLoop = 1; 005485 pWInfo->revMask = m; 005486 } 005487 } 005488 } 005489 }else if( nLoop 005490 && pWInfo->nOBSat==1 005491 && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0 005492 ){ 005493 pWInfo->bOrderedInnerLoop = 1; 005494 } 005495 } 005496 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) 005497 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 005498 ){ 005499 Bitmask revMask = 0; 005500 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, 005501 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask 005502 ); 005503 assert( pWInfo->sorted==0 ); 005504 if( nOrder==pWInfo->pOrderBy->nExpr ){ 005505 pWInfo->sorted = 1; 005506 pWInfo->revMask = revMask; 005507 } 005508 } 005509 } 005510 005511 005512 pWInfo->nRowOut = pFrom->nRow; 005513 005514 /* Free temporary memory and return success */ 005515 sqlite3StackFreeNN(pParse->db, pSpace); 005516 return SQLITE_OK; 005517 } 005518 005519 /* 005520 ** Most queries use only a single table (they are not joins) and have 005521 ** simple == constraints against indexed fields. This routine attempts 005522 ** to plan those simple cases using much less ceremony than the 005523 ** general-purpose query planner, and thereby yield faster sqlite3_prepare() 005524 ** times for the common case. 005525 ** 005526 ** Return non-zero on success, if this query can be handled by this 005527 ** no-frills query planner. Return zero if this query needs the 005528 ** general-purpose query planner. 005529 */ 005530 static int whereShortCut(WhereLoopBuilder *pBuilder){ 005531 WhereInfo *pWInfo; 005532 SrcItem *pItem; 005533 WhereClause *pWC; 005534 WhereTerm *pTerm; 005535 WhereLoop *pLoop; 005536 int iCur; 005537 int j; 005538 Table *pTab; 005539 Index *pIdx; 005540 WhereScan scan; 005541 005542 pWInfo = pBuilder->pWInfo; 005543 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; 005544 assert( pWInfo->pTabList->nSrc>=1 ); 005545 pItem = pWInfo->pTabList->a; 005546 pTab = pItem->pTab; 005547 if( IsVirtual(pTab) ) return 0; 005548 if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){ 005549 testcase( pItem->fg.isIndexedBy ); 005550 testcase( pItem->fg.notIndexed ); 005551 return 0; 005552 } 005553 iCur = pItem->iCursor; 005554 pWC = &pWInfo->sWC; 005555 pLoop = pBuilder->pNew; 005556 pLoop->wsFlags = 0; 005557 pLoop->nSkip = 0; 005558 pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0); 005559 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan); 005560 if( pTerm ){ 005561 testcase( pTerm->eOperator & WO_IS ); 005562 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; 005563 pLoop->aLTerm[0] = pTerm; 005564 pLoop->nLTerm = 1; 005565 pLoop->u.btree.nEq = 1; 005566 /* TUNING: Cost of a rowid lookup is 10 */ 005567 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ 005568 }else{ 005569 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ 005570 int opMask; 005571 assert( pLoop->aLTermSpace==pLoop->aLTerm ); 005572 if( !IsUniqueIndex(pIdx) 005573 || pIdx->pPartIdxWhere!=0 005574 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) 005575 ) continue; 005576 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; 005577 for(j=0; j<pIdx->nKeyCol; j++){ 005578 pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx); 005579 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan); 005580 if( pTerm==0 ) break; 005581 testcase( pTerm->eOperator & WO_IS ); 005582 pLoop->aLTerm[j] = pTerm; 005583 } 005584 if( j!=pIdx->nKeyCol ) continue; 005585 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; 005586 if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){ 005587 pLoop->wsFlags |= WHERE_IDX_ONLY; 005588 } 005589 pLoop->nLTerm = j; 005590 pLoop->u.btree.nEq = j; 005591 pLoop->u.btree.pIndex = pIdx; 005592 /* TUNING: Cost of a unique index lookup is 15 */ 005593 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ 005594 break; 005595 } 005596 } 005597 if( pLoop->wsFlags ){ 005598 pLoop->nOut = (LogEst)1; 005599 pWInfo->a[0].pWLoop = pLoop; 005600 assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] ); 005601 pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */ 005602 pWInfo->a[0].iTabCur = iCur; 005603 pWInfo->nRowOut = 1; 005604 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; 005605 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ 005606 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; 005607 } 005608 if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS; 005609 #ifdef SQLITE_DEBUG 005610 pLoop->cId = '0'; 005611 #endif 005612 #ifdef WHERETRACE_ENABLED 005613 if( sqlite3WhereTrace & 0x02 ){ 005614 sqlite3DebugPrintf("whereShortCut() used to compute solution\n"); 005615 } 005616 #endif 005617 return 1; 005618 } 005619 return 0; 005620 } 005621 005622 /* 005623 ** Helper function for exprIsDeterministic(). 005624 */ 005625 static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){ 005626 if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){ 005627 pWalker->eCode = 0; 005628 return WRC_Abort; 005629 } 005630 return WRC_Continue; 005631 } 005632 005633 /* 005634 ** Return true if the expression contains no non-deterministic SQL 005635 ** functions. Do not consider non-deterministic SQL functions that are 005636 ** part of sub-select statements. 005637 */ 005638 static int exprIsDeterministic(Expr *p){ 005639 Walker w; 005640 memset(&w, 0, sizeof(w)); 005641 w.eCode = 1; 005642 w.xExprCallback = exprNodeIsDeterministic; 005643 w.xSelectCallback = sqlite3SelectWalkFail; 005644 sqlite3WalkExpr(&w, p); 005645 return w.eCode; 005646 } 005647 005648 005649 #ifdef WHERETRACE_ENABLED 005650 /* 005651 ** Display all WhereLoops in pWInfo 005652 */ 005653 static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ 005654 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ 005655 WhereLoop *p; 005656 int i; 005657 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" 005658 "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; 005659 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ 005660 p->cId = zLabel[i%(sizeof(zLabel)-1)]; 005661 sqlite3WhereLoopPrint(p, pWC); 005662 } 005663 } 005664 } 005665 # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C) 005666 #else 005667 # define WHERETRACE_ALL_LOOPS(W,C) 005668 #endif 005669 005670 /* Attempt to omit tables from a join that do not affect the result. 005671 ** For a table to not affect the result, the following must be true: 005672 ** 005673 ** 1) The query must not be an aggregate. 005674 ** 2) The table must be the RHS of a LEFT JOIN. 005675 ** 3) Either the query must be DISTINCT, or else the ON or USING clause 005676 ** must contain a constraint that limits the scan of the table to 005677 ** at most a single row. 005678 ** 4) The table must not be referenced by any part of the query apart 005679 ** from its own USING or ON clause. 005680 ** 5) The table must not have an inner-join ON or USING clause if there is 005681 ** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause 005682 ** might move from the right side to the left side of the RIGHT JOIN. 005683 ** Note: Due to (2), this condition can only arise if the table is 005684 ** the right-most table of a subquery that was flattened into the 005685 ** main query and that subquery was the right-hand operand of an 005686 ** inner join that held an ON or USING clause. 005687 ** 005688 ** For example, given: 005689 ** 005690 ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); 005691 ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); 005692 ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3); 005693 ** 005694 ** then table t2 can be omitted from the following: 005695 ** 005696 ** SELECT v1, v3 FROM t1 005697 ** LEFT JOIN t2 ON (t1.ipk=t2.ipk) 005698 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) 005699 ** 005700 ** or from: 005701 ** 005702 ** SELECT DISTINCT v1, v3 FROM t1 005703 ** LEFT JOIN t2 005704 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) 005705 */ 005706 static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( 005707 WhereInfo *pWInfo, 005708 Bitmask notReady 005709 ){ 005710 int i; 005711 Bitmask tabUsed; 005712 int hasRightJoin; 005713 005714 /* Preconditions checked by the caller */ 005715 assert( pWInfo->nLevel>=2 ); 005716 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) ); 005717 005718 /* These two preconditions checked by the caller combine to guarantee 005719 ** condition (1) of the header comment */ 005720 assert( pWInfo->pResultSet!=0 ); 005721 assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) ); 005722 005723 tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet); 005724 if( pWInfo->pOrderBy ){ 005725 tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy); 005726 } 005727 hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0; 005728 for(i=pWInfo->nLevel-1; i>=1; i--){ 005729 WhereTerm *pTerm, *pEnd; 005730 SrcItem *pItem; 005731 WhereLoop *pLoop; 005732 pLoop = pWInfo->a[i].pWLoop; 005733 pItem = &pWInfo->pTabList->a[pLoop->iTab]; 005734 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue; 005735 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0 005736 && (pLoop->wsFlags & WHERE_ONEROW)==0 005737 ){ 005738 continue; 005739 } 005740 if( (tabUsed & pLoop->maskSelf)!=0 ) continue; 005741 pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm; 005742 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ 005743 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ 005744 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON) 005745 || pTerm->pExpr->w.iJoin!=pItem->iCursor 005746 ){ 005747 break; 005748 } 005749 } 005750 if( hasRightJoin 005751 && ExprHasProperty(pTerm->pExpr, EP_InnerON) 005752 && pTerm->pExpr->w.iJoin==pItem->iCursor 005753 ){ 005754 break; /* restriction (5) */ 005755 } 005756 } 005757 if( pTerm<pEnd ) continue; 005758 WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId)); 005759 notReady &= ~pLoop->maskSelf; 005760 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ 005761 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ 005762 pTerm->wtFlags |= TERM_CODED; 005763 } 005764 } 005765 if( i!=pWInfo->nLevel-1 ){ 005766 int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); 005767 memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); 005768 } 005769 pWInfo->nLevel--; 005770 assert( pWInfo->nLevel>0 ); 005771 } 005772 return notReady; 005773 } 005774 005775 /* 005776 ** Check to see if there are any SEARCH loops that might benefit from 005777 ** using a Bloom filter. Consider a Bloom filter if: 005778 ** 005779 ** (1) The SEARCH happens more than N times where N is the number 005780 ** of rows in the table that is being considered for the Bloom 005781 ** filter. 005782 ** (2) Some searches are expected to find zero rows. (This is determined 005783 ** by the WHERE_SELFCULL flag on the term.) 005784 ** (3) Bloom-filter processing is not disabled. (Checked by the 005785 ** caller.) 005786 ** (4) The size of the table being searched is known by ANALYZE. 005787 ** 005788 ** This block of code merely checks to see if a Bloom filter would be 005789 ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the 005790 ** WhereLoop. The implementation of the Bloom filter comes further 005791 ** down where the code for each WhereLoop is generated. 005792 */ 005793 static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( 005794 const WhereInfo *pWInfo 005795 ){ 005796 int i; 005797 LogEst nSearch = 0; 005798 005799 assert( pWInfo->nLevel>=2 ); 005800 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) ); 005801 for(i=0; i<pWInfo->nLevel; i++){ 005802 WhereLoop *pLoop = pWInfo->a[i].pWLoop; 005803 const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); 005804 SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; 005805 Table *pTab = pItem->pTab; 005806 if( (pTab->tabFlags & TF_HasStat1)==0 ) break; 005807 pTab->tabFlags |= TF_StatsUsed; 005808 if( i>=1 005809 && (pLoop->wsFlags & reqFlags)==reqFlags 005810 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ 005811 && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0) 005812 ){ 005813 if( nSearch > pTab->nRowLogEst ){ 005814 testcase( pItem->fg.jointype & JT_LEFT ); 005815 pLoop->wsFlags |= WHERE_BLOOMFILTER; 005816 pLoop->wsFlags &= ~WHERE_IDX_ONLY; 005817 WHERETRACE(0xffffffff, ( 005818 "-> use Bloom-filter on loop %c because there are ~%.1e " 005819 "lookups into %s which has only ~%.1e rows\n", 005820 pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName, 005821 (double)sqlite3LogEstToInt(pTab->nRowLogEst))); 005822 } 005823 } 005824 nSearch += pLoop->nOut; 005825 } 005826 } 005827 005828 /* 005829 ** The index pIdx is used by a query and contains one or more expressions. 005830 ** In other words pIdx is an index on an expression. iIdxCur is the cursor 005831 ** number for the index and iDataCur is the cursor number for the corresponding 005832 ** table. 005833 ** 005834 ** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for 005835 ** each of the expressions in the index so that the expression code generator 005836 ** will know to replace occurrences of the indexed expression with 005837 ** references to the corresponding column of the index. 005838 */ 005839 static SQLITE_NOINLINE void whereAddIndexedExpr( 005840 Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */ 005841 Index *pIdx, /* The index-on-expression that contains the expressions */ 005842 int iIdxCur, /* Cursor number for pIdx */ 005843 SrcItem *pTabItem /* The FROM clause entry for the table */ 005844 ){ 005845 int i; 005846 IndexedExpr *p; 005847 Table *pTab; 005848 assert( pIdx->bHasExpr ); 005849 pTab = pIdx->pTable; 005850 for(i=0; i<pIdx->nColumn; i++){ 005851 Expr *pExpr; 005852 int j = pIdx->aiColumn[i]; 005853 int bMaybeNullRow; 005854 if( j==XN_EXPR ){ 005855 pExpr = pIdx->aColExpr->a[i].pExpr; 005856 testcase( pTabItem->fg.jointype & JT_LEFT ); 005857 testcase( pTabItem->fg.jointype & JT_RIGHT ); 005858 testcase( pTabItem->fg.jointype & JT_LTORJ ); 005859 bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; 005860 }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ 005861 pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); 005862 bMaybeNullRow = 0; 005863 }else{ 005864 continue; 005865 } 005866 if( sqlite3ExprIsConstant(pExpr) ) continue; 005867 if( pExpr->op==TK_FUNCTION ){ 005868 /* Functions that might set a subtype should not be replaced by the 005869 ** value taken from an expression index since the index omits the 005870 ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ 005871 int n; 005872 FuncDef *pDef; 005873 sqlite3 *db = pParse->db; 005874 assert( ExprUseXList(pExpr) ); 005875 n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; 005876 pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); 005877 if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ 005878 continue; 005879 } 005880 } 005881 p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); 005882 if( p==0 ) break; 005883 p->pIENext = pParse->pIdxEpr; 005884 #ifdef WHERETRACE_ENABLED 005885 if( sqlite3WhereTrace & 0x200 ){ 005886 sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i); 005887 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr); 005888 } 005889 #endif 005890 p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); 005891 p->iDataCur = pTabItem->iCursor; 005892 p->iIdxCur = iIdxCur; 005893 p->iIdxCol = i; 005894 p->bMaybeNullRow = bMaybeNullRow; 005895 if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ 005896 p->aff = pIdx->zColAff[i]; 005897 } 005898 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS 005899 p->zIdxName = pIdx->zName; 005900 #endif 005901 pParse->pIdxEpr = p; 005902 if( p->pIENext==0 ){ 005903 void *pArg = (void*)&pParse->pIdxEpr; 005904 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); 005905 } 005906 } 005907 } 005908 005909 /* 005910 ** Set the reverse-scan order mask to one for all tables in the query 005911 ** with the exception of MATERIALIZED common table expressions that have 005912 ** their own internal ORDER BY clauses. 005913 ** 005914 ** This implements the PRAGMA reverse_unordered_selects=ON setting. 005915 ** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER). 005916 */ 005917 static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ 005918 int ii; 005919 for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){ 005920 SrcItem *pItem = &pWInfo->pTabList->a[ii]; 005921 if( !pItem->fg.isCte 005922 || pItem->u2.pCteUse->eM10d!=M10d_Yes 005923 || NEVER(pItem->pSelect==0) 005924 || pItem->pSelect->pOrderBy==0 005925 ){ 005926 pWInfo->revMask |= MASKBIT(ii); 005927 } 005928 } 005929 } 005930 005931 /* 005932 ** Generate the beginning of the loop used for WHERE clause processing. 005933 ** The return value is a pointer to an opaque structure that contains 005934 ** information needed to terminate the loop. Later, the calling routine 005935 ** should invoke sqlite3WhereEnd() with the return value of this function 005936 ** in order to complete the WHERE clause processing. 005937 ** 005938 ** If an error occurs, this routine returns NULL. 005939 ** 005940 ** The basic idea is to do a nested loop, one loop for each table in 005941 ** the FROM clause of a select. (INSERT and UPDATE statements are the 005942 ** same as a SELECT with only a single table in the FROM clause.) For 005943 ** example, if the SQL is this: 005944 ** 005945 ** SELECT * FROM t1, t2, t3 WHERE ...; 005946 ** 005947 ** Then the code generated is conceptually like the following: 005948 ** 005949 ** foreach row1 in t1 do \ Code generated 005950 ** foreach row2 in t2 do |-- by sqlite3WhereBegin() 005951 ** foreach row3 in t3 do / 005952 ** ... 005953 ** end \ Code generated 005954 ** end |-- by sqlite3WhereEnd() 005955 ** end / 005956 ** 005957 ** Note that the loops might not be nested in the order in which they 005958 ** appear in the FROM clause if a different order is better able to make 005959 ** use of indices. Note also that when the IN operator appears in 005960 ** the WHERE clause, it might result in additional nested loops for 005961 ** scanning through all values on the right-hand side of the IN. 005962 ** 005963 ** There are Btree cursors associated with each table. t1 uses cursor 005964 ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. 005965 ** And so forth. This routine generates code to open those VDBE cursors 005966 ** and sqlite3WhereEnd() generates the code to close them. 005967 ** 005968 ** The code that sqlite3WhereBegin() generates leaves the cursors named 005969 ** in pTabList pointing at their appropriate entries. The [...] code 005970 ** can use OP_Column and OP_Rowid opcodes on these cursors to extract 005971 ** data from the various tables of the loop. 005972 ** 005973 ** If the WHERE clause is empty, the foreach loops must each scan their 005974 ** entire tables. Thus a three-way join is an O(N^3) operation. But if 005975 ** the tables have indices and there are terms in the WHERE clause that 005976 ** refer to those indices, a complete table scan can be avoided and the 005977 ** code will run much faster. Most of the work of this routine is checking 005978 ** to see if there are indices that can be used to speed up the loop. 005979 ** 005980 ** Terms of the WHERE clause are also used to limit which rows actually 005981 ** make it to the "..." in the middle of the loop. After each "foreach", 005982 ** terms of the WHERE clause that use only terms in that loop and outer 005983 ** loops are evaluated and if false a jump is made around all subsequent 005984 ** inner loops (or around the "..." if the test occurs within the inner- 005985 ** most loop) 005986 ** 005987 ** OUTER JOINS 005988 ** 005989 ** An outer join of tables t1 and t2 is conceptually coded as follows: 005990 ** 005991 ** foreach row1 in t1 do 005992 ** flag = 0 005993 ** foreach row2 in t2 do 005994 ** start: 005995 ** ... 005996 ** flag = 1 005997 ** end 005998 ** if flag==0 then 005999 ** move the row2 cursor to a null row 006000 ** goto start 006001 ** fi 006002 ** end 006003 ** 006004 ** ORDER BY CLAUSE PROCESSING 006005 ** 006006 ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause 006007 ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement 006008 ** if there is one. If there is no ORDER BY clause or if this routine 006009 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. 006010 ** 006011 ** The iIdxCur parameter is the cursor number of an index. If 006012 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index 006013 ** to use for OR clause processing. The WHERE clause should use this 006014 ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is 006015 ** the first cursor in an array of cursors for all indices. iIdxCur should 006016 ** be used to compute the appropriate cursor depending on which index is 006017 ** used. 006018 */ 006019 WhereInfo *sqlite3WhereBegin( 006020 Parse *pParse, /* The parser context */ 006021 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ 006022 Expr *pWhere, /* The WHERE clause */ 006023 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ 006024 ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ 006025 Select *pSelect, /* The entire SELECT statement */ 006026 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ 006027 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number 006028 ** If WHERE_USE_LIMIT, then the limit amount */ 006029 ){ 006030 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ 006031 int nTabList; /* Number of elements in pTabList */ 006032 WhereInfo *pWInfo; /* Will become the return value of this function */ 006033 Vdbe *v = pParse->pVdbe; /* The virtual database engine */ 006034 Bitmask notReady; /* Cursors that are not yet positioned */ 006035 WhereLoopBuilder sWLB; /* The WhereLoop builder */ 006036 WhereMaskSet *pMaskSet; /* The expression mask set */ 006037 WhereLevel *pLevel; /* A single level in pWInfo->a[] */ 006038 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ 006039 int ii; /* Loop counter */ 006040 sqlite3 *db; /* Database connection */ 006041 int rc; /* Return code */ 006042 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */ 006043 006044 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( 006045 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 006046 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 006047 )); 006048 006049 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */ 006050 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 006051 || (wctrlFlags & WHERE_USE_LIMIT)==0 ); 006052 006053 /* Variable initialization */ 006054 db = pParse->db; 006055 memset(&sWLB, 0, sizeof(sWLB)); 006056 006057 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ 006058 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); 006059 if( pOrderBy && pOrderBy->nExpr>=BMS ){ 006060 pOrderBy = 0; 006061 wctrlFlags &= ~WHERE_WANT_DISTINCT; 006062 } 006063 006064 /* The number of tables in the FROM clause is limited by the number of 006065 ** bits in a Bitmask 006066 */ 006067 testcase( pTabList->nSrc==BMS ); 006068 if( pTabList->nSrc>BMS ){ 006069 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS); 006070 return 0; 006071 } 006072 006073 /* This function normally generates a nested loop for all tables in 006074 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should 006075 ** only generate code for the first table in pTabList and assume that 006076 ** any cursors associated with subsequent tables are uninitialized. 006077 */ 006078 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc; 006079 006080 /* Allocate and initialize the WhereInfo structure that will become the 006081 ** return value. A single allocation is used to store the WhereInfo 006082 ** struct, the contents of WhereInfo.a[], the WhereClause structure 006083 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte 006084 ** field (type Bitmask) it must be aligned on an 8-byte boundary on 006085 ** some architectures. Hence the ROUND8() below. 006086 */ 006087 nByteWInfo = ROUND8P(sizeof(WhereInfo)); 006088 if( nTabList>1 ){ 006089 nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); 006090 } 006091 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); 006092 if( db->mallocFailed ){ 006093 sqlite3DbFree(db, pWInfo); 006094 pWInfo = 0; 006095 goto whereBeginError; 006096 } 006097 pWInfo->pParse = pParse; 006098 pWInfo->pTabList = pTabList; 006099 pWInfo->pOrderBy = pOrderBy; 006100 #if WHERETRACE_ENABLED 006101 pWInfo->pWhere = pWhere; 006102 #endif 006103 pWInfo->pResultSet = pResultSet; 006104 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; 006105 pWInfo->nLevel = nTabList; 006106 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse); 006107 pWInfo->wctrlFlags = wctrlFlags; 006108 pWInfo->iLimit = iAuxArg; 006109 pWInfo->savedNQueryLoop = pParse->nQueryLoop; 006110 pWInfo->pSelect = pSelect; 006111 memset(&pWInfo->nOBSat, 0, 006112 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); 006113 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); 006114 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ 006115 pMaskSet = &pWInfo->sMaskSet; 006116 pMaskSet->n = 0; 006117 pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be 006118 ** a valid cursor number, to avoid an initial 006119 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */ 006120 sWLB.pWInfo = pWInfo; 006121 sWLB.pWC = &pWInfo->sWC; 006122 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); 006123 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) ); 006124 whereLoopInit(sWLB.pNew); 006125 #ifdef SQLITE_DEBUG 006126 sWLB.pNew->cId = '*'; 006127 #endif 006128 006129 /* Split the WHERE clause into separate subexpressions where each 006130 ** subexpression is separated by an AND operator. 006131 */ 006132 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); 006133 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); 006134 006135 /* Special case: No FROM clause 006136 */ 006137 if( nTabList==0 ){ 006138 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; 006139 if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0 006140 && OptimizationEnabled(db, SQLITE_DistinctOpt) 006141 ){ 006142 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; 006143 } 006144 ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); 006145 }else{ 006146 /* Assign a bit from the bitmask to every term in the FROM clause. 006147 ** 006148 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N. 006149 ** 006150 ** The rule of the previous sentence ensures that if X is the bitmask for 006151 ** a table T, then X-1 is the bitmask for all other tables to the left of T. 006152 ** Knowing the bitmask for all tables to the left of a left join is 006153 ** important. Ticket #3015. 006154 ** 006155 ** Note that bitmasks are created for all pTabList->nSrc tables in 006156 ** pTabList, not just the first nTabList tables. nTabList is normally 006157 ** equal to pTabList->nSrc but might be shortened to 1 if the 006158 ** WHERE_OR_SUBCLAUSE flag is set. 006159 */ 006160 ii = 0; 006161 do{ 006162 createMask(pMaskSet, pTabList->a[ii].iCursor); 006163 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); 006164 }while( (++ii)<pTabList->nSrc ); 006165 #ifdef SQLITE_DEBUG 006166 { 006167 Bitmask mx = 0; 006168 for(ii=0; ii<pTabList->nSrc; ii++){ 006169 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); 006170 assert( m>=mx ); 006171 mx = m; 006172 } 006173 } 006174 #endif 006175 } 006176 006177 /* Analyze all of the subexpressions. */ 006178 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); 006179 if( pSelect && pSelect->pLimit ){ 006180 sqlite3WhereAddLimit(&pWInfo->sWC, pSelect); 006181 } 006182 if( pParse->nErr ) goto whereBeginError; 006183 006184 /* The False-WHERE-Term-Bypass optimization: 006185 ** 006186 ** If there are WHERE terms that are false, then no rows will be output, 006187 ** so skip over all of the code generated here. 006188 ** 006189 ** Conditions: 006190 ** 006191 ** (1) The WHERE term must not refer to any tables in the join. 006192 ** (2) The term must not come from an ON clause on the 006193 ** right-hand side of a LEFT or FULL JOIN. 006194 ** (3) The term must not come from an ON clause, or there must be 006195 ** no RIGHT or FULL OUTER joins in pTabList. 006196 ** (4) If the expression contains non-deterministic functions 006197 ** that are not within a sub-select. This is not required 006198 ** for correctness but rather to preserves SQLite's legacy 006199 ** behaviour in the following two cases: 006200 ** 006201 ** WHERE random()>0; -- eval random() once per row 006202 ** WHERE (SELECT random())>0; -- eval random() just once overall 006203 ** 006204 ** Note that the Where term need not be a constant in order for this 006205 ** optimization to apply, though it does need to be constant relative to 006206 ** the current subquery (condition 1). The term might include variables 006207 ** from outer queries so that the value of the term changes from one 006208 ** invocation of the current subquery to the next. 006209 */ 006210 for(ii=0; ii<sWLB.pWC->nBase; ii++){ 006211 WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ 006212 Expr *pX; /* The expression of pT */ 006213 if( pT->wtFlags & TERM_VIRTUAL ) continue; 006214 pX = pT->pExpr; 006215 assert( pX!=0 ); 006216 assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) ); 006217 if( pT->prereqAll==0 /* Conditions (1) and (2) */ 006218 && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */ 006219 && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */ 006220 && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 ) 006221 ){ 006222 sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL); 006223 pT->wtFlags |= TERM_CODED; 006224 } 006225 } 006226 006227 if( wctrlFlags & WHERE_WANT_DISTINCT ){ 006228 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ 006229 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via 006230 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ 006231 wctrlFlags &= ~WHERE_WANT_DISTINCT; 006232 pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT; 006233 }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ 006234 /* The DISTINCT marking is pointless. Ignore it. */ 006235 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; 006236 }else if( pOrderBy==0 ){ 006237 /* Try to ORDER BY the result set to make distinct processing easier */ 006238 pWInfo->wctrlFlags |= WHERE_DISTINCTBY; 006239 pWInfo->pOrderBy = pResultSet; 006240 } 006241 } 006242 006243 /* Construct the WhereLoop objects */ 006244 #if defined(WHERETRACE_ENABLED) 006245 if( sqlite3WhereTrace & 0xffffffff ){ 006246 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); 006247 if( wctrlFlags & WHERE_USE_LIMIT ){ 006248 sqlite3DebugPrintf(", limit: %d", iAuxArg); 006249 } 006250 sqlite3DebugPrintf(")\n"); 006251 if( sqlite3WhereTrace & 0x8000 ){ 006252 Select sSelect; 006253 memset(&sSelect, 0, sizeof(sSelect)); 006254 sSelect.selFlags = SF_WhereBegin; 006255 sSelect.pSrc = pTabList; 006256 sSelect.pWhere = pWhere; 006257 sSelect.pOrderBy = pOrderBy; 006258 sSelect.pEList = pResultSet; 006259 sqlite3TreeViewSelect(0, &sSelect, 0); 006260 } 006261 if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */ 006262 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); 006263 sqlite3WhereClausePrint(sWLB.pWC); 006264 } 006265 } 006266 #endif 006267 006268 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ 006269 rc = whereLoopAddAll(&sWLB); 006270 if( rc ) goto whereBeginError; 006271 006272 #ifdef SQLITE_ENABLE_STAT4 006273 /* If one or more WhereTerm.truthProb values were used in estimating 006274 ** loop parameters, but then those truthProb values were subsequently 006275 ** changed based on STAT4 information while computing subsequent loops, 006276 ** then we need to rerun the whole loop building process so that all 006277 ** loops will be built using the revised truthProb values. */ 006278 if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ 006279 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); 006280 WHERETRACE(0xffffffff, 006281 ("**** Redo all loop computations due to" 006282 " TERM_HIGHTRUTH changes ****\n")); 006283 while( pWInfo->pLoops ){ 006284 WhereLoop *p = pWInfo->pLoops; 006285 pWInfo->pLoops = p->pNextLoop; 006286 whereLoopDelete(db, p); 006287 } 006288 rc = whereLoopAddAll(&sWLB); 006289 if( rc ) goto whereBeginError; 006290 } 006291 #endif 006292 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); 006293 006294 wherePathSolver(pWInfo, 0); 006295 if( db->mallocFailed ) goto whereBeginError; 006296 if( pWInfo->pOrderBy ){ 006297 wherePathSolver(pWInfo, pWInfo->nRowOut+1); 006298 if( db->mallocFailed ) goto whereBeginError; 006299 } 006300 006301 /* TUNING: Assume that a DISTINCT clause on a subquery reduces 006302 ** the output size by a factor of 8 (LogEst -30). 006303 */ 006304 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ 006305 WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n", 006306 pWInfo->nRowOut, pWInfo->nRowOut-30)); 006307 pWInfo->nRowOut -= 30; 006308 } 006309 006310 } 006311 assert( pWInfo->pTabList!=0 ); 006312 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ 006313 whereReverseScanOrder(pWInfo); 006314 } 006315 if( pParse->nErr ){ 006316 goto whereBeginError; 006317 } 006318 assert( db->mallocFailed==0 ); 006319 #ifdef WHERETRACE_ENABLED 006320 if( sqlite3WhereTrace ){ 006321 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); 006322 if( pWInfo->nOBSat>0 ){ 006323 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); 006324 } 006325 switch( pWInfo->eDistinct ){ 006326 case WHERE_DISTINCT_UNIQUE: { 006327 sqlite3DebugPrintf(" DISTINCT=unique"); 006328 break; 006329 } 006330 case WHERE_DISTINCT_ORDERED: { 006331 sqlite3DebugPrintf(" DISTINCT=ordered"); 006332 break; 006333 } 006334 case WHERE_DISTINCT_UNORDERED: { 006335 sqlite3DebugPrintf(" DISTINCT=unordered"); 006336 break; 006337 } 006338 } 006339 sqlite3DebugPrintf("\n"); 006340 for(ii=0; ii<pWInfo->nLevel; ii++){ 006341 sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); 006342 } 006343 } 006344 #endif 006345 006346 /* Attempt to omit tables from a join that do not affect the result. 006347 ** See the comment on whereOmitNoopJoin() for further information. 006348 ** 006349 ** This query optimization is factored out into a separate "no-inline" 006350 ** procedure to keep the sqlite3WhereBegin() procedure from becoming 006351 ** too large. If sqlite3WhereBegin() becomes too large, that prevents 006352 ** some C-compiler optimizers from in-lining the 006353 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to 006354 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. 006355 */ 006356 notReady = ~(Bitmask)0; 006357 if( pWInfo->nLevel>=2 006358 && pResultSet!=0 /* these two combine to guarantee */ 006359 && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ 006360 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) 006361 ){ 006362 notReady = whereOmitNoopJoin(pWInfo, notReady); 006363 nTabList = pWInfo->nLevel; 006364 assert( nTabList>0 ); 006365 } 006366 006367 /* Check to see if there are any SEARCH loops that might benefit from 006368 ** using a Bloom filter. 006369 */ 006370 if( pWInfo->nLevel>=2 006371 && OptimizationEnabled(db, SQLITE_BloomFilter) 006372 ){ 006373 whereCheckIfBloomFilterIsUseful(pWInfo); 006374 } 006375 006376 #if defined(WHERETRACE_ENABLED) 006377 if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */ 006378 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); 006379 sqlite3WhereClausePrint(sWLB.pWC); 006380 } 006381 WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n")); 006382 #endif 006383 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; 006384 006385 /* If the caller is an UPDATE or DELETE statement that is requesting 006386 ** to use a one-pass algorithm, determine if this is appropriate. 006387 ** 006388 ** A one-pass approach can be used if the caller has requested one 006389 ** and either (a) the scan visits at most one row or (b) each 006390 ** of the following are true: 006391 ** 006392 ** * the caller has indicated that a one-pass approach can be used 006393 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and 006394 ** * the table is not a virtual table, and 006395 ** * either the scan does not use the OR optimization or the caller 006396 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified 006397 ** for DELETE). 006398 ** 006399 ** The last qualification is because an UPDATE statement uses 006400 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can 006401 ** use a one-pass approach, and this is not set accurately for scans 006402 ** that use the OR optimization. 006403 */ 006404 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); 006405 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ 006406 int wsFlags = pWInfo->a[0].pWLoop->wsFlags; 006407 int bOnerow = (wsFlags & WHERE_ONEROW)!=0; 006408 assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); 006409 if( bOnerow || ( 006410 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) 006411 && !IsVirtual(pTabList->a[0].pTab) 006412 && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) 006413 && OptimizationEnabled(db, SQLITE_OnePass) 006414 )){ 006415 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; 006416 if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ 006417 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ 006418 bFordelete = OPFLAG_FORDELETE; 006419 } 006420 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY); 006421 } 006422 } 006423 } 006424 006425 /* Open all tables in the pTabList and any indices selected for 006426 ** searching those tables. 006427 */ 006428 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){ 006429 Table *pTab; /* Table to open */ 006430 int iDb; /* Index of database containing table/index */ 006431 SrcItem *pTabItem; 006432 006433 pTabItem = &pTabList->a[pLevel->iFrom]; 006434 pTab = pTabItem->pTab; 006435 iDb = sqlite3SchemaToIndex(db, pTab->pSchema); 006436 pLoop = pLevel->pWLoop; 006437 if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ 006438 /* Do nothing */ 006439 }else 006440 #ifndef SQLITE_OMIT_VIRTUALTABLE 006441 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ 006442 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); 006443 int iCur = pTabItem->iCursor; 006444 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); 006445 }else if( IsVirtual(pTab) ){ 006446 /* noop */ 006447 }else 006448 #endif 006449 if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0 006450 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0) 006451 || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0 006452 ){ 006453 int op = OP_OpenRead; 006454 if( pWInfo->eOnePass!=ONEPASS_OFF ){ 006455 op = OP_OpenWrite; 006456 pWInfo->aiCurOnePass[0] = pTabItem->iCursor; 006457 }; 006458 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); 006459 assert( pTabItem->iCursor==pLevel->iTabCur ); 006460 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); 006461 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); 006462 if( pWInfo->eOnePass==ONEPASS_OFF 006463 && pTab->nCol<BMS 006464 && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0 006465 && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0 006466 ){ 006467 /* If we know that only a prefix of the record will be used, 006468 ** it is advantageous to reduce the "column count" field in 006469 ** the P4 operand of the OP_OpenRead/Write opcode. */ 006470 Bitmask b = pTabItem->colUsed; 006471 int n = 0; 006472 for(; b; b=b>>1, n++){} 006473 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); 006474 assert( n<=pTab->nCol ); 006475 } 006476 #ifdef SQLITE_ENABLE_CURSOR_HINTS 006477 if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){ 006478 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); 006479 }else 006480 #endif 006481 { 006482 sqlite3VdbeChangeP5(v, bFordelete); 006483 } 006484 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK 006485 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, 006486 (const u8*)&pTabItem->colUsed, P4_INT64); 006487 #endif 006488 }else{ 006489 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); 006490 } 006491 if( pLoop->wsFlags & WHERE_INDEXED ){ 006492 Index *pIx = pLoop->u.btree.pIndex; 006493 int iIndexCur; 006494 int op = OP_OpenRead; 006495 /* iAuxArg is always set to a positive value if ONEPASS is possible */ 006496 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); 006497 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) 006498 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 006499 ){ 006500 /* This is one term of an OR-optimization using the PRIMARY KEY of a 006501 ** WITHOUT ROWID table. No need for a separate index */ 006502 iIndexCur = pLevel->iTabCur; 006503 op = 0; 006504 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ 006505 Index *pJ = pTabItem->pTab->pIndex; 006506 iIndexCur = iAuxArg; 006507 assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); 006508 while( ALWAYS(pJ) && pJ!=pIx ){ 006509 iIndexCur++; 006510 pJ = pJ->pNext; 006511 } 006512 op = OP_OpenWrite; 006513 pWInfo->aiCurOnePass[1] = iIndexCur; 006514 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ 006515 iIndexCur = iAuxArg; 006516 op = OP_ReopenIdx; 006517 }else{ 006518 iIndexCur = pParse->nTab++; 006519 if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ 006520 whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); 006521 } 006522 if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){ 006523 wherePartIdxExpr( 006524 pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem 006525 ); 006526 } 006527 } 006528 pLevel->iIdxCur = iIndexCur; 006529 assert( pIx!=0 ); 006530 assert( pIx->pSchema==pTab->pSchema ); 006531 assert( iIndexCur>=0 ); 006532 if( op ){ 006533 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); 006534 sqlite3VdbeSetP4KeyInfo(pParse, pIx); 006535 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 006536 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 006537 && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0 006538 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 006539 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 006540 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED 006541 ){ 006542 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); 006543 } 006544 VdbeComment((v, "%s", pIx->zName)); 006545 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK 006546 { 006547 u64 colUsed = 0; 006548 int ii, jj; 006549 for(ii=0; ii<pIx->nColumn; ii++){ 006550 jj = pIx->aiColumn[ii]; 006551 if( jj<0 ) continue; 006552 if( jj>63 ) jj = 63; 006553 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; 006554 colUsed |= ((u64)1)<<(ii<63 ? ii : 63); 006555 } 006556 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, 006557 (u8*)&colUsed, P4_INT64); 006558 } 006559 #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ 006560 } 006561 } 006562 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); 006563 if( (pTabItem->fg.jointype & JT_RIGHT)!=0 006564 && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0 006565 ){ 006566 WhereRightJoin *pRJ = pLevel->pRJ; 006567 pRJ->iMatch = pParse->nTab++; 006568 pRJ->regBloom = ++pParse->nMem; 006569 sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom); 006570 pRJ->regReturn = ++pParse->nMem; 006571 sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn); 006572 assert( pTab==pTabItem->pTab ); 006573 if( HasRowid(pTab) ){ 006574 KeyInfo *pInfo; 006575 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1); 006576 pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0); 006577 if( pInfo ){ 006578 pInfo->aColl[0] = 0; 006579 pInfo->aSortFlags[0] = 0; 006580 sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO); 006581 } 006582 }else{ 006583 Index *pPk = sqlite3PrimaryKeyIndex(pTab); 006584 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol); 006585 sqlite3VdbeSetP4KeyInfo(pParse, pPk); 006586 } 006587 pLoop->wsFlags &= ~WHERE_IDX_ONLY; 006588 /* The nature of RIGHT JOIN processing is such that it messes up 006589 ** the output order. So omit any ORDER BY/GROUP BY elimination 006590 ** optimizations. We need to do an actual sort for RIGHT JOIN. */ 006591 pWInfo->nOBSat = 0; 006592 pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED; 006593 } 006594 } 006595 pWInfo->iTop = sqlite3VdbeCurrentAddr(v); 006596 if( db->mallocFailed ) goto whereBeginError; 006597 006598 /* Generate the code to do the search. Each iteration of the for 006599 ** loop below generates code for a single nested loop of the VM 006600 ** program. 006601 */ 006602 for(ii=0; ii<nTabList; ii++){ 006603 int addrExplain; 006604 int wsFlags; 006605 SrcItem *pSrc; 006606 if( pParse->nErr ) goto whereBeginError; 006607 pLevel = &pWInfo->a[ii]; 006608 wsFlags = pLevel->pWLoop->wsFlags; 006609 pSrc = &pTabList->a[pLevel->iFrom]; 006610 if( pSrc->fg.isMaterialized ){ 006611 if( pSrc->fg.isCorrelated ){ 006612 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); 006613 }else{ 006614 int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); 006615 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); 006616 sqlite3VdbeJumpHere(v, iOnce); 006617 } 006618 } 006619 assert( pTabList == pWInfo->pTabList ); 006620 if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ 006621 if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){ 006622 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX 006623 constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel); 006624 #endif 006625 }else{ 006626 sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady); 006627 } 006628 if( db->mallocFailed ) goto whereBeginError; 006629 } 006630 addrExplain = sqlite3WhereExplainOneScan( 006631 pParse, pTabList, pLevel, wctrlFlags 006632 ); 006633 pLevel->addrBody = sqlite3VdbeCurrentAddr(v); 006634 notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady); 006635 pWInfo->iContinue = pLevel->addrCont; 006636 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){ 006637 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); 006638 } 006639 } 006640 006641 /* Done. */ 006642 VdbeModuleComment((v, "Begin WHERE-core")); 006643 pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v); 006644 return pWInfo; 006645 006646 /* Jump here if malloc fails */ 006647 whereBeginError: 006648 if( pWInfo ){ 006649 pParse->nQueryLoop = pWInfo->savedNQueryLoop; 006650 whereInfoFree(db, pWInfo); 006651 } 006652 #ifdef WHERETRACE_ENABLED 006653 /* Prevent harmless compiler warnings about debugging routines 006654 ** being declared but never used */ 006655 sqlite3ShowWhereLoopList(0); 006656 #endif /* WHERETRACE_ENABLED */ 006657 return 0; 006658 } 006659 006660 /* 006661 ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the 006662 ** index rather than the main table. In SQLITE_DEBUG mode, we want 006663 ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine 006664 ** does that. 006665 */ 006666 #ifndef SQLITE_DEBUG 006667 # define OpcodeRewriteTrace(D,K,P) /* no-op */ 006668 #else 006669 # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P) 006670 static void sqlite3WhereOpcodeRewriteTrace( 006671 sqlite3 *db, 006672 int pc, 006673 VdbeOp *pOp 006674 ){ 006675 if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; 006676 sqlite3VdbePrintOp(0, pc, pOp); 006677 } 006678 #endif 006679 006680 #ifdef SQLITE_DEBUG 006681 /* 006682 ** Return true if cursor iCur is opened by instruction k of the 006683 ** bytecode. Used inside of assert() only. 006684 */ 006685 static int cursorIsOpen(Vdbe *v, int iCur, int k){ 006686 while( k>=0 ){ 006687 VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); 006688 if( pOp->p1!=iCur ) continue; 006689 if( pOp->opcode==OP_Close ) return 0; 006690 if( pOp->opcode==OP_OpenRead ) return 1; 006691 if( pOp->opcode==OP_OpenWrite ) return 1; 006692 if( pOp->opcode==OP_OpenDup ) return 1; 006693 if( pOp->opcode==OP_OpenAutoindex ) return 1; 006694 if( pOp->opcode==OP_OpenEphemeral ) return 1; 006695 } 006696 return 0; 006697 } 006698 #endif /* SQLITE_DEBUG */ 006699 006700 /* 006701 ** Generate the end of the WHERE loop. See comments on 006702 ** sqlite3WhereBegin() for additional information. 006703 */ 006704 void sqlite3WhereEnd(WhereInfo *pWInfo){ 006705 Parse *pParse = pWInfo->pParse; 006706 Vdbe *v = pParse->pVdbe; 006707 int i; 006708 WhereLevel *pLevel; 006709 WhereLoop *pLoop; 006710 SrcList *pTabList = pWInfo->pTabList; 006711 sqlite3 *db = pParse->db; 006712 int iEnd = sqlite3VdbeCurrentAddr(v); 006713 int nRJ = 0; 006714 006715 /* Generate loop termination code. 006716 */ 006717 VdbeModuleComment((v, "End WHERE-core")); 006718 for(i=pWInfo->nLevel-1; i>=0; i--){ 006719 int addr; 006720 pLevel = &pWInfo->a[i]; 006721 if( pLevel->pRJ ){ 006722 /* Terminate the subroutine that forms the interior of the loop of 006723 ** the RIGHT JOIN table */ 006724 WhereRightJoin *pRJ = pLevel->pRJ; 006725 sqlite3VdbeResolveLabel(v, pLevel->addrCont); 006726 pLevel->addrCont = 0; 006727 pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v); 006728 sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1); 006729 VdbeCoverage(v); 006730 nRJ++; 006731 } 006732 pLoop = pLevel->pWLoop; 006733 if( pLevel->op!=OP_Noop ){ 006734 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT 006735 int addrSeek = 0; 006736 Index *pIdx; 006737 int n; 006738 if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED 006739 && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */ 006740 && (pLoop->wsFlags & WHERE_INDEXED)!=0 006741 && (pIdx = pLoop->u.btree.pIndex)->hasStat1 006742 && (n = pLoop->u.btree.nDistinctCol)>0 006743 && pIdx->aiRowLogEst[n]>=36 006744 ){ 006745 int r1 = pParse->nMem+1; 006746 int j, op; 006747 for(j=0; j<n; j++){ 006748 sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j); 006749 } 006750 pParse->nMem += n+1; 006751 op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT; 006752 addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n); 006753 VdbeCoverageIf(v, op==OP_SeekLT); 006754 VdbeCoverageIf(v, op==OP_SeekGT); 006755 sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); 006756 } 006757 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ 006758 /* The common case: Advance to the next row */ 006759 if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); 006760 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); 006761 sqlite3VdbeChangeP5(v, pLevel->p5); 006762 VdbeCoverage(v); 006763 VdbeCoverageIf(v, pLevel->op==OP_Next); 006764 VdbeCoverageIf(v, pLevel->op==OP_Prev); 006765 VdbeCoverageIf(v, pLevel->op==OP_VNext); 006766 if( pLevel->regBignull ){ 006767 sqlite3VdbeResolveLabel(v, pLevel->addrBignull); 006768 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); 006769 VdbeCoverage(v); 006770 } 006771 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT 006772 if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); 006773 #endif 006774 }else if( pLevel->addrCont ){ 006775 sqlite3VdbeResolveLabel(v, pLevel->addrCont); 006776 } 006777 if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){ 006778 struct InLoop *pIn; 006779 int j; 006780 sqlite3VdbeResolveLabel(v, pLevel->addrNxt); 006781 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ 006782 assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull 006783 || pParse->db->mallocFailed ); 006784 sqlite3VdbeJumpHere(v, pIn->addrInTop+1); 006785 if( pIn->eEndLoopOp!=OP_Noop ){ 006786 if( pIn->nPrefix ){ 006787 int bEarlyOut = 006788 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 006789 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0; 006790 if( pLevel->iLeftJoin ){ 006791 /* For LEFT JOIN queries, cursor pIn->iCur may not have been 006792 ** opened yet. This occurs for WHERE clauses such as 006793 ** "a = ? AND b IN (...)", where the index is on (a, b). If 006794 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may 006795 ** never have been coded, but the body of the loop run to 006796 ** return the null-row. So, if the cursor is not open yet, 006797 ** jump over the OP_Next or OP_Prev instruction about to 006798 ** be coded. */ 006799 sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, 006800 sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut); 006801 VdbeCoverage(v); 006802 } 006803 if( bEarlyOut ){ 006804 sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur, 006805 sqlite3VdbeCurrentAddr(v)+2, 006806 pIn->iBase, pIn->nPrefix); 006807 VdbeCoverage(v); 006808 /* Retarget the OP_IsNull against the left operand of IN so 006809 ** it jumps past the OP_IfNoHope. This is because the 006810 ** OP_IsNull also bypasses the OP_Affinity opcode that is 006811 ** required by OP_IfNoHope. */ 006812 sqlite3VdbeJumpHere(v, pIn->addrInTop+1); 006813 } 006814 } 006815 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); 006816 VdbeCoverage(v); 006817 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev); 006818 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next); 006819 } 006820 sqlite3VdbeJumpHere(v, pIn->addrInTop-1); 006821 } 006822 } 006823 sqlite3VdbeResolveLabel(v, pLevel->addrBrk); 006824 if( pLevel->pRJ ){ 006825 sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1); 006826 VdbeCoverage(v); 006827 } 006828 if( pLevel->addrSkip ){ 006829 sqlite3VdbeGoto(v, pLevel->addrSkip); 006830 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); 006831 sqlite3VdbeJumpHere(v, pLevel->addrSkip); 006832 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); 006833 } 006834 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS 006835 if( pLevel->addrLikeRep ){ 006836 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1), 006837 pLevel->addrLikeRep); 006838 VdbeCoverage(v); 006839 } 006840 #endif 006841 if( pLevel->iLeftJoin ){ 006842 int ws = pLoop->wsFlags; 006843 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); 006844 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); 006845 if( (ws & WHERE_IDX_ONLY)==0 ){ 006846 assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); 006847 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); 006848 } 006849 if( (ws & WHERE_INDEXED) 006850 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx) 006851 ){ 006852 if( ws & WHERE_MULTI_OR ){ 006853 Index *pIx = pLevel->u.pCoveringIdx; 006854 int iDb = sqlite3SchemaToIndex(db, pIx->pSchema); 006855 sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb); 006856 sqlite3VdbeSetP4KeyInfo(pParse, pIx); 006857 } 006858 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); 006859 } 006860 if( pLevel->op==OP_Return ){ 006861 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); 006862 }else{ 006863 sqlite3VdbeGoto(v, pLevel->addrFirst); 006864 } 006865 sqlite3VdbeJumpHere(v, addr); 006866 } 006867 VdbeModuleComment((v, "End WHERE-loop%d: %s", i, 006868 pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); 006869 } 006870 006871 assert( pWInfo->nLevel<=pTabList->nSrc ); 006872 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){ 006873 int k, last; 006874 VdbeOp *pOp, *pLastOp; 006875 Index *pIdx = 0; 006876 SrcItem *pTabItem = &pTabList->a[pLevel->iFrom]; 006877 Table *pTab = pTabItem->pTab; 006878 assert( pTab!=0 ); 006879 pLoop = pLevel->pWLoop; 006880 006881 /* Do RIGHT JOIN processing. Generate code that will output the 006882 ** unmatched rows of the right operand of the RIGHT JOIN with 006883 ** all of the columns of the left operand set to NULL. 006884 */ 006885 if( pLevel->pRJ ){ 006886 sqlite3WhereRightJoinLoop(pWInfo, i, pLevel); 006887 continue; 006888 } 006889 006890 /* For a co-routine, change all OP_Column references to the table of 006891 ** the co-routine into OP_Copy of result contained in a register. 006892 ** OP_Rowid becomes OP_Null. 006893 */ 006894 if( pTabItem->fg.viaCoroutine ){ 006895 testcase( pParse->db->mallocFailed ); 006896 translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, 006897 pTabItem->regResult, 0); 006898 continue; 006899 } 006900 006901 /* If this scan uses an index, make VDBE code substitutions to read data 006902 ** from the index instead of from the table where possible. In some cases 006903 ** this optimization prevents the table from ever being read, which can 006904 ** yield a significant performance boost. 006905 ** 006906 ** Calls to the code generator in between sqlite3WhereBegin and 006907 ** sqlite3WhereEnd will have created code that references the table 006908 ** directly. This loop scans all that code looking for opcodes 006909 ** that reference the table and converts them into opcodes that 006910 ** reference the index. 006911 */ 006912 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ 006913 pIdx = pLoop->u.btree.pIndex; 006914 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ 006915 pIdx = pLevel->u.pCoveringIdx; 006916 } 006917 if( pIdx 006918 && !db->mallocFailed 006919 ){ 006920 if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){ 006921 last = iEnd; 006922 }else{ 006923 last = pWInfo->iEndWhere; 006924 } 006925 if( pIdx->bHasExpr ){ 006926 IndexedExpr *p = pParse->pIdxEpr; 006927 while( p ){ 006928 if( p->iIdxCur==pLevel->iIdxCur ){ 006929 #ifdef WHERETRACE_ENABLED 006930 if( sqlite3WhereTrace & 0x200 ){ 006931 sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n", 006932 p->iIdxCur, p->iIdxCol); 006933 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr); 006934 } 006935 #endif 006936 p->iDataCur = -1; 006937 p->iIdxCur = -1; 006938 } 006939 p = p->pIENext; 006940 } 006941 } 006942 k = pLevel->addrBody + 1; 006943 #ifdef SQLITE_DEBUG 006944 if( db->flags & SQLITE_VdbeAddopTrace ){ 006945 printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n", 006946 pLevel->iTabCur, pLevel->iIdxCur, k, last-1); 006947 } 006948 /* Proof that the "+1" on the k value above is safe */ 006949 pOp = sqlite3VdbeGetOp(v, k - 1); 006950 assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur ); 006951 assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur ); 006952 assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur ); 006953 #endif 006954 pOp = sqlite3VdbeGetOp(v, k); 006955 pLastOp = pOp + (last - k); 006956 assert( pOp<=pLastOp ); 006957 do{ 006958 if( pOp->p1!=pLevel->iTabCur ){ 006959 /* no-op */ 006960 }else if( pOp->opcode==OP_Column 006961 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC 006962 || pOp->opcode==OP_Offset 006963 #endif 006964 ){ 006965 int x = pOp->p2; 006966 assert( pIdx->pTable==pTab ); 006967 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC 006968 if( pOp->opcode==OP_Offset ){ 006969 /* Do not need to translate the column number */ 006970 }else 006971 #endif 006972 if( !HasRowid(pTab) ){ 006973 Index *pPk = sqlite3PrimaryKeyIndex(pTab); 006974 x = pPk->aiColumn[x]; 006975 assert( x>=0 ); 006976 }else{ 006977 testcase( x!=sqlite3StorageColumnToTable(pTab,x) ); 006978 x = sqlite3StorageColumnToTable(pTab,x); 006979 } 006980 x = sqlite3TableColumnToIndex(pIdx, x); 006981 if( x>=0 ){ 006982 pOp->p2 = x; 006983 pOp->p1 = pLevel->iIdxCur; 006984 OpcodeRewriteTrace(db, k, pOp); 006985 }else{ 006986 /* Unable to translate the table reference into an index 006987 ** reference. Verify that this is harmless - that the 006988 ** table being referenced really is open. 006989 */ 006990 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC 006991 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 006992 || cursorIsOpen(v,pOp->p1,k) 006993 || pOp->opcode==OP_Offset 006994 ); 006995 #else 006996 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 006997 || cursorIsOpen(v,pOp->p1,k) 006998 ); 006999 #endif 007000 } 007001 }else if( pOp->opcode==OP_Rowid ){ 007002 pOp->p1 = pLevel->iIdxCur; 007003 pOp->opcode = OP_IdxRowid; 007004 OpcodeRewriteTrace(db, k, pOp); 007005 }else if( pOp->opcode==OP_IfNullRow ){ 007006 pOp->p1 = pLevel->iIdxCur; 007007 OpcodeRewriteTrace(db, k, pOp); 007008 } 007009 #ifdef SQLITE_DEBUG 007010 k++; 007011 #endif 007012 }while( (++pOp)<pLastOp ); 007013 #ifdef SQLITE_DEBUG 007014 if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n"); 007015 #endif 007016 } 007017 } 007018 007019 /* The "break" point is here, just past the end of the outer loop. 007020 ** Set it. 007021 */ 007022 sqlite3VdbeResolveLabel(v, pWInfo->iBreak); 007023 007024 /* Final cleanup 007025 */ 007026 pParse->nQueryLoop = pWInfo->savedNQueryLoop; 007027 whereInfoFree(db, pWInfo); 007028 pParse->withinRJSubrtn -= nRJ; 007029 return; 007030 }