From 060ae29eafbca089824cfbcedc9bc36b7fda5d5e Mon Sep 17 00:00:00 2001 From: Kahrl Date: Sun, 4 Jan 2015 05:10:07 +0100 Subject: [PATCH] Update bundled SQLite to 3.8.7.4 --- src/sqlite/sqlite3.c | 69146 +++++++++++++++++++++++++++-------------- src/sqlite/sqlite3.h | 1424 +- 2 files changed, 47336 insertions(+), 23234 deletions(-) diff --git a/src/sqlite/sqlite3.c b/src/sqlite/sqlite3.c index 2c426c21e..81e9c733c 100644 --- a/src/sqlite/sqlite3.c +++ b/src/sqlite/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.7.1. By combining all the individual C code files into this +** version 3.8.7.4. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -60,6 +60,11 @@ ** in Red Hat 6.0, so the code won't work. Hence, for maximum binary ** portability you should omit LFS. ** +** The previous paragraph was written in 2005. (This paragraph is written +** on 2008-11-28.) These days, all Linux kernels support large files, so +** you should probably leave LFS enabled. But some embedded platforms might +** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. +** ** Similar is true for Mac OS X. LFS is only supported on Mac OS X 9 and later. */ #ifndef SQLITE_DISABLE_LFS @@ -70,475 +75,51 @@ # define _LARGEFILE_SOURCE 1 #endif -/* -** Include the configuration header output by 'configure' if we're using the -** autoconf-based build -*/ -#ifdef _HAVE_SQLITE_CONFIG_H -#include "config.h" -#endif - -/************** Include sqliteLimit.h in the middle of sqliteInt.h ***********/ -/************** Begin file sqliteLimit.h *************************************/ -/* -** 2007 May 7 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file defines various limits of what SQLite can process. -*/ - -/* -** The maximum length of a TEXT or BLOB in bytes. This also -** limits the size of a row in a table or index. -** -** The hard limit is the ability of a 32-bit signed integer -** to count the size: 2^31-1 or 2147483647. -*/ -#ifndef SQLITE_MAX_LENGTH -# define SQLITE_MAX_LENGTH 1000000000 -#endif - -/* -** This is the maximum number of -** -** * Columns in a table -** * Columns in an index -** * Columns in a view -** * Terms in the SET clause of an UPDATE statement -** * Terms in the result set of a SELECT statement -** * Terms in the GROUP BY or ORDER BY clauses of a SELECT statement. -** * Terms in the VALUES clause of an INSERT statement -** -** The hard upper limit here is 32676. Most database people will -** tell you that in a well-normalized database, you usually should -** not have more than a dozen or so columns in any table. And if -** that is the case, there is no point in having more than a few -** dozen values in any of the other situations described above. -*/ -#ifndef SQLITE_MAX_COLUMN -# define SQLITE_MAX_COLUMN 2000 -#endif - -/* -** The maximum length of a single SQL statement in bytes. -** -** It used to be the case that setting this value to zero would -** turn the limit off. That is no longer true. It is not possible -** to turn this limit off. -*/ -#ifndef SQLITE_MAX_SQL_LENGTH -# define SQLITE_MAX_SQL_LENGTH 1000000000 -#endif - -/* -** The maximum depth of an expression tree. This is limited to -** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might -** want to place more severe limits on the complexity of an -** expression. -** -** A value of 0 used to mean that the limit was not enforced. -** But that is no longer true. The limit is now strictly enforced -** at all times. -*/ -#ifndef SQLITE_MAX_EXPR_DEPTH -# define SQLITE_MAX_EXPR_DEPTH 1000 -#endif - -/* -** The maximum number of terms in a compound SELECT statement. -** The code generator for compound SELECT statements does one -** level of recursion for each term. A stack overflow can result -** if the number of terms is too large. In practice, most SQL -** never has more than 3 or 4 terms. Use a value of 0 to disable -** any limit on the number of terms in a compount SELECT. -*/ -#ifndef SQLITE_MAX_COMPOUND_SELECT -# define SQLITE_MAX_COMPOUND_SELECT 500 -#endif - -/* -** The maximum number of opcodes in a VDBE program. -** Not currently enforced. -*/ -#ifndef SQLITE_MAX_VDBE_OP -# define SQLITE_MAX_VDBE_OP 25000 -#endif - -/* -** The maximum number of arguments to an SQL function. -*/ -#ifndef SQLITE_MAX_FUNCTION_ARG -# define SQLITE_MAX_FUNCTION_ARG 127 -#endif - -/* -** The maximum number of in-memory pages to use for the main database -** table and for temporary tables. The SQLITE_DEFAULT_CACHE_SIZE -*/ -#ifndef SQLITE_DEFAULT_CACHE_SIZE -# define SQLITE_DEFAULT_CACHE_SIZE 2000 -#endif -#ifndef SQLITE_DEFAULT_TEMP_CACHE_SIZE -# define SQLITE_DEFAULT_TEMP_CACHE_SIZE 500 -#endif - -/* -** The default number of frames to accumulate in the log file before -** checkpointing the database in WAL mode. -*/ -#ifndef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT -# define SQLITE_DEFAULT_WAL_AUTOCHECKPOINT 1000 -#endif - -/* -** The maximum number of attached databases. This must be between 0 -** and 62. The upper bound on 62 is because a 64-bit integer bitmap -** is used internally to track attached databases. -*/ -#ifndef SQLITE_MAX_ATTACHED -# define SQLITE_MAX_ATTACHED 10 -#endif - - -/* -** The maximum value of a ?nnn wildcard that the parser will accept. -*/ -#ifndef SQLITE_MAX_VARIABLE_NUMBER -# define SQLITE_MAX_VARIABLE_NUMBER 999 -#endif - -/* Maximum page size. The upper bound on this value is 65536. This a limit -** imposed by the use of 16-bit offsets within each page. -** -** Earlier versions of SQLite allowed the user to change this value at -** compile time. This is no longer permitted, on the grounds that it creates -** a library that is technically incompatible with an SQLite library -** compiled with a different limit. If a process operating on a database -** with a page-size of 65536 bytes crashes, then an instance of SQLite -** compiled with the default page-size limit will not be able to rollback -** the aborted transaction. This could lead to database corruption. -*/ -#ifdef SQLITE_MAX_PAGE_SIZE -# undef SQLITE_MAX_PAGE_SIZE -#endif -#define SQLITE_MAX_PAGE_SIZE 65536 - - -/* -** The default size of a database page. -*/ -#ifndef SQLITE_DEFAULT_PAGE_SIZE -# define SQLITE_DEFAULT_PAGE_SIZE 1024 -#endif -#if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE -# undef SQLITE_DEFAULT_PAGE_SIZE -# define SQLITE_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE -#endif - -/* -** Ordinarily, if no value is explicitly provided, SQLite creates databases -** with page size SQLITE_DEFAULT_PAGE_SIZE. However, based on certain -** device characteristics (sector-size and atomic write() support), -** SQLite may choose a larger value. This constant is the maximum value -** SQLite will choose on its own. -*/ -#ifndef SQLITE_MAX_DEFAULT_PAGE_SIZE -# define SQLITE_MAX_DEFAULT_PAGE_SIZE 8192 -#endif -#if SQLITE_MAX_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE -# undef SQLITE_MAX_DEFAULT_PAGE_SIZE -# define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE -#endif - - -/* -** Maximum number of pages in one database file. -** -** This is really just the default value for the max_page_count pragma. -** This value can be lowered (or raised) at run-time using that the -** max_page_count macro. -*/ -#ifndef SQLITE_MAX_PAGE_COUNT -# define SQLITE_MAX_PAGE_COUNT 1073741823 -#endif - -/* -** Maximum length (in bytes) of the pattern in a LIKE or GLOB -** operator. -*/ -#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH -# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 -#endif - -/* -** Maximum depth of recursion for triggers. -** -** A value of 1 means that a trigger program will not be able to itself -** fire any triggers. A value of 0 means that no trigger programs at all -** may be executed. -*/ -#ifndef SQLITE_MAX_TRIGGER_DEPTH -# define SQLITE_MAX_TRIGGER_DEPTH 1000 -#endif - -/************** End of sqliteLimit.h *****************************************/ -/************** Continuing where we left off in sqliteInt.h ******************/ - -/* Disable nuisance warnings on Borland compilers */ -#if defined(__BORLANDC__) -#pragma warn -rch /* unreachable code */ -#pragma warn -ccc /* Condition is always true or false */ -#pragma warn -aus /* Assigned value is never used */ -#pragma warn -csu /* Comparing signed and unsigned */ -#pragma warn -spa /* Suspicious pointer arithmetic */ -#endif - /* Needed for various definitions... */ -#ifndef _GNU_SOURCE +#if defined(__GNUC__) && !defined(_GNU_SOURCE) # define _GNU_SOURCE #endif -/* -** Include standard header files as necessary -*/ -#ifdef HAVE_STDINT_H -#include -#endif -#ifdef HAVE_INTTYPES_H -#include +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE #endif /* -** The number of samples of an index that SQLite takes in order to -** construct a histogram of the table content when running ANALYZE -** and with SQLITE_ENABLE_STAT2 +** For MinGW, check to see if we can include the header file containing its +** version information, among other things. Normally, this internal MinGW +** header file would [only] be included automatically by other MinGW header +** files; however, the contained version information is now required by this +** header file to work around binary compatibility issues (see below) and +** this is the only known way to reliably obtain it. This entire #if block +** would be completely unnecessary if there was any other way of detecting +** MinGW via their preprocessor (e.g. if they customized their GCC to define +** some MinGW-specific macros). When compiling for MinGW, either the +** _HAVE_MINGW_H or _HAVE__MINGW_H (note the extra underscore) macro must be +** defined; otherwise, detection of conditions specific to MinGW will be +** disabled. */ -#define SQLITE_INDEX_SAMPLES 10 - -/* -** The following macros are used to cast pointers to integers and -** integers to pointers. The way you do this varies from one compiler -** to the next, so we have developed the following set of #if statements -** to generate appropriate macros for a wide range of compilers. -** -** The correct "ANSI" way to do this is to use the intptr_t type. -** Unfortunately, that typedef is not available on all compilers, or -** if it is available, it requires an #include of specific headers -** that vary from one machine to the next. -** -** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on -** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)). -** So we have to define the macros in different ways depending on the -** compiler. -*/ -#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ -# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) -#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ -# define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) -# define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) -#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ -# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) -#else /* Generates a warning - but it always works */ -# define SQLITE_INT_TO_PTR(X) ((void*)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(X)) +#if defined(_HAVE_MINGW_H) +# include "mingw.h" +#elif defined(_HAVE__MINGW_H) +# include "_mingw.h" #endif /* -** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. -** 0 means mutexes are permanently disable and the library is never -** threadsafe. 1 means the library is serialized which is the highest -** level of threadsafety. 2 means the libary is multithreaded - multiple -** threads can use SQLite as long as no two threads try to use the same -** database connection at the same time. -** -** Older versions of SQLite used an optional THREADSAFE macro. -** We support that for legacy. +** For MinGW version 4.x (and higher), check to see if the _USE_32BIT_TIME_T +** define is required to maintain binary compatibility with the MSVC runtime +** library in use (e.g. for Windows XP). */ -#if !defined(SQLITE_THREADSAFE) -#if defined(THREADSAFE) -# define SQLITE_THREADSAFE THREADSAFE -#else -# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ -#endif +#if !defined(_USE_32BIT_TIME_T) && !defined(_USE_64BIT_TIME_T) && \ + defined(_WIN32) && !defined(_WIN64) && \ + defined(__MINGW_MAJOR_VERSION) && __MINGW_MAJOR_VERSION >= 4 && \ + defined(__MSVCRT__) +# define _USE_32BIT_TIME_T #endif -/* -** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. -** It determines whether or not the features related to -** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can -** be overridden at runtime using the sqlite3_config() API. +/* The public SQLite interface. The _FILE_OFFSET_BITS macro must appear +** first in QNX. Also, the _USE_32BIT_TIME_T macro must appear first for +** MinGW. */ -#if !defined(SQLITE_DEFAULT_MEMSTATUS) -# define SQLITE_DEFAULT_MEMSTATUS 1 -#endif - -/* -** Exactly one of the following macros must be defined in order to -** specify which memory allocation subsystem to use. -** -** SQLITE_SYSTEM_MALLOC // Use normal system malloc() -** SQLITE_MEMDEBUG // Debugging version of system malloc() -** -** (Historical note: There used to be several other options, but we've -** pared it down to just these two.) -** -** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as -** the default. -*/ -#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)>1 -# error "At most one of the following compile-time configuration options\ - is allows: SQLITE_SYSTEM_MALLOC, SQLITE_MEMDEBUG" -#endif -#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)==0 -# define SQLITE_SYSTEM_MALLOC 1 -#endif - -/* -** If SQLITE_MALLOC_SOFT_LIMIT is not zero, then try to keep the -** sizes of memory allocations below this value where possible. -*/ -#if !defined(SQLITE_MALLOC_SOFT_LIMIT) -# define SQLITE_MALLOC_SOFT_LIMIT 1024 -#endif - -/* -** We need to define _XOPEN_SOURCE as follows in order to enable -** recursive mutexes on most Unix systems. But Mac OS X is different. -** The _XOPEN_SOURCE define causes problems for Mac OS X we are told, -** so it is omitted there. See ticket #2673. -** -** Later we learn that _XOPEN_SOURCE is poorly or incorrectly -** implemented on some systems. So we avoid defining it at all -** if it is already defined or if it is unneeded because we are -** not doing a threadsafe build. Ticket #2681. -** -** See also ticket #2741. -*/ -#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE_THREADSAFE -# define _XOPEN_SOURCE 500 /* Needed to enable pthread recursive mutexes */ -#endif - -/* -** The TCL headers are only needed when compiling the TCL bindings. -*/ -#if defined(SQLITE_TCL) || defined(TCLSH) -# include -#endif - -/* -** Many people are failing to set -DNDEBUG=1 when compiling SQLite. -** Setting NDEBUG makes the code smaller and run faster. So the following -** lines are added to automatically set NDEBUG unless the -DSQLITE_DEBUG=1 -** option is set. Thus NDEBUG becomes an opt-in rather than an opt-out -** feature. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - -/* -** The testcase() macro is used to aid in coverage testing. When -** doing coverage testing, the condition inside the argument to -** testcase() must be evaluated both true and false in order to -** get full branch coverage. The testcase() macro is inserted -** to help ensure adequate test coverage in places where simple -** condition/decision coverage is inadequate. For example, testcase() -** can be used to make sure boundary values are tested. For -** bitmask tests, testcase() can be used to make sure each bit -** is significant and used at least once. On switch statements -** where multiple cases go to the same block of code, testcase() -** can insure that all cases are evaluated. -** -*/ -#ifdef SQLITE_COVERAGE_TEST -SQLITE_PRIVATE void sqlite3Coverage(int); -# define testcase(X) if( X ){ sqlite3Coverage(__LINE__); } -#else -# define testcase(X) -#endif - -/* -** The TESTONLY macro is used to enclose variable declarations or -** other bits of code that are needed to support the arguments -** within testcase() and assert() macros. -*/ -#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST) -# define TESTONLY(X) X -#else -# define TESTONLY(X) -#endif - -/* -** Sometimes we need a small amount of code such as a variable initialization -** to setup for a later assert() statement. We do not want this code to -** appear when assert() is disabled. The following macro is therefore -** used to contain that setup code. The "VVA" acronym stands for -** "Verification, Validation, and Accreditation". In other words, the -** code within VVA_ONLY() will only run during verification processes. -*/ -#ifndef NDEBUG -# define VVA_ONLY(X) X -#else -# define VVA_ONLY(X) -#endif - -/* -** The ALWAYS and NEVER macros surround boolean expressions which -** are intended to always be true or false, respectively. Such -** expressions could be omitted from the code completely. But they -** are included in a few cases in order to enhance the resilience -** of SQLite to unexpected behavior - to make the code "self-healing" -** or "ductile" rather than being "brittle" and crashing at the first -** hint of unplanned behavior. -** -** In other words, ALWAYS and NEVER are added for defensive code. -** -** When doing coverage testing ALWAYS and NEVER are hard-coded to -** be true and false so that the unreachable code then specify will -** not be counted as untested code. -*/ -#if defined(SQLITE_COVERAGE_TEST) -# define ALWAYS(X) (1) -# define NEVER(X) (0) -#elif !defined(NDEBUG) -# define ALWAYS(X) ((X)?1:(assert(0),0)) -# define NEVER(X) ((X)?(assert(0),1):0) -#else -# define ALWAYS(X) (X) -# define NEVER(X) (X) -#endif - -/* -** Return true (non-zero) if the input is a integer that is too large -** to fit in 32-bits. This macro is used inside of various testcase() -** macros to verify that we have tested SQLite for large-file support. -*/ -#define IS_BIG_INT(X) (((X)&~(i64)0xffffffff)!=0) - -/* -** The macro unlikely() is a hint that surrounds a boolean -** expression that is usually false. Macro likely() surrounds -** a boolean expression that is usually true. GCC is able to -** use these hints to generate better code, sometimes. -*/ -#if defined(__GNUC__) && 0 -# define likely(X) __builtin_expect((X),1) -# define unlikely(X) __builtin_expect((X),0) -#else -# define likely(X) !!(X) -# define unlikely(X) !!(X) -#endif - /************** Include sqlite3.h in the middle of sqliteInt.h ***************/ /************** Begin file sqlite3.h *****************************************/ /* @@ -650,9 +231,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.7.1" -#define SQLITE_VERSION_NUMBER 3007007 -#define SQLITE_SOURCE_ID "2011-06-28 17:39:05 af0d91adf497f5f36ec3813f04235a6e195a605f" +#define SQLITE_VERSION "3.8.7.4" +#define SQLITE_VERSION_NUMBER 3008007 +#define SQLITE_SOURCE_ID "2014-12-09 01:34:36 f66f7a17b78ba617acde90fc810107f34f1a1f2e" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -720,7 +301,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N); ** CAPI3REF: Test To See If The Library Is Threadsafe ** ** ^The sqlite3_threadsafe() function returns zero if and only if -** SQLite was compiled mutexing code omitted due to the +** SQLite was compiled with mutexing code omitted due to the ** [SQLITE_THREADSAFE] compile-time option being set to 0. ** ** SQLite can be compiled with or without mutexes. When @@ -762,7 +343,8 @@ SQLITE_API int sqlite3_threadsafe(void); ** the opaque structure named "sqlite3". It is useful to think of an sqlite3 ** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and ** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] -** is its destructor. There are many other interfaces (such as +** and [sqlite3_close_v2()] are its destructors. There are many other +** interfaces (such as ** [sqlite3_prepare_v2()], [sqlite3_create_function()], and ** [sqlite3_busy_timeout()] to name but three) that are methods on an ** sqlite3 object. @@ -809,28 +391,46 @@ typedef sqlite_uint64 sqlite3_uint64; /* ** CAPI3REF: Closing A Database Connection ** -** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. -** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is -** successfully destroyed and all associated resources are deallocated. +** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors +** for the [sqlite3] object. +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if +** the [sqlite3] object is successfully destroyed and all associated +** resources are deallocated. ** -** Applications must [sqlite3_finalize | finalize] all [prepared statements] -** and [sqlite3_blob_close | close] all [BLOB handles] associated with -** the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close() is called on a [database connection] that still has -** outstanding [prepared statements] or [BLOB handles], then it returns -** SQLITE_BUSY. +** ^If the database connection is associated with unfinalized prepared +** statements or unfinished sqlite3_backup objects then sqlite3_close() +** will leave the database connection open and return [SQLITE_BUSY]. +** ^If sqlite3_close_v2() is called with unfinalized prepared statements +** and/or unfinished sqlite3_backups, then the database connection becomes +** an unusable "zombie" which will automatically be deallocated when the +** last prepared statement is finalized or the last sqlite3_backup is +** finished. The sqlite3_close_v2() interface is intended for use with +** host languages that are garbage collected, and where the order in which +** destructors are called is arbitrary. ** -** ^If [sqlite3_close()] is invoked while a transaction is open, +** Applications should [sqlite3_finalize | finalize] all [prepared statements], +** [sqlite3_blob_close | close] all [BLOB handles], and +** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +** with the [sqlite3] object prior to attempting to close the object. ^If +** sqlite3_close_v2() is called on a [database connection] that still has +** outstanding [prepared statements], [BLOB handles], and/or +** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation +** of resources is deferred until all [prepared statements], [BLOB handles], +** and [sqlite3_backup] objects are also destroyed. +** +** ^If an [sqlite3] object is destroyed while a transaction is open, ** the transaction is automatically rolled back. ** -** The C parameter to [sqlite3_close(C)] must be either a NULL +** The C parameter to [sqlite3_close(C)] and [sqlite3_close_v2(C)] +** must be either a NULL ** pointer or an [sqlite3] object pointer obtained ** from [sqlite3_open()], [sqlite3_open16()], or ** [sqlite3_open_v2()], and not previously closed. -** ^Calling sqlite3_close() with a NULL pointer argument is a -** harmless no-op. +** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer +** argument is a harmless no-op. */ -SQLITE_API int sqlite3_close(sqlite3 *); +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); /* ** The type for a callback function. @@ -894,7 +494,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); **
    **
  • The application must insure that the 1st parameter to sqlite3_exec() ** is a valid and open [database connection]. -**
  • The application must not close [database connection] specified by +**
  • The application must not close the [database connection] specified by ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. @@ -910,16 +510,14 @@ SQLITE_API int sqlite3_exec( /* ** CAPI3REF: Result Codes -** KEYWORDS: SQLITE_OK {error code} {error codes} -** KEYWORDS: {result code} {result codes} +** KEYWORDS: {result code definitions} ** ** Many SQLite functions return an integer result code from the set shown -** here in order to indicates success or failure. +** here in order to indicate success or failure. ** ** New error codes may be added in future versions of SQLite. ** -** See also: [SQLITE_IOERR_READ | extended result codes], -** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes]. +** See also: [extended result code definitions] */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ @@ -949,32 +547,27 @@ SQLITE_API int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ /* ** CAPI3REF: Extended Result Codes -** KEYWORDS: {extended error code} {extended error codes} -** KEYWORDS: {extended result code} {extended result codes} +** KEYWORDS: {extended result code definitions} ** -** In its default configuration, SQLite API routines return one of 26 integer -** [SQLITE_OK | result codes]. However, experience has shown that many of +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of ** these result codes are too coarse-grained. They do not provide as ** much information about problems as programmers might like. In an effort to ** address this, newer versions of SQLite (version 3.3.8 and later) include ** support for additional result codes that provide more detailed information -** about errors. The extended result codes are enabled or disabled +** about errors. These [extended result codes] are enabled or disabled ** on a per database connection basis using the -** [sqlite3_extended_result_codes()] API. -** -** Some of the available extended result codes are listed here. -** One may expect the number of extended result codes will be expand -** over time. Software that uses extended result codes should expect -** to see new result codes in future releases of SQLite. -** -** The SQLITE_OK result code will never be extended. It will always -** be exactly zero. +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. */ #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) @@ -998,12 +591,37 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) +#define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) +#define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) +#define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +#define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -1019,6 +637,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ #define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ #define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MEMORY 0x00000080 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ #define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ #define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ @@ -1038,7 +657,7 @@ SQLITE_API int sqlite3_exec( ** CAPI3REF: Device Characteristics ** ** The xDeviceCharacteristics method of the [sqlite3_io_methods] -** object returns an integer which is a vector of the these +** object returns an integer which is a vector of these ** bit values expressing I/O characteristics of the mass storage ** device that holds the file that the [sqlite3_io_methods] ** refers to. @@ -1052,7 +671,15 @@ SQLITE_API int sqlite3_exec( ** first then the size of the file is extended, never the other ** way around. The SQLITE_IOCAP_SEQUENTIAL property means that ** information is written to disk in the same order as calls -** to xWrite(). +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -1066,6 +693,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOCAP_SAFE_APPEND 0x00000200 #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 /* ** CAPI3REF: File Locking Levels @@ -1172,7 +801,7 @@ struct sqlite3_file { ** locking strategy (for example to use dot-file locks), to inquire ** about the status of a lock, or to break stale locks. The SQLite ** core reserves all opcodes less than 100 for its own use. -** A [SQLITE_FCNTL_LOCKSTATE | list of opcodes] less than 100 is available. +** A [file control opcodes | list of opcodes] less than 100 is available. ** Applications that define a custom xFileControl method should use opcodes ** greater than 100 to avoid conflicts. VFS implementations should ** return [SQLITE_NOTFOUND] for file control opcodes that they do not @@ -1237,11 +866,15 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; /* ** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} ** ** These integer constants are opcodes for the xFileControl method ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] @@ -1254,7 +887,8 @@ struct sqlite3_io_methods { ** into an integer that the pArg argument points to. This capability ** is used during testing and only needs to be supported when SQLITE_TEST ** is defined. -** +**
      +**
    • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS ** layer a hint of how large the database file will grow to be during the ** current transaction. This hint is not guaranteed to be accurate but it @@ -1262,6 +896,7 @@ struct sqlite3_io_methods { ** file space based on this hint in order to help writes to the database ** file run faster. ** +**
    • [[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified ** by the user. The fourth argument to [sqlite3_file_control()] should @@ -1270,30 +905,195 @@ struct sqlite3_io_methods { ** chunks (say 1MB at a time), may reduce file-system fragmentation and ** improve performance on some systems. ** +**
    • [[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database ** connection. See the [sqlite3_file_control()] documentation for ** additional information. ** -** ^(The [SQLITE_FCNTL_SYNC_OMITTED] opcode is generated internally by -** SQLite and sent to all VFSes in place of a call to the xSync method -** when the database connection has [PRAGMA synchronous] set to OFF.)^ -** Some specialized VFSes need this signal in order to operate correctly -** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most -** VFSes do not need this signal and should silently ignore this opcode. -** Applications should not call [sqlite3_file_control()] with this -** opcode as doing so may disrupt the operation of the specialized VFSes -** that do require it. +**
    • [[SQLITE_FCNTL_SYNC_OMITTED]] +** No longer in use. +** +**
    • [[SQLITE_FCNTL_SYNC]] +** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and +** sent to the VFS immediately before the xSync method is invoked on a +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** of the xSync method. In most cases, the pointer argument passed with +** this file-control is NULL. However, if the database file is being synced +** as part of a multi-database commit, the argument points to a nul-terminated +** string containing the transactions master-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. +** +**
    • [[SQLITE_FCNTL_COMMIT_PHASETWO]] +** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite +** and sent to the VFS after a transaction has been committed immediately +** but before the database is unlocked. VFSes that do not need this signal +** should silently ignore this opcode. Applications should not call +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. +** +**
    • [[SQLITE_FCNTL_WIN32_AV_RETRY]] +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to provide robustness in the presence of +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete operations up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows these two values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer i the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** +**
    • [[SQLITE_FCNTL_PERSIST_WAL]] +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary +** write ahead log and shared memory files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +**
    • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +**
    • [[SQLITE_FCNTL_OVERWRITE]] +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. +** +**
    • [[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +**
    • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** +**
    • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
    • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
    • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** +**
    • [[SQLITE_FCNTL_TRACE]] +** The [SQLITE_FCNTL_TRACE] file control provides advisory information +** to the VFS about what the higher layers of the SQLite stack are doing. +** This file control is used by some VFS activity tracing [shims]. +** The argument is a zero-terminated string. Higher layers in the +** SQLite stack may generate instances of this file control if +** the [SQLITE_USE_FCNTL_TRACE] compile-time option is enabled. +** +**
    • [[SQLITE_FCNTL_HAS_MOVED]] +** The [SQLITE_FCNTL_HAS_MOVED] file control interprets its argument as a +** pointer to an integer and it writes a boolean into that integer depending +** on whether or not the file has been renamed, moved, or deleted since it +** was first opened. +** +**
    • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** +**
    */ -#define SQLITE_FCNTL_LOCKSTATE 1 -#define SQLITE_GET_LOCKPROXYFILE 2 -#define SQLITE_SET_LOCKPROXYFILE 3 -#define SQLITE_LAST_ERRNO 4 -#define SQLITE_FCNTL_SIZE_HINT 5 -#define SQLITE_FCNTL_CHUNK_SIZE 6 -#define SQLITE_FCNTL_FILE_POINTER 7 -#define SQLITE_FCNTL_SYNC_OMITTED 8 - +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_GET_LOCKPROXYFILE 2 +#define SQLITE_SET_LOCKPROXYFILE 3 +#define SQLITE_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_TRACE 19 +#define SQLITE_FCNTL_HAS_MOVED 20 +#define SQLITE_FCNTL_SYNC 21 +#define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 /* ** CAPI3REF: Mutex Handle @@ -1348,7 +1148,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** from xFullPathname() with an optional suffix added. ** ^If a suffix is added to the zFilename parameter, it will ** consist of a single "-" character followed by no more than -** 10 alphanumeric and/or "-" characters. +** 11 alphanumeric and/or "-" characters. ** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, @@ -1721,16 +1521,10 @@ SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); ** order to verify that SQLite recovers gracefully from such ** conditions. ** -** The xMalloc and xFree methods must work like the -** malloc() and free() functions from the standard C library. -** The xRealloc method must work like realloc() from the standard C library -** with the exception that if the second argument to xRealloc is zero, -** xRealloc must be a no-op - it must not perform any allocation or -** deallocation. ^SQLite guarantees that the second argument to +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to ** xRealloc is always a value returned by a prior call to xRoundup. -** And so in cases where xRoundup always returns a positive number, -** xRealloc can perform exactly as the standard library realloc() and -** still be in compliance with this specification. ** ** xSize should return the allocated size of a memory allocation ** previously obtained from xMalloc or xRealloc. The allocated size @@ -1744,7 +1538,7 @@ SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); ** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, ** that causes the corresponding memory allocation to fail. ** -** The xInit method initializes the memory allocator. (For example, +** The xInit method initializes the memory allocator. For example, ** it might allocate any require mutexes or initialize internal data ** structures. The xShutdown method is invoked (indirectly) by ** [sqlite3_shutdown()] and should deallocate any resources acquired @@ -1885,7 +1679,7 @@ struct sqlite3_mem_methods { **
    ^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. +** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. ** There are three arguments to this option: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page @@ -1916,8 +1710,8 @@ struct sqlite3_mem_methods { ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. -** The minimum allocation size is capped at 2^12. Reasonable values -** for the minimum allocation size are 2^5 through 2^8.
    +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8. ** ** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    **
    ^(This option takes a single argument which is a pointer to an @@ -1954,19 +1748,21 @@ struct sqlite3_mem_methods { ** verb to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
    ** -** [[SQLITE_CONFIG_PCACHE]]
    SQLITE_CONFIG_PCACHE
    +** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    **
    ^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods] object. This object specifies the interface +** an [sqlite3_pcache_methods2] object. This object specifies the interface ** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.
    ** -** [[SQLITE_CONFIG_GETPCACHE]]
    SQLITE_CONFIG_GETPCACHE
    +** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    **
    ^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods] object. SQLite copies of the current +** [sqlite3_pcache_methods2] object. SQLite copies of the current ** page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    -**
    ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -1984,17 +1780,73 @@ struct sqlite3_mem_methods { ** function must be threadsafe.
    ** ** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI -**
    This option takes a single argument of type int. If non-zero, then +**
    ^(This option takes a single argument of type int. If non-zero, then ** URI handling is globally enabled. If the parameter is zero, then URI handling -** is globally disabled. If URI handling is globally enabled, all filenames +** is globally disabled.)^ ^If URI handling is globally enabled, all filenames ** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database -** connection is opened. If it is globally disabled, filenames are +** connection is opened. ^If it is globally disabled, filenames are ** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the -** database connection is opened. By default, URI handling is globally +** database connection is opened. ^(By default, URI handling is globally ** disabled. The default value may be changed by compiling with the -** [SQLITE_USE_URI] symbol defined. +** [SQLITE_USE_URI] symbol defined.)^ +** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN +**
    ^This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. ^The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +**
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +**
    These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. +**
    +** +** [[SQLITE_CONFIG_SQLLOG]] +**
    SQLITE_CONFIG_SQLLOG +**
    This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    ^SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** ^The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ +** ^If either argument to this option is negative, then that argument is +** changed to its compile-time default. +** +** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] +**
    SQLITE_CONFIG_WIN32_HEAPSIZE +**
    ^This option is only available if SQLite is compiled for Windows +** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. +** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** that specifies the maximum size of the created heap. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2010,10 +1862,16 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ /* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ #define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ -#define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ #define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ #define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2090,19 +1948,21 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); /* ** CAPI3REF: Last Insert Rowid ** -** ^Each entry in an SQLite table has a unique 64-bit signed +** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) +** has a unique 64-bit signed ** integer key called the [ROWID | "rowid"]. ^The rowid is always available ** as an undeclared column named ROWID, OID, or _ROWID_ as long as those ** names are not also used by explicitly declared columns. ^If ** the table has a column of type [INTEGER PRIMARY KEY] then that column ** is another alias for the rowid. ** -** ^This routine returns the [rowid] of the most recent -** successful [INSERT] into the database from the [database connection] -** in the first argument. ^As of SQLite version 3.7.7, this routines -** records the last insert rowid of both ordinary tables and [virtual tables]. -** ^If no successful [INSERT]s -** have ever occurred on that database connection, zero is returned. +** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the +** most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. +** ^Inserts into [WITHOUT ROWID] tables are not recorded. +** ^If no successful [INSERT]s into rowid tables +** have ever occurred on the database connection D, +** then sqlite3_last_insert_rowid(D) returns zero. ** ** ^(If an [INSERT] occurs within a trigger or within a [virtual table] ** method, then this routine will return the [rowid] of the inserted @@ -2294,27 +2154,33 @@ SQLITE_API int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors ** -** ^This routine sets a callback function that might be invoked whenever -** an attempt is made to open a database table that another thread -** or process has locked. +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. ** -** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] +** ^If the busy callback is NULL, then [SQLITE_BUSY] ** is returned immediately upon encountering the lock. ^If the busy callback ** is not NULL, then the callback might be invoked with two arguments. ** ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for this locking event. ^If the +** been invoked for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. +** access the database and [SQLITE_BUSY] is returned +** to the application. ** ^If the callback returns non-zero, then another attempt -** is made to open the database for reading and the cycle repeats. +** is made to access the database and the cycle repeats. ** ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. +** to the application instead of invoking the +** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -2328,28 +2194,15 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ** ^The default busy callback is NULL. ** -** ^The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] -** when SQLite is in the middle of a large transaction where all the -** changes will not fit into the in-memory cache. SQLite will -** already hold a RESERVED lock on the database file, but it needs -** to promote this lock to EXCLUSIVE so that it can spill cache -** pages into the database file without harm to concurrent -** readers. ^If it is unable to promote the lock, then the in-memory -** cache will be left in an inconsistent state and so the error -** code is promoted from the relatively benign [SQLITE_BUSY] to -** the more severe [SQLITE_IOERR_BLOCKED]. ^This error code promotion -** forces an automatic rollback of the changes. See the -** -** CorruptionFollowingBusyError wiki page for a discussion of why -** this is important. -** ** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any ** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] -** will also set or clear the busy handler. +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. ** ** The busy callback should not take any actions which modify the -** database connection that invoked the busy handler. Any such actions +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions ** result in undefined behavior. ** ** A busy handler must not close the database connection @@ -2365,15 +2218,17 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** will sleep multiple times until at least "ms" milliseconds of sleeping ** have accumulated. ^After at least "ms" milliseconds of sleeping, ** the handler returns 0 which causes [sqlite3_step()] to return -** [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. +** [SQLITE_BUSY]. ** ** ^Calling this routine with an argument less than or equal to zero ** turns off all busy handlers. ** ** ^(There can only be a single busy handler for a particular -** [database connection] any any given moment. If another busy handler +** [database connection] at any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); @@ -2498,7 +2353,7 @@ SQLITE_API void sqlite3_free_table(char **result); ** All of the usual printf() formatting options apply. In addition, there ** is are "%q", "%Q", and "%z" options. ** -** ^(The %q option works like %s in that it substitutes a null-terminated +** ^(The %q option works like %s in that it substitutes a nul-terminated ** string from the argument list. But %q also doubles every '\'' character. ** %q is designed for use inside a string literal.)^ By doubling each '\'' ** character it escapes that character and allows it to be inserted into @@ -2573,6 +2428,10 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns ** a NULL pointer. ** +** ^The sqlite3_malloc64(N) routine works just like +** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead +** of a signed 32-bit integer. +** ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is @@ -2584,24 +2443,38 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** might result if sqlite3_free() is called with a non-NULL pointer that ** was not obtained from sqlite3_malloc() or sqlite3_realloc(). ** -** ^(The sqlite3_realloc() interface attempts to resize a -** prior memory allocation to be at least N bytes, where N is the -** second parameter. The memory allocation to be resized is the first -** parameter.)^ ^ If the first parameter to sqlite3_realloc() +** ^The sqlite3_realloc(X,N) interface attempts to resize a +** prior memory allocation X to be at least N bytes. +** ^If the X parameter to sqlite3_realloc(X,N) ** is a NULL pointer then its behavior is identical to calling -** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc(). -** ^If the second parameter to sqlite3_realloc() is zero or +** sqlite3_malloc(N). +** ^If the N parameter to sqlite3_realloc(X,N) is zero or ** negative then the behavior is exactly the same as calling -** sqlite3_free(P) where P is the first parameter to sqlite3_realloc(). -** ^sqlite3_realloc() returns a pointer to a memory allocation -** of at least N bytes in size or NULL if sufficient memory is unavailable. +** sqlite3_free(X). +** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation +** of at least N bytes in size or NULL if insufficient memory is available. ** ^If M is the size of the prior allocation, then min(N,M) bytes ** of the prior allocation are copied into the beginning of buffer returned -** by sqlite3_realloc() and the prior allocation is freed. -** ^If sqlite3_realloc() returns NULL, then the prior allocation -** is not freed. +** by sqlite3_realloc(X,N) and the prior allocation is freed. +** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the +** prior allocation is not freed. ** -** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() +** ^The sqlite3_realloc64(X,N) interfaces works the same as +** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead +** of a 32-bit signed integer. +** +** ^If X is a memory allocation previously obtained from sqlite3_malloc(), +** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then +** sqlite3_msize(X) returns the size of that memory allocation in bytes. +** ^The value returned by sqlite3_msize(X) might be larger than the number +** of bytes requested when X was allocated. ^If X is a NULL pointer then +** sqlite3_msize(X) returns zero. If X points to something that is not +** the beginning of memory allocation, or if it points to a formerly +** valid memory allocation that has now been freed, then the behavior +** of sqlite3_msize(X) is undefined and possibly harmful. +** +** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(), +** sqlite3_malloc64(), and sqlite3_realloc64() ** is always aligned to at least an 8 byte boundary, or to a ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time ** option is used. @@ -2611,12 +2484,12 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** implementation of these routines to be omitted. That capability ** is no longer provided. Only built-in memory allocators can be used. ** -** The Windows OS interface layer calls +** Prior to SQLite version 3.7.10, the Windows OS interface layer called ** the system malloc() and free() directly when converting ** filenames between the UTF-8 encoding used by SQLite ** and whatever filename encoding is used by the particular Windows -** installation. Memory allocation errors are detected, but -** they are reported back as [SQLITE_CANTOPEN] or +** installation. Memory allocation errors were detected, but +** they were reported back as [SQLITE_CANTOPEN] or ** [SQLITE_IOERR] rather than [SQLITE_NOMEM]. ** ** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] @@ -2629,8 +2502,11 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** [sqlite3_free()] or [sqlite3_realloc()]. */ SQLITE_API void *sqlite3_malloc(int); +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); SQLITE_API void *sqlite3_realloc(void*, int); +SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); SQLITE_API void sqlite3_free(void*); +SQLITE_API sqlite3_uint64 sqlite3_msize(void*); /* ** CAPI3REF: Memory Allocator Statistics @@ -2668,11 +2544,13 @@ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. +** ^If N is less than one, then P can be a NULL pointer. ** -** ^The first time this routine is invoked (either internally or by -** the application) the PRNG is seeded using randomness obtained -** from the xRandomness method of the default [sqlite3_vfs] object. -** ^On all subsequent invocations, the pseudo-randomness is generated +** ^If this routine has not been previously called or if the previous +** call had N less than one, then the PRNG is seeded using randomness +** obtained from the xRandomness method of the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more then +** the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ @@ -2773,8 +2651,8 @@ SQLITE_API int sqlite3_set_authorizer( ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. ** -** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] -** from the [sqlite3_vtab_on_conflict()] interface. +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -2832,6 +2710,7 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_FUNCTION 31 /* NULL Function Name */ #define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ #define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ /* ** CAPI3REF: Tracing And Profiling Functions @@ -2847,6 +2726,9 @@ SQLITE_API int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -2872,9 +2754,10 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, ** interface is to keep a GUI updated during a large query. ** ** ^The parameter P is passed through as the only parameter to the -** callback function X. ^The parameter N is the number of +** callback function X. ^The parameter N is the approximate number of ** [virtual machine instructions] that are evaluated between successive -** invocations of the callback X. +** invocations of the callback X. ^If N is less than one then the progress +** handler is disabled. ** ** ^Only a single progress handler may be defined at one time per ** [database connection]; setting a new progress handler cancels the @@ -2910,9 +2793,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an English language description of the error following a failure of any ** of the sqlite3_open() routines. ** -** ^The default encoding for the database will be UTF-8 if -** sqlite3_open() or sqlite3_open_v2() is called and -** UTF-16 in the native byte order if sqlite3_open16() is used. +** ^The default encoding will be UTF-8 for databases created using +** sqlite3_open() or sqlite3_open_v2(). ^The default encoding for databases +** created using sqlite3_open16() will be UTF-16 in the native byte order. ** ** Whether or not an error occurs when it is opened, resources ** associated with the [database connection] handle should be released by @@ -3000,13 +2883,14 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** then it is interpreted as an absolute path. ^If the path does not begin ** with a '/' (meaning that the authority section is omitted from the URI) ** then the path is interpreted as a relative path. -** ^On windows, the first component of an absolute path -** is a drive specification (e.g. "C:"). +** ^(On windows, the first component of an absolute path +** is a drive specification (e.g. "C:").)^ ** ** [[core URI query parameters]] ** The query component of a URI may contain parameters that are interpreted ** either by SQLite itself, or by a [VFS | custom VFS implementation]. -** SQLite interprets the following three query parameters: +** SQLite and its built-in [VFSes] interpret the +** following query parameters: ** **
      **
    • vfs: ^The "vfs" parameter may be used to specify the name of @@ -3017,18 +2901,20 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** present, then the VFS specified by the option takes precedence over ** the value passed as the fourth parameter to sqlite3_open_v2(). ** -**
    • mode: ^(The mode parameter may be set to either "ro", "rw" or -** "rwc". Attempting to set it to any other value is an error)^. +**
    • mode: ^(The mode parameter may be set to either "ro", "rw", +** "rwc", or "memory". Attempting to set it to any other value is +** an error)^. ** ^If "ro" is specified, then the database is opened for read-only ** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the -** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** third argument to sqlite3_open_v2(). ^If the mode option is set to ** "rw", then the database is opened for read-write (but not create) ** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had ** been set. ^Value "rwc" is equivalent to setting both -** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If sqlite3_open_v2() is -** used, it is an error to specify a value for the mode parameter that is -** less restrictive than that specified by the flags passed as the third -** parameter. +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If the mode option is +** set to "memory" then a pure [in-memory database] that never reads +** or writes from disk is used. ^It is an error to specify a value for +** the mode parameter that is less restrictive than that specified by +** the flags passed in the third parameter to sqlite3_open_v2(). ** **
    • cache: ^The cache parameter may be set to either "shared" or ** "private". ^Setting it to "shared" is equivalent to setting the @@ -3036,8 +2922,30 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
    • psow: ^The psow parameter indicates whether or not the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. +** +**
    • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
    • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** **
    ** ** ^Specifying an unknown parameter in the query component of a URI is not an @@ -3067,8 +2975,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** Open file "data.db" in the current directory for read-only access. ** Regardless of whether or not shared-cache mode is enabled by ** default, use a private cache. -** file:/home/fred/data.db?vfs=unix-nolock -** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +** file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. ** file:data.db?mode=readonly ** An error. "readonly" is not a valid option for the "mode" parameter. ** @@ -3087,6 +2996,12 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** codepage is currently defined. Filenames containing international ** characters must be converted to UTF-8 prior to passing them into ** sqlite3_open() or sqlite3_open_v2(). +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling sqlite3_open() or sqlite3_open_v2(). Otherwise, various +** features that require the use of temporary files may fail. +** +** See also: [sqlite3_temp_directory] */ SQLITE_API int sqlite3_open( const char *filename, /* Database filename (UTF-8) */ @@ -3106,21 +3021,45 @@ SQLITE_API int sqlite3_open_v2( /* ** CAPI3REF: Obtain Values For URI Parameters ** -** This is a utility routine, useful to VFS implementations, that checks +** These are utility routines, useful to VFS implementations, that check ** to see if a database file was a URI that contained a specific query -** parameter, and if so obtains the value of the query parameter. +** parameter, and if so obtains the value of that query parameter. ** -** The zFilename argument is the filename pointer passed into the xOpen() -** method of a VFS implementation. The zParam argument is the name of the -** query parameter we seek. This routine returns the value of the zParam -** parameter if it exists. If the parameter does not exist, this routine -** returns a NULL pointer. +** If F is the database filename pointer passed into the xOpen() method of +** a VFS implementation when the flags parameter to xOpen() has one or +** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and +** P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. ** -** If the zFilename argument to this function is not a pointer that SQLite -** passed into the xOpen VFS method, then the behavior of this routine -** is undefined and probably undesirable. +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P is does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that SQLite passed into the xOpen +** VFS method, then the behavior of this routine is undefined and probably +** undesirable. */ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); /* @@ -3142,6 +3081,11 @@ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char * ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -3160,6 +3104,7 @@ SQLITE_API int sqlite3_errcode(sqlite3 *db); SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); SQLITE_API const char *sqlite3_errmsg(sqlite3*); SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); /* ** CAPI3REF: SQL Statement Object @@ -3278,6 +3223,10 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
    SQLITE_LIMIT_WORKER_THREADS
    +**
    The maximum number of auxiliary worker threads that a single +** [prepared statement] may start.
    )^ ** */ #define SQLITE_LIMIT_LENGTH 0 @@ -3291,6 +3240,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 /* ** CAPI3REF: Compiling An SQL Statement @@ -3316,7 +3266,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** that the supplied string is nul-terminated, then there is a small ** performance advantage to be gained by passing an nByte parameter that ** is equal to the number of bytes in the input string including -** the nul-terminator bytes. +** the nul-terminator bytes as this saves SQLite from having to +** make a copy of the input string. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -3346,7 +3297,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
    • ** **
  • @@ -3367,8 +3319,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ^The specific value of WHERE-clause [parameter] might influence the ** choice of query plan if the parameter is the left-hand side of a [LIKE] ** or [GLOB] operator or if the parameter is compared to an indexed column -** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. -** the +** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. **
    • ** */ @@ -3441,6 +3392,25 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has not run to completion and/or has not +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); + /* ** CAPI3REF: Dynamically Typed Value Object ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} @@ -3531,18 +3501,31 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the ** number of bytes in the value, not the number of characters.)^ -** ^If the fourth parameter is negative, the length of the string is +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is ** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() or sqlite3_bind_text64() then +** that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occur at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. ** -** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and -** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or +** ^The fifth argument to the BLOB and string binding interfaces +** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), -** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** to dispose of the BLOB or string even if the call to bind API fails. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. @@ -3550,6 +3533,14 @@ typedef struct sqlite3_context sqlite3_context; ** SQLite makes its own private copy of the data immediately, before ** the sqlite3_bind_*() routine returns. ** +** ^The sixth argument to sqlite3_bind_text64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_text64() is not one of the +** allowed values shown above, or if the text encoding is different +** from the encoding specified by the sixth parameter, then the behavior +** is undefined. +** ** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that ** is filled with zeroes. ^A zeroblob uses a fixed amount of memory ** (just an integer to hold its size) while it is being processed. @@ -3570,6 +3561,9 @@ typedef struct sqlite3_context sqlite3_context; ** ** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an ** [error code] if anything goes wrong. +** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB +** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or +** [SQLITE_MAX_LENGTH]. ** ^[SQLITE_RANGE] is returned if the parameter ** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. ** @@ -3577,12 +3571,16 @@ typedef struct sqlite3_context sqlite3_context; ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, + void(*)(void*)); SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); -SQLITE_API int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, + void(*)(void*), unsigned char encoding); SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); @@ -3870,6 +3868,12 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of ** interfaces) then sqlite3_data_count(P) returns 0. ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. ** ** See also: [sqlite3_column_count()] */ @@ -3969,7 +3973,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), -** even empty strings, are always zero terminated. ^The return +** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an @@ -3992,19 +3996,19 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ** NULL INTEGER Result is 0 ** NULL FLOAT Result is 0.0 -** NULL TEXT Result is NULL pointer -** NULL BLOB Result is NULL pointer +** NULL TEXT Result is a NULL pointer +** NULL BLOB Result is a NULL pointer ** INTEGER FLOAT Convert from integer to float ** INTEGER TEXT ASCII rendering of the integer ** INTEGER BLOB Same as INTEGER->TEXT -** FLOAT INTEGER Convert from float to integer +** FLOAT INTEGER [CAST] to INTEGER ** FLOAT TEXT ASCII rendering of the float -** FLOAT BLOB Same as FLOAT->TEXT -** TEXT INTEGER Use atoi() -** TEXT FLOAT Use atof() +** FLOAT BLOB [CAST] to BLOB +** TEXT INTEGER [CAST] to INTEGER +** TEXT FLOAT [CAST] to REAL ** TEXT BLOB No change -** BLOB INTEGER Convert to TEXT then use atoi() -** BLOB FLOAT Convert to TEXT then use atof() +** BLOB INTEGER [CAST] to INTEGER +** BLOB FLOAT [CAST] to REAL ** BLOB TEXT Add a zero terminator if needed ** ** )^ @@ -4060,7 +4064,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings ** and BLOBs is freed automatically. Do not pass the pointers returned -** [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** ** ^(If a memory allocation error occurs during the evaluation of any @@ -4169,15 +4173,24 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Every SQL function implementation must be able to work -** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be -** more efficient with one encoding than another. ^An application may -** invoke sqlite3_create_function() or sqlite3_create_function16() multiple -** times with the same function but with different values of eTextRep. +** its parameters. The application should set this parameter to +** [SQLITE_UTF16LE] if the function implementation invokes +** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the +** implementation invokes [sqlite3_value_text16be()] on an input, or +** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] +** otherwise. ^The same SQL function may be registered multiple times using +** different preferred text encodings, with different implementations for +** each encoding. ** ^When multiple implementations of the same function are available, SQLite ** will pick the one that involves the least amount of data conversion. -** If there is only a single implementation which does not care what text -** encoding is used, then the fourth argument should be [SQLITE_ANY]. +** +** ^The fourth parameter may optionally be ORed with [SQLITE_DETERMINISTIC] +** to signal that the function will always return the same result given +** the same inputs within a single SQL statement. Most SQL functions are +** deterministic. The built-in [random()] SQL function is an example of a +** function that is not deterministic. The SQLite query planner is able to +** perform additional optimizations on deterministic functions, so use +** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. ** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ @@ -4263,9 +4276,19 @@ SQLITE_API int sqlite3_create_function_v2( #define SQLITE_UTF16LE 2 #define SQLITE_UTF16BE 3 #define SQLITE_UTF16 4 /* Use native byte order */ -#define SQLITE_ANY 5 /* sqlite3_create_function only */ +#define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ +/* +** CAPI3REF: Function Flags +** +** These constants may be ORed together with the +** [SQLITE_UTF8 | preferred text encoding] as the fourth argument +** to [sqlite3_create_function()], [sqlite3_create_function16()], or +** [sqlite3_create_function_v2()]. +*/ +#define SQLITE_DETERMINISTIC 0x800 + /* ** CAPI3REF: Deprecated Functions ** DEPRECATED @@ -4282,7 +4305,8 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -4305,7 +4329,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** object results in undefined behavior. ** ** ^These routines work just like the corresponding [column access functions] -** except that these routines take a single [protected sqlite3_value] object +** except that these routines take a single [protected sqlite3_value] object ** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. ** ** ^The sqlite3_value_text16() interface extracts a UTF-16 string @@ -4362,14 +4386,17 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. @@ -4412,41 +4439,49 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data ** -** The following two functions may be used by scalar SQL functions to +** These functions may be used by (non-aggregate) SQL functions to ** associate metadata with argument values. If the same value is passed to ** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated metadata may be preserved. This may -** be used, for example, to add a regular-expression matching scalar -** function. The compiled version of the regular expression is stored as -** metadata associated with the SQL value passed as the regular expression -** pattern. The compiled regular expression can be reused on multiple -** invocations of the same function so that the original pattern string -** does not need to be recompiled on each invocation. +** some circumstances the associated metadata may be preserved. An example +** of where this might be useful is in a regular-expression matching +** function. The compiled version of the regular expression can be stored as +** metadata associated with the pattern string. +** Then as long as the pattern string remains the same, +** the compiled regular expression can be reused on multiple +** invocations of the same function. ** ** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata ** associated by the sqlite3_set_auxdata() function with the Nth argument -** value to the application-defined function. ^If no metadata has been ever -** been set for the Nth argument of the function, or if the corresponding -** function parameter has changed since the meta-data was set, -** then sqlite3_get_auxdata() returns a NULL pointer. +** value to the application-defined function. ^If there is no metadata +** associated with the function argument, this sqlite3_get_auxdata() interface +** returns a NULL pointer. ** -** ^The sqlite3_set_auxdata() interface saves the metadata -** pointed to by its 3rd parameter as the metadata for the N-th -** argument of the application-defined function. Subsequent -** calls to sqlite3_get_auxdata() might return this data, if it has -** not been destroyed. -** ^If it is not NULL, SQLite will invoke the destructor -** function given by the 4th parameter to sqlite3_set_auxdata() on -** the metadata when the corresponding function parameter changes -** or when the SQL statement completes, whichever comes first. +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th +** argument of the application-defined function. ^Subsequent +** calls to sqlite3_get_auxdata(C,N) return P from the most recent +** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or +** NULL if the metadata has been discarded. +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, +** SQLite will invoke the destructor function X with parameter P exactly +** once, when the metadata is discarded. +** SQLite is free to discard the metadata at any time, including:
      +**
    • when the corresponding function parameter changes, or +**
    • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement, or +**
    • when sqlite3_set_auxdata() is invoked again on the same parameter, or +**
    • during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.
    )^ ** -** SQLite is free to call the destructor and drop metadata on any -** parameter of any function at any time. ^The only guarantee is that -** the destructor will be called before the metadata is dropped. +** Note the last bullet in particular. The destructor X in +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the +** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() +** should be called near the end of the function implementation and the +** function implementation should not make any use of P after +** sqlite3_set_auxdata() has been called. ** ** ^(In practice, metadata is preserved between function calls for -** expressions that are constant at compile time. This includes literal -** values and [parameters].)^ +** function parameters that are compile-time constants, including literal +** values and [parameters] and expressions composed from the same.)^ ** ** These routines must be called from the same thread in which ** the SQL function is running. @@ -4467,7 +4502,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -4520,11 +4555,11 @@ typedef void (*sqlite3_destructor_type)(void*); ** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() ** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. ** -** ^The sqlite3_result_toobig() interface causes SQLite to throw an error -** indicating that a string or BLOB is too long to represent. +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. ** -** ^The sqlite3_result_nomem() interface causes SQLite to throw an error -** indicating that a memory allocation failed. +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. ** ** ^The sqlite3_result_int() interface sets the return value ** of the application-defined function to be the 32-bit signed integer @@ -4541,6 +4576,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_text64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces @@ -4549,7 +4588,12 @@ typedef void (*sqlite3_destructor_type)(void*); ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined -** function result. +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. ** ^If the 4th parameter to the sqlite3_result_text* interfaces ** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that ** function as the destructor on the text or BLOB result when it has @@ -4579,6 +4623,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** the [sqlite3_context] pointer, the results are undefined. */ SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); SQLITE_API void sqlite3_result_double(sqlite3_context*, double); SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); @@ -4589,6 +4634,8 @@ SQLITE_API void sqlite3_result_int(sqlite3_context*, int); SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); SQLITE_API void sqlite3_result_null(sqlite3_context*); SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); @@ -4746,6 +4793,11 @@ SQLITE_API int sqlite3_key( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The key */ ); +SQLITE_API int sqlite3_key_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The key */ +); /* ** Change the key on an open database. If the current database is not @@ -4759,6 +4811,11 @@ SQLITE_API int sqlite3_rekey( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The new key */ ); +SQLITE_API int sqlite3_rekey_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The new key */ +); /* ** Specify the activation key for a SEE database. Unless @@ -4808,6 +4865,13 @@ SQLITE_API int sqlite3_sleep(int); ** is a NULL pointer, then SQLite performs a search for an appropriate ** temporary file directory. ** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** ** It is not safe to read or modify this variable in more than one ** thread at a time. It is not safe to read or modify this variable ** if a [database connection] is being used at the same time in a separate @@ -4826,9 +4890,66 @@ SQLITE_API int sqlite3_sleep(int); ** Hence, if this variable is modified directly, either it should be ** made NULL or made to point to memory obtained from [sqlite3_malloc] ** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various +** features that require the use of temporary files may fail. Here is an +** example of how to do this using C++ with the Windows Runtime: +** +** 
    +** LPCWSTR zPath = Windows::Storage::ApplicationData::Current->
+**       TemporaryFolder->Path->Data();
+** char zPathBuf[MAX_PATH + 1];
+** memset(zPathBuf, 0, sizeof(zPathBuf));
+** WideCharToMultiByte(CP_UTF8, 0, zPath, -1, zPathBuf, sizeof(zPathBuf),
+**       NULL, NULL);
+** sqlite3_temp_directory = sqlite3_mprintf("%s", zPathBuf);
+**
    */ SQLITE_API char *sqlite3_temp_directory; +/* +** CAPI3REF: Name Of The Folder Holding Database Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all database files +** specified with a relative pathname and created or accessed by +** SQLite when using a built-in windows [sqlite3_vfs | VFS] will be assumed +** to be relative to that directory.)^ ^If this variable is a NULL +** pointer, then SQLite assumes that all database files specified +** with a relative pathname are relative to the current directory +** for the process. Only the windows VFS makes use of this global +** variable; it is ignored by the unix VFS. +** +** Changing the value of this variable while a database connection is +** open can result in a corrupt database. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [data_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [data_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [data_store_directory pragma] should be avoided. +*/ +SQLITE_API char *sqlite3_data_directory; + /* ** CAPI3REF: Test For Auto-Commit Mode ** KEYWORDS: {autocommit mode} @@ -4864,6 +4985,31 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); */ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +/* +** CAPI3REF: Return The Filename For A Database Connection +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename +** associated with database N of connection D. ^The main database file +** has the name "main". If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** a NULL pointer is returned. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + /* ** CAPI3REF: Find the next prepared statement ** @@ -4899,13 +5045,15 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); ** on the same [database connection] D, or NULL for ** the first call for each function on D. ** +** The commit and rollback hook callbacks are not reentrant. ** The callback implementation must not do anything that will modify ** the database connection that invoked the callback. Any actions ** to modify the database connection must be deferred until after the ** completion of the [sqlite3_step()] call that triggered the commit ** or rollback hook in the first place. -** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their -** database connections for the meaning of "modify" in this paragraph. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. ** ** ^Registering a NULL function disables the callback. ** @@ -4931,12 +5079,13 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument -** to be invoked whenever a row is updated, inserted or deleted. +** to be invoked whenever a row is updated, inserted or deleted in +** a rowid table. ** ^Any callback set by a previous call to this function ** for the same database connection is overridden. ** ** ^The second argument is a pointer to the function to invoke when a -** row is updated, inserted or deleted. +** row is updated, inserted or deleted in a rowid table. ** ^The first argument to the callback is a copy of the third argument ** to sqlite3_update_hook(). ** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], @@ -4949,6 +5098,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ** ^(The update hook is not invoked when internal system tables are ** modified (i.e. sqlite_master and sqlite_sequence).)^ +** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. ** ** ^In the current implementation, the update hook ** is not invoked when duplication rows are deleted because of an @@ -4980,7 +5130,6 @@ SQLITE_API void *sqlite3_update_hook( /* ** CAPI3REF: Enable Or Disable Shared Pager Cache -** KEYWORDS: {shared cache} ** ** ^(This routine enables or disables the sharing of the database cache ** and schema data structures between [database connection | connections] @@ -5003,6 +5152,9 @@ SQLITE_API void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** ** See Also: [SQLite Shared-Cache Mode] */ SQLITE_API int sqlite3_enable_shared_cache(int); @@ -5018,9 +5170,24 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** which might be more or less than the amount requested. ** ^The sqlite3_release_memory() routine is a no-op returning zero ** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** See also: [sqlite3_db_release_memory()] */ SQLITE_API int sqlite3_release_memory(int); +/* +** CAPI3REF: Free Memory Used By A Database Connection +** +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is in effect even +** when the [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. +** +** See also: [sqlite3_release_memory()] +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3*); + /* ** CAPI3REF: Impose A Limit On Heap Size ** @@ -5035,7 +5202,8 @@ SQLITE_API int sqlite3_release_memory(int); ** is advisory only. ** ** ^The return value from sqlite3_soft_heap_limit64() is the size of -** the soft heap limit prior to the call. ^If the argument N is negative +** the soft heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative ** then no change is made to the soft heap limit. Hence, the current ** size of the soft heap limit can be determined by invoking ** sqlite3_soft_heap_limit64() with a negative argument. @@ -5051,7 +5219,7 @@ SQLITE_API int sqlite3_release_memory(int); ** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and ** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. **
  • An alternative page cache implementation is specified using -** [sqlite3_config]([SQLITE_CONFIG_PCACHE],...). +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). **
  • The page cache allocates from its own memory pool supplied ** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than ** from the heap. @@ -5164,11 +5332,20 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -5194,11 +5371,11 @@ SQLITE_API int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -5210,7 +5387,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes @@ -5238,10 +5415,23 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** on the list of automatic extensions is a harmless no-op. ^No entry point ** will be called more than once for each database connection that is opened. ** -** See also: [sqlite3_reset_auto_extension()]. +** See also: [sqlite3_reset_auto_extension()] +** and [sqlite3_cancel_auto_extension()] */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); +/* +** CAPI3REF: Cancel Automatic Extension Loading +** +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the +** initialization routine X that was registered using a prior call to +** [sqlite3_auto_extension(X)]. ^The [sqlite3_cancel_auto_extension(X)] +** routine returns 1 if initialization routine X was successfully +** unregistered and it returns 0 if X was not on the list of initialization +** routines. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); + /* ** CAPI3REF: Reset Automatic Extension Loading ** @@ -5366,10 +5556,22 @@ struct sqlite3_module { ** the correct order to satisfy the ORDER BY clause so that no separate ** sorting step is required. ** -** ^The estimatedCost value is an estimate of the cost of doing the -** particular lookup. A full scan of a table with N entries should have -** a cost of N. A binary search of a table of N entries should have a -** cost of approximately log(N). +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite version 3.8.2. If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to included crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. */ struct sqlite3_index_info { /* Inputs */ @@ -5394,7 +5596,9 @@ struct sqlite3_index_info { char *idxStr; /* String, possibly obtained from sqlite3_malloc */ int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ int orderByConsumed; /* True if output is already ordered */ - double estimatedCost; /* Estimated cost of using this index */ + double estimatedCost; /* Estimated cost of using this index */ + /* Fields below are only available in SQLite 3.8.2 and later */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ }; /* @@ -5598,6 +5802,9 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** +** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] +** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. +** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces ** and the built-in [zeroblob] SQL function can be used, if desired, ** to create an empty, zero-filled blob in which to read or write using @@ -5792,17 +5999,16 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** implementations are available in the SQLite core: ** **
      -**
    • SQLITE_MUTEX_OS2 -**
    • SQLITE_MUTEX_PTHREAD +**
    • SQLITE_MUTEX_PTHREADS **
    • SQLITE_MUTEX_W32 **
    • SQLITE_MUTEX_NOOP **
    )^ ** ** ^The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_OS2, -** SQLITE_MUTEX_PTHREAD, and SQLITE_MUTEX_W32 implementations -** are appropriate for use on OS/2, Unix, and Windows. +** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix +** and Windows. ** ** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex @@ -5823,10 +6029,12 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU -**
  • SQLITE_MUTEX_STATIC_LRU2 +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 **
)^ ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -5991,7 +6199,7 @@ struct sqlite3_mutex_methods { ** ^These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provided versions of these +** ^The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. @@ -6030,6 +6238,9 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection @@ -6119,10 +6330,15 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 -#define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 -#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19 -#define SQLITE_TESTCTRL_LAST 19 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ +#define SQLITE_TESTCTRL_NEVER_CORRUPT 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_SORTER_MMAP 24 +#define SQLITE_TESTCTRL_LAST 24 /* ** CAPI3REF: SQLite Runtime Status @@ -6313,12 +6529,12 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** the current value is always zero.)^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
SQLITE_DBSTATUS_CACHE_USED
-**
This parameter returns the approximate number of of bytes of heap +**
This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
SQLITE_DBSTATUS_SCHEMA_USED
-**
This parameter returns the approximate number of of bytes of heap +**
This parameter returns the approximate number of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ ** ^The full amount of memory used by the schemas is reported, even if the @@ -6327,11 +6543,40 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
SQLITE_DBSTATUS_STMT_USED
-**
This parameter returns the approximate number of of bytes of heap +**
This parameter returns the approximate number of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. **
+** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
SQLITE_DBSTATUS_CACHE_HIT
+**
This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
SQLITE_DBSTATUS_CACHE_MISS
+**
This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
SQLITE_DBSTATUS_CACHE_WRITE
+**
This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
+** +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
SQLITE_DBSTATUS_DEFERRED_FKS
+**
This parameter returns zero for the current value if and only if +** all foreign key constraints (deferred or immediate) have been +** resolved.)^ ^The highwater mark is always 0. +**
** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -6341,7 +6586,11 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 -#define SQLITE_DBSTATUS_MAX 6 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_DEFERRED_FKS 10 +#define SQLITE_DBSTATUS_MAX 10 /* Largest defined DBSTATUS */ /* @@ -6396,11 +6645,20 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** improvement performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run. ** +** [[SQLITE_STMTSTATUS_VM_STEP]]
SQLITE_STMTSTATUS_VM_STEP
+**
^This is the number of virtual machine operations executed +** by the prepared statement if that number is less than or equal +** to 2147483647. The number of virtual machine operations can be +** used as a proxy for the total work done by the prepared statement. +** If the number of virtual machine operations exceeds 2147483647 +** then the value returned by this statement status code is undefined. +**
** */ #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 #define SQLITE_STMTSTATUS_SORT 2 #define SQLITE_STMTSTATUS_AUTOINDEX 3 +#define SQLITE_STMTSTATUS_VM_STEP 4 /* ** CAPI3REF: Custom Page Cache Object @@ -6411,17 +6669,33 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** sqlite3_pcache object except by holding and passing pointers ** to the object. ** -** See [sqlite3_pcache_methods] for additional information. +** See [sqlite3_pcache_methods2] for additional information. */ typedef struct sqlite3_pcache sqlite3_pcache; +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + /* ** CAPI3REF: Application Defined Page Cache. ** KEYWORDS: {page cache} ** -** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ +** instance of the sqlite3_pcache_methods2 structure.)^ ** In many applications, most of the heap memory allocated by ** SQLite is used for the page cache. ** By implementing a @@ -6435,7 +6709,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** extreme measure that is only needed by the most demanding applications. ** The built-in page cache is recommended for most uses. ** -** ^(The contents of the sqlite3_pcache_methods structure are copied to an +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ @@ -6444,7 +6718,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() -** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ ** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. ** ^(If the xInit() method is NULL, then the @@ -6471,17 +6745,15 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. ^szPage will not be a power of two. ^szPage -** will the page size of the database file that is to be cached plus an -** increment (here called "R") of less than 250. SQLite will use the -** extra R bytes on each page to store metadata about the underlying -** database page on disk. The value of R depends +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends ** on the SQLite version, the target platform, and how SQLite was compiled. -** ^(R is constant for a particular build of SQLite. Except, there are two -** distinct values of R when SQLite is compiled with the proprietary -** ZIPVFS extension.)^ ^The second argument to -** xCreate(), bPurgeable, is true if the cache being created will -** be used to cache database pages of a file stored on disk, or +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will @@ -6505,11 +6777,16 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ** [[the xFetch() page cache methods]] ** The xFetch() method locates a page in the cache and returns a pointer to -** the page, or a NULL pointer. -** A "page", in this context, means a buffer of szPage bytes aligned at an -** 8-byte boundary. The page to be fetched is determined by the key. ^The -** minimum key value is 1. After it has been retrieved using xFetch, the page -** is considered to be "pinned". +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". ** ** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content @@ -6518,7 +6795,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** parameter to help it determined what action to take: ** ** -**
createFlag Behaviour when page is not already in cache +**
createFlag Behavior when page is not already in cache **
0 Do not allocate a new page. Return NULL. **
1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. @@ -6562,8 +6839,37 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] -** handle invalid, and will not use it with any other sqlite3_pcache_methods +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 ** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. */ typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; struct sqlite3_pcache_methods { @@ -6580,6 +6886,7 @@ struct sqlite3_pcache_methods { void (*xDestroy)(sqlite3_pcache*); }; + /* ** CAPI3REF: Online Backup Object ** @@ -6909,17 +7216,33 @@ SQLITE_API int sqlite3_unlock_notify( /* ** CAPI3REF: String Comparison ** -** ^The [sqlite3_strnicmp()] API allows applications and extensions to -** compare the contents of two buffers containing UTF-8 strings in a -** case-independent fashion, using the same definition of case independence -** that SQLite uses internally when comparing identifiers. +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ +SQLITE_API int sqlite3_stricmp(const char *, const char *); SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + /* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. @@ -6999,6 +7322,9 @@ SQLITE_API void *sqlite3_wal_hook( ** ^The [wal_autocheckpoint pragma] can be used to invoke this interface ** from SQL. ** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] ** pages. The use of this interface @@ -7015,6 +7341,10 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** empty string, then a checkpoint is run on all databases of ** connection D. ^If the database connection D is not in ** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a +** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. +** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL +** or RESET checkpoint. ** ** ^The [wal_checkpoint pragma] can be used to invoke this interface ** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the @@ -7037,10 +7367,12 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** Checkpoint as many frames as possible without waiting for any database ** readers or writers to finish. Sync the db file if all frames in the log ** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] +** is never invoked. ** **
SQLITE_CHECKPOINT_FULL
-** This mode blocks (calls the busy-handler callback) until there is no +** This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database ** snapshot. It then checkpoints all frames in the log file and syncs the ** database file. This call blocks database writers while it is running, @@ -7048,7 +7380,8 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** **
SQLITE_CHECKPOINT_RESTART
** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the busy-handler callback) +** checkpointing the log file it blocks (calls the +** [sqlite3_busy_handler|busy-handler callback]) ** until all readers are reading from the database file only. This ensures ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, @@ -7186,6 +7519,7 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); /* ** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to ** inform a [virtual table] implementation what the [ON CONFLICT] mode @@ -7214,7 +7548,7 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); #if 0 } /* End of the 'extern "C"' block */ #endif -#endif +#endif /* _SQLITE3_H_ */ /* ** 2010 August 30 @@ -7238,6 +7572,16 @@ extern "C" { #endif typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif /* ** Register a geometry callback named zGeom that can be used as part of an @@ -7248,7 +7592,7 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; SQLITE_API int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), void *pContext ); @@ -7260,11 +7604,60 @@ SQLITE_API int sqlite3_rtree_geometry_callback( struct sqlite3_rtree_geometry { void *pContext; /* Copy of pContext passed to s_r_g_c() */ int nParam; /* Size of array aParam[] */ - double *aParam; /* Parameters passed to SQL geom function */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ void *pUser; /* Callback implementation user data */ void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ }; +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + #if 0 } /* end of the 'extern "C"' block */ @@ -7275,6 +7668,492 @@ struct sqlite3_rtree_geometry { /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ + +/* +** Include the configuration header output by 'configure' if we're using the +** autoconf-based build +*/ +#ifdef _HAVE_SQLITE_CONFIG_H +#include "config.h" +#endif + +/************** Include sqliteLimit.h in the middle of sqliteInt.h ***********/ +/************** Begin file sqliteLimit.h *************************************/ +/* +** 2007 May 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file defines various limits of what SQLite can process. +*/ + +/* +** The maximum length of a TEXT or BLOB in bytes. This also +** limits the size of a row in a table or index. +** +** The hard limit is the ability of a 32-bit signed integer +** to count the size: 2^31-1 or 2147483647. +*/ +#ifndef SQLITE_MAX_LENGTH +# define SQLITE_MAX_LENGTH 1000000000 +#endif + +/* +** This is the maximum number of +** +** * Columns in a table +** * Columns in an index +** * Columns in a view +** * Terms in the SET clause of an UPDATE statement +** * Terms in the result set of a SELECT statement +** * Terms in the GROUP BY or ORDER BY clauses of a SELECT statement. +** * Terms in the VALUES clause of an INSERT statement +** +** The hard upper limit here is 32676. Most database people will +** tell you that in a well-normalized database, you usually should +** not have more than a dozen or so columns in any table. And if +** that is the case, there is no point in having more than a few +** dozen values in any of the other situations described above. +*/ +#ifndef SQLITE_MAX_COLUMN +# define SQLITE_MAX_COLUMN 2000 +#endif + +/* +** The maximum length of a single SQL statement in bytes. +** +** It used to be the case that setting this value to zero would +** turn the limit off. That is no longer true. It is not possible +** to turn this limit off. +*/ +#ifndef SQLITE_MAX_SQL_LENGTH +# define SQLITE_MAX_SQL_LENGTH 1000000000 +#endif + +/* +** The maximum depth of an expression tree. This is limited to +** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might +** want to place more severe limits on the complexity of an +** expression. +** +** A value of 0 used to mean that the limit was not enforced. +** But that is no longer true. The limit is now strictly enforced +** at all times. +*/ +#ifndef SQLITE_MAX_EXPR_DEPTH +# define SQLITE_MAX_EXPR_DEPTH 1000 +#endif + +/* +** The maximum number of terms in a compound SELECT statement. +** The code generator for compound SELECT statements does one +** level of recursion for each term. A stack overflow can result +** if the number of terms is too large. In practice, most SQL +** never has more than 3 or 4 terms. Use a value of 0 to disable +** any limit on the number of terms in a compount SELECT. +*/ +#ifndef SQLITE_MAX_COMPOUND_SELECT +# define SQLITE_MAX_COMPOUND_SELECT 500 +#endif + +/* +** The maximum number of opcodes in a VDBE program. +** Not currently enforced. +*/ +#ifndef SQLITE_MAX_VDBE_OP +# define SQLITE_MAX_VDBE_OP 25000 +#endif + +/* +** The maximum number of arguments to an SQL function. +*/ +#ifndef SQLITE_MAX_FUNCTION_ARG +# define SQLITE_MAX_FUNCTION_ARG 127 +#endif + +/* +** The maximum number of in-memory pages to use for the main database +** table and for temporary tables. The SQLITE_DEFAULT_CACHE_SIZE +*/ +#ifndef SQLITE_DEFAULT_CACHE_SIZE +# define SQLITE_DEFAULT_CACHE_SIZE 2000 +#endif +#ifndef SQLITE_DEFAULT_TEMP_CACHE_SIZE +# define SQLITE_DEFAULT_TEMP_CACHE_SIZE 500 +#endif + +/* +** The default number of frames to accumulate in the log file before +** checkpointing the database in WAL mode. +*/ +#ifndef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT +# define SQLITE_DEFAULT_WAL_AUTOCHECKPOINT 1000 +#endif + +/* +** The maximum number of attached databases. This must be between 0 +** and 62. The upper bound on 62 is because a 64-bit integer bitmap +** is used internally to track attached databases. +*/ +#ifndef SQLITE_MAX_ATTACHED +# define SQLITE_MAX_ATTACHED 10 +#endif + + +/* +** The maximum value of a ?nnn wildcard that the parser will accept. +*/ +#ifndef SQLITE_MAX_VARIABLE_NUMBER +# define SQLITE_MAX_VARIABLE_NUMBER 999 +#endif + +/* Maximum page size. The upper bound on this value is 65536. This a limit +** imposed by the use of 16-bit offsets within each page. +** +** Earlier versions of SQLite allowed the user to change this value at +** compile time. This is no longer permitted, on the grounds that it creates +** a library that is technically incompatible with an SQLite library +** compiled with a different limit. If a process operating on a database +** with a page-size of 65536 bytes crashes, then an instance of SQLite +** compiled with the default page-size limit will not be able to rollback +** the aborted transaction. This could lead to database corruption. +*/ +#ifdef SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_PAGE_SIZE +#endif +#define SQLITE_MAX_PAGE_SIZE 65536 + + +/* +** The default size of a database page. +*/ +#ifndef SQLITE_DEFAULT_PAGE_SIZE +# define SQLITE_DEFAULT_PAGE_SIZE 1024 +#endif +#if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE +# undef SQLITE_DEFAULT_PAGE_SIZE +# define SQLITE_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE +#endif + +/* +** Ordinarily, if no value is explicitly provided, SQLite creates databases +** with page size SQLITE_DEFAULT_PAGE_SIZE. However, based on certain +** device characteristics (sector-size and atomic write() support), +** SQLite may choose a larger value. This constant is the maximum value +** SQLite will choose on its own. +*/ +#ifndef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE 8192 +#endif +#if SQLITE_MAX_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE +#endif + + +/* +** Maximum number of pages in one database file. +** +** This is really just the default value for the max_page_count pragma. +** This value can be lowered (or raised) at run-time using that the +** max_page_count macro. +*/ +#ifndef SQLITE_MAX_PAGE_COUNT +# define SQLITE_MAX_PAGE_COUNT 1073741823 +#endif + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Maximum depth of recursion for triggers. +** +** A value of 1 means that a trigger program will not be able to itself +** fire any triggers. A value of 0 means that no trigger programs at all +** may be executed. +*/ +#ifndef SQLITE_MAX_TRIGGER_DEPTH +# define SQLITE_MAX_TRIGGER_DEPTH 1000 +#endif + +/************** End of sqliteLimit.h *****************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* Disable nuisance warnings on Borland compilers */ +#if defined(__BORLANDC__) +#pragma warn -rch /* unreachable code */ +#pragma warn -ccc /* Condition is always true or false */ +#pragma warn -aus /* Assigned value is never used */ +#pragma warn -csu /* Comparing signed and unsigned */ +#pragma warn -spa /* Suspicious pointer arithmetic */ +#endif + +/* +** Include standard header files as necessary +*/ +#ifdef HAVE_STDINT_H +#include +#endif +#ifdef HAVE_INTTYPES_H +#include +#endif + +/* +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. +** +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that vary from one machine to the next. +** +** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on +** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)). +** So we have to define the macros in different ways depending on the +** compiler. +*/ +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) +#else /* Generates a warning - but it always works */ +# define SQLITE_INT_TO_PTR(X) ((void*)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(X)) +#endif + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define SQLITE_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define SQLITE_NOINLINE __declspec(noinline) +#else +# define SQLITE_NOINLINE +#endif + +/* +** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. +** 0 means mutexes are permanently disable and the library is never +** threadsafe. 1 means the library is serialized which is the highest +** level of threadsafety. 2 means the library is multithreaded - multiple +** threads can use SQLite as long as no two threads try to use the same +** database connection at the same time. +** +** Older versions of SQLite used an optional THREADSAFE macro. +** We support that for legacy. +*/ +#if !defined(SQLITE_THREADSAFE) +# if defined(THREADSAFE) +# define SQLITE_THREADSAFE THREADSAFE +# else +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ +# endif +#endif + +/* +** Powersafe overwrite is on by default. But can be turned off using +** the -DSQLITE_POWERSAFE_OVERWRITE=0 command-line option. +*/ +#ifndef SQLITE_POWERSAFE_OVERWRITE +# define SQLITE_POWERSAFE_OVERWRITE 1 +#endif + +/* +** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. +** It determines whether or not the features related to +** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can +** be overridden at runtime using the sqlite3_config() API. +*/ +#if !defined(SQLITE_DEFAULT_MEMSTATUS) +# define SQLITE_DEFAULT_MEMSTATUS 1 +#endif + +/* +** Exactly one of the following macros must be defined in order to +** specify which memory allocation subsystem to use. +** +** SQLITE_SYSTEM_MALLOC // Use normal system malloc() +** SQLITE_WIN32_MALLOC // Use Win32 native heap API +** SQLITE_ZERO_MALLOC // Use a stub allocator that always fails +** SQLITE_MEMDEBUG // Debugging version of system malloc() +** +** On Windows, if the SQLITE_WIN32_MALLOC_VALIDATE macro is defined and the +** assert() macro is enabled, each call into the Win32 native heap subsystem +** will cause HeapValidate to be called. If heap validation should fail, an +** assertion will be triggered. +** +** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as +** the default. +*/ +#if defined(SQLITE_SYSTEM_MALLOC) \ + + defined(SQLITE_WIN32_MALLOC) \ + + defined(SQLITE_ZERO_MALLOC) \ + + defined(SQLITE_MEMDEBUG)>1 +# error "Two or more of the following compile-time configuration options\ + are defined but at most one is allowed:\ + SQLITE_SYSTEM_MALLOC, SQLITE_WIN32_MALLOC, SQLITE_MEMDEBUG,\ + SQLITE_ZERO_MALLOC" +#endif +#if defined(SQLITE_SYSTEM_MALLOC) \ + + defined(SQLITE_WIN32_MALLOC) \ + + defined(SQLITE_ZERO_MALLOC) \ + + defined(SQLITE_MEMDEBUG)==0 +# define SQLITE_SYSTEM_MALLOC 1 +#endif + +/* +** If SQLITE_MALLOC_SOFT_LIMIT is not zero, then try to keep the +** sizes of memory allocations below this value where possible. +*/ +#if !defined(SQLITE_MALLOC_SOFT_LIMIT) +# define SQLITE_MALLOC_SOFT_LIMIT 1024 +#endif + +/* +** We need to define _XOPEN_SOURCE as follows in order to enable +** recursive mutexes on most Unix systems and fchmod() on OpenBSD. +** But _XOPEN_SOURCE define causes problems for Mac OS X, so omit +** it. +*/ +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) +# define _XOPEN_SOURCE 600 +#endif + +/* +** NDEBUG and SQLITE_DEBUG are opposites. It should always be true that +** defined(NDEBUG)==!defined(SQLITE_DEBUG). If this is not currently true, +** make it true by defining or undefining NDEBUG. +** +** Setting NDEBUG makes the code smaller and faster by disabling the +** assert() statements in the code. So we want the default action +** to be for NDEBUG to be set and NDEBUG to be undefined only if SQLITE_DEBUG +** is set. Thus NDEBUG becomes an opt-in rather than an opt-out +** feature. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +/* +** Enable SQLITE_ENABLE_EXPLAIN_COMMENTS if SQLITE_DEBUG is turned on. +*/ +#if !defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) && defined(SQLITE_DEBUG) +# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 +#endif + +/* +** The testcase() macro is used to aid in coverage testing. When +** doing coverage testing, the condition inside the argument to +** testcase() must be evaluated both true and false in order to +** get full branch coverage. The testcase() macro is inserted +** to help ensure adequate test coverage in places where simple +** condition/decision coverage is inadequate. For example, testcase() +** can be used to make sure boundary values are tested. For +** bitmask tests, testcase() can be used to make sure each bit +** is significant and used at least once. On switch statements +** where multiple cases go to the same block of code, testcase() +** can insure that all cases are evaluated. +** +*/ +#ifdef SQLITE_COVERAGE_TEST +SQLITE_PRIVATE void sqlite3Coverage(int); +# define testcase(X) if( X ){ sqlite3Coverage(__LINE__); } +#else +# define testcase(X) +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +/* +** Sometimes we need a small amount of code such as a variable initialization +** to setup for a later assert() statement. We do not want this code to +** appear when assert() is disabled. The following macro is therefore +** used to contain that setup code. The "VVA" acronym stands for +** "Verification, Validation, and Accreditation". In other words, the +** code within VVA_ONLY() will only run during verification processes. +*/ +#ifndef NDEBUG +# define VVA_ONLY(X) X +#else +# define VVA_ONLY(X) +#endif + +/* +** The ALWAYS and NEVER macros surround boolean expressions which +** are intended to always be true or false, respectively. Such +** expressions could be omitted from the code completely. But they +** are included in a few cases in order to enhance the resilience +** of SQLite to unexpected behavior - to make the code "self-healing" +** or "ductile" rather than being "brittle" and crashing at the first +** hint of unplanned behavior. +** +** In other words, ALWAYS and NEVER are added for defensive code. +** +** When doing coverage testing ALWAYS and NEVER are hard-coded to +** be true and false so that the unreachable code they specify will +** not be counted as untested code. +*/ +#if defined(SQLITE_COVERAGE_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +/* +** Return true (non-zero) if the input is an integer that is too large +** to fit in 32-bits. This macro is used inside of various testcase() +** macros to verify that we have tested SQLite for large-file support. +*/ +#define IS_BIG_INT(X) (((X)&~(i64)0xffffffff)!=0) + +/* +** The macro unlikely() is a hint that surrounds a boolean +** expression that is usually false. Macro likely() surrounds +** a boolean expression that is usually true. These hints could, +** in theory, be used by the compiler to generate better code, but +** currently they are just comments for human readers. +*/ +#define likely(X) (X) +#define unlikely(X) (X) + /************** Include hash.h in the middle of sqliteInt.h ******************/ /************** Begin file hash.h ********************************************/ /* @@ -7288,7 +8167,7 @@ struct sqlite3_rtree_geometry { ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. */ #ifndef _SQLITE_HASH_H_ @@ -7338,15 +8217,15 @@ struct Hash { struct HashElem { HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ - const char *pKey; int nKey; /* Key associated with this element */ + const char *pKey; /* Key associated with this element */ }; /* ** Access routines. To delete, insert a NULL pointer. */ SQLITE_PRIVATE void sqlite3HashInit(Hash*); -SQLITE_PRIVATE void *sqlite3HashInsert(Hash*, const char *pKey, int nKey, void *pData); -SQLITE_PRIVATE void *sqlite3HashFind(const Hash*, const char *pKey, int nKey); +SQLITE_PRIVATE void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); +SQLITE_PRIVATE void *sqlite3HashFind(const Hash*, const char *pKey); SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* @@ -7378,163 +8257,165 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /************** Continuing where we left off in sqliteInt.h ******************/ /************** Include parse.h in the middle of sqliteInt.h *****************/ /************** Begin file parse.h *******************************************/ -#define TK_SEMI 1 -#define TK_EXPLAIN 2 -#define TK_QUERY 3 -#define TK_PLAN 4 -#define TK_BEGIN 5 -#define TK_TRANSACTION 6 -#define TK_DEFERRED 7 -#define TK_IMMEDIATE 8 -#define TK_EXCLUSIVE 9 -#define TK_COMMIT 10 -#define TK_END 11 -#define TK_ROLLBACK 12 -#define TK_SAVEPOINT 13 -#define TK_RELEASE 14 -#define TK_TO 15 -#define TK_TABLE 16 -#define TK_CREATE 17 -#define TK_IF 18 -#define TK_NOT 19 -#define TK_EXISTS 20 -#define TK_TEMP 21 -#define TK_LP 22 -#define TK_RP 23 -#define TK_AS 24 -#define TK_COMMA 25 -#define TK_ID 26 -#define TK_INDEXED 27 -#define TK_ABORT 28 -#define TK_ACTION 29 -#define TK_AFTER 30 -#define TK_ANALYZE 31 -#define TK_ASC 32 -#define TK_ATTACH 33 -#define TK_BEFORE 34 -#define TK_BY 35 -#define TK_CASCADE 36 -#define TK_CAST 37 -#define TK_COLUMNKW 38 -#define TK_CONFLICT 39 -#define TK_DATABASE 40 -#define TK_DESC 41 -#define TK_DETACH 42 -#define TK_EACH 43 -#define TK_FAIL 44 -#define TK_FOR 45 -#define TK_IGNORE 46 -#define TK_INITIALLY 47 -#define TK_INSTEAD 48 -#define TK_LIKE_KW 49 -#define TK_MATCH 50 -#define TK_NO 51 -#define TK_KEY 52 -#define TK_OF 53 -#define TK_OFFSET 54 -#define TK_PRAGMA 55 -#define TK_RAISE 56 -#define TK_REPLACE 57 -#define TK_RESTRICT 58 -#define TK_ROW 59 -#define TK_TRIGGER 60 -#define TK_VACUUM 61 -#define TK_VIEW 62 -#define TK_VIRTUAL 63 -#define TK_REINDEX 64 -#define TK_RENAME 65 -#define TK_CTIME_KW 66 -#define TK_ANY 67 -#define TK_OR 68 -#define TK_AND 69 -#define TK_IS 70 -#define TK_BETWEEN 71 -#define TK_IN 72 -#define TK_ISNULL 73 -#define TK_NOTNULL 74 -#define TK_NE 75 -#define TK_EQ 76 -#define TK_GT 77 -#define TK_LE 78 -#define TK_LT 79 -#define TK_GE 80 -#define TK_ESCAPE 81 -#define TK_BITAND 82 -#define TK_BITOR 83 -#define TK_LSHIFT 84 -#define TK_RSHIFT 85 -#define TK_PLUS 86 -#define TK_MINUS 87 -#define TK_STAR 88 -#define TK_SLASH 89 -#define TK_REM 90 -#define TK_CONCAT 91 -#define TK_COLLATE 92 -#define TK_BITNOT 93 -#define TK_STRING 94 -#define TK_JOIN_KW 95 -#define TK_CONSTRAINT 96 -#define TK_DEFAULT 97 -#define TK_NULL 98 -#define TK_PRIMARY 99 -#define TK_UNIQUE 100 -#define TK_CHECK 101 -#define TK_REFERENCES 102 -#define TK_AUTOINCR 103 -#define TK_ON 104 -#define TK_INSERT 105 -#define TK_DELETE 106 -#define TK_UPDATE 107 -#define TK_SET 108 -#define TK_DEFERRABLE 109 -#define TK_FOREIGN 110 -#define TK_DROP 111 -#define TK_UNION 112 -#define TK_ALL 113 -#define TK_EXCEPT 114 -#define TK_INTERSECT 115 -#define TK_SELECT 116 -#define TK_DISTINCT 117 -#define TK_DOT 118 -#define TK_FROM 119 -#define TK_JOIN 120 -#define TK_USING 121 -#define TK_ORDER 122 -#define TK_GROUP 123 -#define TK_HAVING 124 -#define TK_LIMIT 125 -#define TK_WHERE 126 -#define TK_INTO 127 -#define TK_VALUES 128 -#define TK_INTEGER 129 -#define TK_FLOAT 130 -#define TK_BLOB 131 -#define TK_REGISTER 132 -#define TK_VARIABLE 133 -#define TK_CASE 134 -#define TK_WHEN 135 -#define TK_THEN 136 -#define TK_ELSE 137 -#define TK_INDEX 138 -#define TK_ALTER 139 -#define TK_ADD 140 -#define TK_TO_TEXT 141 -#define TK_TO_BLOB 142 -#define TK_TO_NUMERIC 143 -#define TK_TO_INT 144 -#define TK_TO_REAL 145 -#define TK_ISNOT 146 -#define TK_END_OF_FILE 147 -#define TK_ILLEGAL 148 -#define TK_SPACE 149 -#define TK_UNCLOSED_STRING 150 -#define TK_FUNCTION 151 -#define TK_COLUMN 152 -#define TK_AGG_FUNCTION 153 -#define TK_AGG_COLUMN 154 -#define TK_CONST_FUNC 155 -#define TK_UMINUS 156 -#define TK_UPLUS 157 +#define TK_SEMI 1 +#define TK_EXPLAIN 2 +#define TK_QUERY 3 +#define TK_PLAN 4 +#define TK_BEGIN 5 +#define TK_TRANSACTION 6 +#define TK_DEFERRED 7 +#define TK_IMMEDIATE 8 +#define TK_EXCLUSIVE 9 +#define TK_COMMIT 10 +#define TK_END 11 +#define TK_ROLLBACK 12 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_TABLE 16 +#define TK_CREATE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_WITHOUT 25 +#define TK_COMMA 26 +#define TK_ID 27 +#define TK_INDEXED 28 +#define TK_ABORT 29 +#define TK_ACTION 30 +#define TK_AFTER 31 +#define TK_ANALYZE 32 +#define TK_ASC 33 +#define TK_ATTACH 34 +#define TK_BEFORE 35 +#define TK_BY 36 +#define TK_CASCADE 37 +#define TK_CAST 38 +#define TK_COLUMNKW 39 +#define TK_CONFLICT 40 +#define TK_DATABASE 41 +#define TK_DESC 42 +#define TK_DETACH 43 +#define TK_EACH 44 +#define TK_FAIL 45 +#define TK_FOR 46 +#define TK_IGNORE 47 +#define TK_INITIALLY 48 +#define TK_INSTEAD 49 +#define TK_LIKE_KW 50 +#define TK_MATCH 51 +#define TK_NO 52 +#define TK_KEY 53 +#define TK_OF 54 +#define TK_OFFSET 55 +#define TK_PRAGMA 56 +#define TK_RAISE 57 +#define TK_RECURSIVE 58 +#define TK_REPLACE 59 +#define TK_RESTRICT 60 +#define TK_ROW 61 +#define TK_TRIGGER 62 +#define TK_VACUUM 63 +#define TK_VIEW 64 +#define TK_VIRTUAL 65 +#define TK_WITH 66 +#define TK_REINDEX 67 +#define TK_RENAME 68 +#define TK_CTIME_KW 69 +#define TK_ANY 70 +#define TK_OR 71 +#define TK_AND 72 +#define TK_IS 73 +#define TK_BETWEEN 74 +#define TK_IN 75 +#define TK_ISNULL 76 +#define TK_NOTNULL 77 +#define TK_NE 78 +#define TK_EQ 79 +#define TK_GT 80 +#define TK_LE 81 +#define TK_LT 82 +#define TK_GE 83 +#define TK_ESCAPE 84 +#define TK_BITAND 85 +#define TK_BITOR 86 +#define TK_LSHIFT 87 +#define TK_RSHIFT 88 +#define TK_PLUS 89 +#define TK_MINUS 90 +#define TK_STAR 91 +#define TK_SLASH 92 +#define TK_REM 93 +#define TK_CONCAT 94 +#define TK_COLLATE 95 +#define TK_BITNOT 96 +#define TK_STRING 97 +#define TK_JOIN_KW 98 +#define TK_CONSTRAINT 99 +#define TK_DEFAULT 100 +#define TK_NULL 101 +#define TK_PRIMARY 102 +#define TK_UNIQUE 103 +#define TK_CHECK 104 +#define TK_REFERENCES 105 +#define TK_AUTOINCR 106 +#define TK_ON 107 +#define TK_INSERT 108 +#define TK_DELETE 109 +#define TK_UPDATE 110 +#define TK_SET 111 +#define TK_DEFERRABLE 112 +#define TK_FOREIGN 113 +#define TK_DROP 114 +#define TK_UNION 115 +#define TK_ALL 116 +#define TK_EXCEPT 117 +#define TK_INTERSECT 118 +#define TK_SELECT 119 +#define TK_VALUES 120 +#define TK_DISTINCT 121 +#define TK_DOT 122 +#define TK_FROM 123 +#define TK_JOIN 124 +#define TK_USING 125 +#define TK_ORDER 126 +#define TK_GROUP 127 +#define TK_HAVING 128 +#define TK_LIMIT 129 +#define TK_WHERE 130 +#define TK_INTO 131 +#define TK_INTEGER 132 +#define TK_FLOAT 133 +#define TK_BLOB 134 +#define TK_VARIABLE 135 +#define TK_CASE 136 +#define TK_WHEN 137 +#define TK_THEN 138 +#define TK_ELSE 139 +#define TK_INDEX 140 +#define TK_ALTER 141 +#define TK_ADD 142 +#define TK_TO_TEXT 143 +#define TK_TO_BLOB 144 +#define TK_TO_NUMERIC 145 +#define TK_TO_INT 146 +#define TK_TO_REAL 147 +#define TK_ISNOT 148 +#define TK_END_OF_FILE 149 +#define TK_ILLEGAL 150 +#define TK_SPACE 151 +#define TK_UNCLOSED_STRING 152 +#define TK_FUNCTION 153 +#define TK_COLUMN 154 +#define TK_AGG_FUNCTION 155 +#define TK_AGG_COLUMN 156 +#define TK_UMINUS 157 +#define TK_UPLUS 158 +#define TK_REGISTER 159 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -7583,7 +8464,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); */ #define SQLITE_MAX_FILE_FORMAT 4 #ifndef SQLITE_DEFAULT_FILE_FORMAT -# define SQLITE_DEFAULT_FILE_FORMAT 1 +# define SQLITE_DEFAULT_FILE_FORMAT 4 #endif /* @@ -7600,8 +8481,30 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); */ #ifndef SQLITE_TEMP_STORE # define SQLITE_TEMP_STORE 1 +# define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif +/* +** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** to zero. +*/ +#if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 +#endif +#ifndef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 8 +#endif +#ifndef SQLITE_DEFAULT_WORKER_THREADS +# define SQLITE_DEFAULT_WORKER_THREADS 0 +#endif +#if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS +#endif + + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -7610,6 +8513,17 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) #endif +/* +** Macros to compute minimum and maximum of two numbers. +*/ +#define MIN(A,B) ((A)<(B)?(A):(B)) +#define MAX(A,B) ((A)>(B)?(A):(B)) + +/* +** Swap two objects of type TYPE. +*/ +#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} + /* ** Check to see if this machine uses EBCDIC. (Yes, believe it or ** not, there are still machines out there that use EBCDIC.) @@ -7681,24 +8595,78 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ */ #define SQLITE_MAX_U32 ((((u64)1)<<32)-1) +/* +** The datatype used to store estimates of the number of rows in a +** table or index. This is an unsigned integer type. For 99.9% of +** the world, a 32-bit integer is sufficient. But a 64-bit integer +** can be used at compile-time if desired. +*/ +#ifdef SQLITE_64BIT_STATS + typedef u64 tRowcnt; /* 64-bit only if requested at compile-time */ +#else + typedef u32 tRowcnt; /* 32-bit is the default */ +#endif + +/* +** Estimated quantities used for query planning are stored as 16-bit +** logarithms. For quantity X, the value stored is 10*log2(X). This +** gives a possible range of values of approximately 1.0e986 to 1e-986. +** But the allowed values are "grainy". Not every value is representable. +** For example, quantities 16 and 17 are both represented by a LogEst +** of 40. However, since LogEst quantaties are suppose to be estimates, +** not exact values, this imprecision is not a problem. +** +** "LogEst" is short for "Logarithmic Estimate". +** +** Examples: +** 1 -> 0 20 -> 43 10000 -> 132 +** 2 -> 10 25 -> 46 25000 -> 146 +** 3 -> 16 100 -> 66 1000000 -> 199 +** 4 -> 20 1000 -> 99 1048576 -> 200 +** 10 -> 33 1024 -> 100 4294967296 -> 320 +** +** The LogEst can be negative to indicate fractional values. +** Examples: +** +** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 +*/ +typedef INT16_TYPE LogEst; + /* ** Macros to determine whether the machine is big or little endian, -** evaluated at runtime. +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. */ #ifdef SQLITE_AMALGAMATION SQLITE_PRIVATE const int sqlite3one = 1; #else SQLITE_PRIVATE const int sqlite3one; #endif -#if defined(i386) || defined(__i386__) || defined(_M_IX86)\ - || defined(__x86_64) || defined(__x86_64__) +#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 1234 # define SQLITE_BIGENDIAN 0 # define SQLITE_LITTLEENDIAN 1 # define SQLITE_UTF16NATIVE SQLITE_UTF16LE -#else +#endif +#if (defined(sparc) || defined(__ppc__)) \ + && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 4321 +# define SQLITE_BIGENDIAN 1 +# define SQLITE_LITTLEENDIAN 0 +# define SQLITE_UTF16NATIVE SQLITE_UTF16BE +#endif +#if !defined(SQLITE_BYTEORDER) +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) -# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) +# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) #endif /* @@ -7726,7 +8694,7 @@ SQLITE_PRIVATE const int sqlite3one; ** all alignment restrictions correct. ** ** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the -** underlying malloc() implemention might return us 4-byte aligned +** underlying malloc() implementation might return us 4-byte aligned ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC @@ -7735,6 +8703,73 @@ SQLITE_PRIVATE const int sqlite3one; # define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) #endif +/* +** Disable MMAP on platforms where it is known to not work +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +#endif + +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +# define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +# define SQLITE_DEFAULT_MMAP_SIZE_xc 1 /* Exclude from ctime.c */ +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif + +/* +** Only one of SQLITE_ENABLE_STAT3 or SQLITE_ENABLE_STAT4 can be defined. +** Priority is given to SQLITE_ENABLE_STAT4. If either are defined, also +** define SQLITE_ENABLE_STAT3_OR_STAT4 +*/ +#ifdef SQLITE_ENABLE_STAT4 +# undef SQLITE_ENABLE_STAT3 +# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +#elif SQLITE_ENABLE_STAT3 +# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +#elif SQLITE_ENABLE_STAT3_OR_STAT4 +# undef SQLITE_ENABLE_STAT3_OR_STAT4 +#endif + +/* +** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not +** the Select query generator tracing logic is turned on. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_SELECTTRACE) +# define SELECTTRACE_ENABLED 1 +#else +# define SELECTTRACE_ENABLED 0 +#endif /* ** An instance of the following structure is used to store the busy-handler @@ -7777,10 +8812,19 @@ struct BusyHandler { #define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) /* -** The following value as a destructor means to use sqlite3DbFree(). -** This is an internal extension to SQLITE_STATIC and SQLITE_TRANSIENT. +** Determine if the argument is a power of two */ -#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3DbFree) +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + +/* +** The following value as a destructor means to use sqlite3DbFree(). +** The sqlite3DbFree() routine requires two parameters instead of the +** one parameter that destructors normally want. So we have to introduce +** this magic value that the code knows to handle differently. Any +** pointer will work here as long as it is distinct from SQLITE_STATIC +** and SQLITE_TRANSIENT. +*/ +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3MallocSize) /* ** When SQLITE_OMIT_WSD is defined, it means that the target platform does @@ -7855,14 +8899,18 @@ typedef struct LookasideSlot LookasideSlot; typedef struct Module Module; typedef struct NameContext NameContext; typedef struct Parse Parse; +typedef struct PrintfArguments PrintfArguments; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SQLiteThread SQLiteThread; +typedef struct SelectDest SelectDest; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; typedef struct TableLock TableLock; typedef struct Token Token; +typedef struct TreeView TreeView; typedef struct Trigger Trigger; typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; @@ -7870,9 +8918,8 @@ typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; -typedef struct WherePlan WherePlan; typedef struct WhereInfo WhereInfo; -typedef struct WhereLevel WhereLevel; +typedef struct With With; /* ** Defer sourcing vdbe.h and btree.h until after the "u8" and @@ -7940,14 +8987,16 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( ** pager.h. */ #define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ -#define BTREE_NO_READLOCK 2 /* Omit readlocks on readonly files */ -#define BTREE_MEMORY 4 /* This is an in-memory DB */ -#define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ -#define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ +#define BTREE_MEMORY 2 /* This is an in-memory DB */ +#define BTREE_SINGLE 4 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 8 /* Use of a hash implementation is OK */ SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); +#if SQLITE_MAX_MMAP_SIZE>0 +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif +SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); @@ -7955,13 +9004,16 @@ SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree*); +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); +#endif SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); -SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*); +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*,int,int); SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree*,int); SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree*, int*, int flags); SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree*); @@ -7993,11 +9045,14 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); -SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree*, int); +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree*, int, int); SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); +SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); + /* ** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta ** should be one of the following values. The integer values are assigned @@ -8018,6 +9073,13 @@ SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); #define BTREE_TEXT_ENCODING 5 #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 + +/* +** Values that may be OR'd together to form the second argument of an +** sqlite3BtreeCursorHints() call. +*/ +#define BTREE_BULKLOAD 0x00000001 SQLITE_PRIVATE int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ @@ -8037,7 +9099,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( int bias, int *pRes ); -SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*, int*); +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, @@ -8049,21 +9112,20 @@ SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor*, int *pAmt); -SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor*, int *pAmt); +SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); SQLITE_PRIVATE int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE void sqlite3BtreeSetCachedRowid(BtCursor*, sqlite3_int64); -SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeGetCachedRowid(BtCursor*); SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *); +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); - SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); +SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); #ifndef NDEBUG SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); @@ -8146,6 +9208,7 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); */ #ifndef _SQLITE_VDBE_H_ #define _SQLITE_VDBE_H_ +/* #include */ /* ** A single VDBE is an opaque structure named "Vdbe". Only routines @@ -8158,7 +9221,6 @@ typedef struct Vdbe Vdbe; ** The names of the following types declared in vdbeInt.h are required ** for the VdbeOp definition. */ -typedef struct VdbeFunc VdbeFunc; typedef struct Mem Mem; typedef struct SubProgram SubProgram; @@ -8182,21 +9244,24 @@ struct VdbeOp { i64 *pI64; /* Used when p4type is P4_INT64 */ double *pReal; /* Used when p4type is P4_REAL */ FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ - VdbeFunc *pVdbeFunc; /* Used when p4type is P4_VDBEFUNC */ CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ Mem *pMem; /* Used when p4type is P4_MEM */ VTable *pVtab; /* Used when p4type is P4_VTAB */ KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + int (*xAdvance)(BtCursor *, int *); } p4; -#ifdef SQLITE_DEBUG +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS char *zComment; /* Comment to improve readability */ #endif #ifdef VDBE_PROFILE - int cnt; /* Number of times this instruction was executed */ + u32 cnt; /* Number of times this instruction was executed */ u64 cycles; /* Total time spent executing this instruction */ #endif +#ifdef SQLITE_VDBE_COVERAGE + int iSrcLine; /* Source-code line that generated this opcode */ +#endif }; typedef struct VdbeOp VdbeOp; @@ -8209,6 +9274,7 @@ struct SubProgram { int nOp; /* Elements in aOp[] */ int nMem; /* Number of memory cells required */ int nCsr; /* Number of cursors required */ + int nOnce; /* Number of OP_Once instructions */ void *token; /* id that may be used to recursive triggers */ SubProgram *pNext; /* Next sub-program already visited */ }; @@ -8234,7 +9300,6 @@ typedef struct VdbeOpList VdbeOpList; #define P4_COLLSEQ (-4) /* P4 is a pointer to a CollSeq structure */ #define P4_FUNCDEF (-5) /* P4 is a pointer to a FuncDef structure */ #define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ -#define P4_VDBEFUNC (-7) /* P4 is a pointer to a VdbeFunc structure */ #define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ #define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ #define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ @@ -8244,16 +9309,13 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INT32 (-14) /* P4 is a 32-bit signed integer */ #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ +#define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ -/* When adding a P4 argument using P4_KEYINFO, a copy of the KeyInfo structure -** is made. That copy is freed when the Vdbe is finalized. But if the -** argument is P4_KEYINFO_HANDOFF, the passed in pointer is used. It still -** gets freed when the Vdbe is finalized so it still should be obtained -** from a single sqliteMalloc(). But no copy is made and the calling -** function should *not* try to free the KeyInfo. -*/ -#define P4_KEYINFO_HANDOFF (-16) -#define P4_KEYINFO_STATIC (-17) +/* Error message codes for OP_Halt */ +#define P5_ConstraintNotNull 1 +#define P5_ConstraintUnique 2 +#define P5_ConstraintCheck 3 +#define P5_ConstraintFK 4 /* ** The Vdbe.aColName array contains 5n Mem structures, where n is the @@ -8290,153 +9352,161 @@ typedef struct VdbeOpList VdbeOpList; /************** Begin file opcodes.h *****************************************/ /* Automatically generated. Do not edit */ /* See the mkopcodeh.awk script for details */ -#define OP_Goto 1 -#define OP_Gosub 2 -#define OP_Return 3 -#define OP_Yield 4 -#define OP_HaltIfNull 5 -#define OP_Halt 6 -#define OP_Integer 7 -#define OP_Int64 8 -#define OP_Real 130 /* same as TK_FLOAT */ -#define OP_String8 94 /* same as TK_STRING */ -#define OP_String 9 -#define OP_Null 10 -#define OP_Blob 11 -#define OP_Variable 12 -#define OP_Move 13 -#define OP_Copy 14 -#define OP_SCopy 15 -#define OP_ResultRow 16 -#define OP_Concat 91 /* same as TK_CONCAT */ -#define OP_Add 86 /* same as TK_PLUS */ -#define OP_Subtract 87 /* same as TK_MINUS */ -#define OP_Multiply 88 /* same as TK_STAR */ -#define OP_Divide 89 /* same as TK_SLASH */ -#define OP_Remainder 90 /* same as TK_REM */ -#define OP_CollSeq 17 -#define OP_Function 18 -#define OP_BitAnd 82 /* same as TK_BITAND */ -#define OP_BitOr 83 /* same as TK_BITOR */ -#define OP_ShiftLeft 84 /* same as TK_LSHIFT */ -#define OP_ShiftRight 85 /* same as TK_RSHIFT */ -#define OP_AddImm 20 -#define OP_MustBeInt 21 -#define OP_RealAffinity 22 -#define OP_ToText 141 /* same as TK_TO_TEXT */ -#define OP_ToBlob 142 /* same as TK_TO_BLOB */ -#define OP_ToNumeric 143 /* same as TK_TO_NUMERIC*/ -#define OP_ToInt 144 /* same as TK_TO_INT */ -#define OP_ToReal 145 /* same as TK_TO_REAL */ -#define OP_Eq 76 /* same as TK_EQ */ -#define OP_Ne 75 /* same as TK_NE */ -#define OP_Lt 79 /* same as TK_LT */ -#define OP_Le 78 /* same as TK_LE */ -#define OP_Gt 77 /* same as TK_GT */ -#define OP_Ge 80 /* same as TK_GE */ -#define OP_Permutation 23 -#define OP_Compare 24 -#define OP_Jump 25 -#define OP_And 69 /* same as TK_AND */ -#define OP_Or 68 /* same as TK_OR */ -#define OP_Not 19 /* same as TK_NOT */ -#define OP_BitNot 93 /* same as TK_BITNOT */ -#define OP_If 26 -#define OP_IfNot 27 -#define OP_IsNull 73 /* same as TK_ISNULL */ -#define OP_NotNull 74 /* same as TK_NOTNULL */ -#define OP_Column 28 -#define OP_Affinity 29 -#define OP_MakeRecord 30 -#define OP_Count 31 -#define OP_Savepoint 32 -#define OP_AutoCommit 33 -#define OP_Transaction 34 -#define OP_ReadCookie 35 -#define OP_SetCookie 36 -#define OP_VerifyCookie 37 -#define OP_OpenRead 38 -#define OP_OpenWrite 39 -#define OP_OpenAutoindex 40 -#define OP_OpenEphemeral 41 -#define OP_OpenPseudo 42 -#define OP_Close 43 -#define OP_SeekLt 44 -#define OP_SeekLe 45 -#define OP_SeekGe 46 -#define OP_SeekGt 47 -#define OP_Seek 48 -#define OP_NotFound 49 -#define OP_Found 50 -#define OP_IsUnique 51 -#define OP_NotExists 52 -#define OP_Sequence 53 -#define OP_NewRowid 54 -#define OP_Insert 55 -#define OP_InsertInt 56 -#define OP_Delete 57 -#define OP_ResetCount 58 -#define OP_RowKey 59 -#define OP_RowData 60 -#define OP_Rowid 61 -#define OP_NullRow 62 -#define OP_Last 63 -#define OP_Sort 64 -#define OP_Rewind 65 -#define OP_Prev 66 -#define OP_Next 67 -#define OP_IdxInsert 70 -#define OP_IdxDelete 71 -#define OP_IdxRowid 72 -#define OP_IdxLT 81 -#define OP_IdxGE 92 -#define OP_Destroy 95 -#define OP_Clear 96 -#define OP_CreateIndex 97 -#define OP_CreateTable 98 -#define OP_ParseSchema 99 -#define OP_LoadAnalysis 100 -#define OP_DropTable 101 -#define OP_DropIndex 102 -#define OP_DropTrigger 103 -#define OP_IntegrityCk 104 -#define OP_RowSetAdd 105 -#define OP_RowSetRead 106 -#define OP_RowSetTest 107 -#define OP_Program 108 -#define OP_Param 109 -#define OP_FkCounter 110 -#define OP_FkIfZero 111 -#define OP_MemMax 112 -#define OP_IfPos 113 -#define OP_IfNeg 114 -#define OP_IfZero 115 -#define OP_AggStep 116 -#define OP_AggFinal 117 -#define OP_Checkpoint 118 -#define OP_JournalMode 119 -#define OP_Vacuum 120 -#define OP_IncrVacuum 121 -#define OP_Expire 122 -#define OP_TableLock 123 -#define OP_VBegin 124 -#define OP_VCreate 125 -#define OP_VDestroy 126 -#define OP_VOpen 127 -#define OP_VFilter 128 -#define OP_VColumn 129 -#define OP_VNext 131 -#define OP_VRename 132 -#define OP_VUpdate 133 -#define OP_Pagecount 134 -#define OP_MaxPgcnt 135 -#define OP_Trace 136 -#define OP_Noop 137 -#define OP_Explain 138 - -/* The following opcode values are never used */ -#define OP_NotUsed_139 139 -#define OP_NotUsed_140 140 +#define OP_Function 1 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_Savepoint 2 +#define OP_AutoCommit 3 +#define OP_Transaction 4 +#define OP_SorterNext 5 +#define OP_PrevIfOpen 6 +#define OP_NextIfOpen 7 +#define OP_Prev 8 +#define OP_Next 9 +#define OP_AggStep 10 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_Checkpoint 11 +#define OP_JournalMode 12 +#define OP_Vacuum 13 +#define OP_VFilter 14 /* synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 15 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 16 +#define OP_Gosub 17 +#define OP_Return 18 +#define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ +#define OP_InitCoroutine 20 +#define OP_EndCoroutine 21 +#define OP_Yield 22 +#define OP_HaltIfNull 23 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 24 +#define OP_Integer 25 /* synopsis: r[P2]=P1 */ +#define OP_Int64 26 /* synopsis: r[P2]=P4 */ +#define OP_String 27 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 28 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 29 /* synopsis: r[P1]=NULL */ +#define OP_Blob 30 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 31 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 32 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 33 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 34 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 35 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 36 +#define OP_AddImm 37 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_MustBeInt 38 +#define OP_RealAffinity 39 +#define OP_Cast 40 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 41 +#define OP_Compare 42 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_Jump 43 +#define OP_Once 44 +#define OP_If 45 +#define OP_IfNot 46 +#define OP_Column 47 /* synopsis: r[P3]=PX */ +#define OP_Affinity 48 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 49 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 50 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 51 +#define OP_SetCookie 52 +#define OP_ReopenIdx 53 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 54 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 55 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenAutoindex 56 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 57 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 58 +#define OP_SequenceTest 59 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 60 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 61 +#define OP_SeekLT 62 /* synopsis: key=r[P3@P4] */ +#define OP_SeekLE 63 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGE 64 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGT 65 /* synopsis: key=r[P3@P4] */ +#define OP_Seek 66 /* synopsis: intkey=r[P2] */ +#define OP_NoConflict 67 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 68 /* synopsis: key=r[P3@P4] */ +#define OP_Found 69 /* synopsis: key=r[P3@P4] */ +#define OP_NotExists 70 /* synopsis: intkey=r[P3] */ +#define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ +#define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +#define OP_Sequence 73 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 74 /* synopsis: r[P2]=rowid */ +#define OP_Insert 75 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ +#define OP_Eq 79 /* same as TK_EQ, synopsis: if r[P1]==r[P3] goto P2 */ +#define OP_Gt 80 /* same as TK_GT, synopsis: if r[P1]>r[P3] goto P2 */ +#define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ +#define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ +#define OP_InsertInt 84 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 89 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 90 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 91 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 92 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 93 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 94 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_Delete 95 +#define OP_BitNot 96 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ +#define OP_String8 97 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_ResetCount 98 +#define OP_SorterCompare 99 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 100 /* synopsis: r[P2]=data */ +#define OP_RowKey 101 /* synopsis: r[P2]=key */ +#define OP_RowData 102 /* synopsis: r[P2]=data */ +#define OP_Rowid 103 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 104 +#define OP_Last 105 +#define OP_SorterSort 106 +#define OP_Sort 107 +#define OP_Rewind 108 +#define OP_SorterInsert 109 +#define OP_IdxInsert 110 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 111 /* synopsis: key=r[P2@P3] */ +#define OP_IdxRowid 112 /* synopsis: r[P2]=rowid */ +#define OP_IdxLE 113 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGT 114 /* synopsis: key=r[P3@P4] */ +#define OP_IdxLT 115 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGE 116 /* synopsis: key=r[P3@P4] */ +#define OP_Destroy 117 +#define OP_Clear 118 +#define OP_ResetSorter 119 +#define OP_CreateIndex 120 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_CreateTable 121 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_ParseSchema 122 +#define OP_LoadAnalysis 123 +#define OP_DropTable 124 +#define OP_DropIndex 125 +#define OP_DropTrigger 126 +#define OP_IntegrityCk 127 +#define OP_RowSetAdd 128 /* synopsis: rowset(P1)=r[P2] */ +#define OP_RowSetRead 129 /* synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 130 /* synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 131 +#define OP_Param 132 +#define OP_Real 133 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_FkCounter 134 /* synopsis: fkctr[P1]+=P2 */ +#define OP_FkIfZero 135 /* synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_MemMax 136 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_IfPos 137 /* synopsis: if r[P1]>0 goto P2 */ +#define OP_IfNeg 138 /* synopsis: r[P1]+=P3, if r[P1]<0 goto P2 */ +#define OP_IfZero 139 /* synopsis: r[P1]+=P3, if r[P1]==0 goto P2 */ +#define OP_AggFinal 140 /* synopsis: accum=r[P1] N=P2 */ +#define OP_IncrVacuum 141 +#define OP_Expire 142 +#define OP_TableLock 143 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 144 +#define OP_VCreate 145 +#define OP_VDestroy 146 +#define OP_VOpen 147 +#define OP_VColumn 148 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VNext 149 +#define OP_VRename 150 +#define OP_Pagecount 151 +#define OP_MaxPgcnt 152 +#define OP_Init 153 /* synopsis: Start at P2 */ +#define OP_Noop 154 +#define OP_Explain 155 /* Properties such as "out2" or "jump" that are specified in @@ -8451,25 +9521,26 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT2 0x0020 /* out2: P2 is an output */ #define OPFLG_OUT3 0x0040 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x01, 0x05, 0x04, 0x04, 0x10, 0x00, 0x02,\ -/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x00, 0x24, 0x24,\ -/* 16 */ 0x00, 0x00, 0x00, 0x24, 0x04, 0x05, 0x04, 0x00,\ -/* 24 */ 0x00, 0x01, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\ -/* 32 */ 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00, 0x00,\ -/* 40 */ 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11,\ -/* 48 */ 0x08, 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x00,\ -/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01,\ -/* 64 */ 0x01, 0x01, 0x01, 0x01, 0x4c, 0x4c, 0x08, 0x00,\ -/* 72 */ 0x02, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\ -/* 80 */ 0x15, 0x01, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c,\ -/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x01, 0x24, 0x02, 0x02,\ -/* 96 */ 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 104 */ 0x00, 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01,\ -/* 112 */ 0x08, 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\ -/* 120 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x02,\ -/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\ -/* 144 */ 0x04, 0x04,} +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,\ +/* 8 */ 0x01, 0x01, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00,\ +/* 16 */ 0x01, 0x01, 0x04, 0x24, 0x01, 0x04, 0x05, 0x10,\ +/* 24 */ 0x00, 0x02, 0x02, 0x02, 0x02, 0x00, 0x02, 0x02,\ +/* 32 */ 0x00, 0x00, 0x20, 0x00, 0x00, 0x04, 0x05, 0x04,\ +/* 40 */ 0x04, 0x00, 0x00, 0x01, 0x01, 0x05, 0x05, 0x00,\ +/* 48 */ 0x00, 0x00, 0x02, 0x02, 0x10, 0x00, 0x00, 0x00,\ +/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x11,\ +/* 64 */ 0x11, 0x11, 0x08, 0x11, 0x11, 0x11, 0x11, 0x4c,\ +/* 72 */ 0x4c, 0x02, 0x02, 0x00, 0x05, 0x05, 0x15, 0x15,\ +/* 80 */ 0x15, 0x15, 0x15, 0x15, 0x00, 0x4c, 0x4c, 0x4c,\ +/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x00,\ +/* 96 */ 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,\ +/* 104 */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x08, 0x08, 0x00,\ +/* 112 */ 0x02, 0x01, 0x01, 0x01, 0x01, 0x02, 0x00, 0x00,\ +/* 120 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 128 */ 0x0c, 0x45, 0x15, 0x01, 0x02, 0x02, 0x00, 0x01,\ +/* 136 */ 0x08, 0x05, 0x05, 0x05, 0x00, 0x01, 0x00, 0x00,\ +/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02,\ +/* 152 */ 0x02, 0x01, 0x00, 0x00,} /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -8478,35 +9549,36 @@ typedef struct VdbeOpList VdbeOpList; ** Prototypes for the VDBE interface. See comments on the implementation ** for a description of what each of these routines does. */ -SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(sqlite3*); +SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); -SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe*, int addr, int N); +SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); +SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); -SQLITE_PRIVATE void sqlite3VdbeTrace(Vdbe*,FILE*); #endif SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe*); @@ -8518,29 +9590,82 @@ SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, int); SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); -SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetValue(Vdbe*, int, u8); +SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif +SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); -SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,char*,int); -SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*); +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); + +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif - -#ifndef NDEBUG +/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on +** each VDBE opcode. +** +** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op +** comments in VDBE programs that show key decision points in the code +** generator. +*/ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe*, const char*, ...); # define VdbeComment(X) sqlite3VdbeComment X SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); # define VdbeNoopComment(X) sqlite3VdbeNoopComment X +# ifdef SQLITE_ENABLE_MODULE_COMMENTS +# define VdbeModuleComment(X) sqlite3VdbeNoopComment X +# else +# define VdbeModuleComment(X) +# endif #else # define VdbeComment(X) # define VdbeNoopComment(X) +# define VdbeModuleComment(X) +#endif + +/* +** The VdbeCoverage macros are used to set a coverage testing point +** for VDBE branch instructions. The coverage testing points are line +** numbers in the sqlite3.c source file. VDBE branch coverage testing +** only works with an amalagmation build. That's ok since a VDBE branch +** coverage build designed for testing the test suite only. No application +** should ever ship with VDBE branch coverage measuring turned on. +** +** VdbeCoverage(v) // Mark the previously coded instruction +** // as a branch +** +** VdbeCoverageIf(v, conditional) // Mark previous if conditional true +** +** VdbeCoverageAlwaysTaken(v) // Previous branch is always taken +** +** VdbeCoverageNeverTaken(v) // Previous branch is never taken +** +** Every VDBE branch operation must be tagged with one of the macros above. +** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and +** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() +** routine in vdbe.c, alerting the developer to the missed tag. +*/ +#ifdef SQLITE_VDBE_COVERAGE +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); +# define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageAlwaysTaken(v) sqlite3VdbeSetLineNumber(v,2); +# define VdbeCoverageNeverTaken(v) sqlite3VdbeSetLineNumber(v,1); +# define VDBE_OFFSET_LINENO(x) (__LINE__+x) +#else +# define VdbeCoverage(v) +# define VdbeCoverageIf(v,x) +# define VdbeCoverageAlwaysTaken(v) +# define VdbeCoverageNeverTaken(v) +# define VDBE_OFFSET_LINENO(x) 0 #endif #endif @@ -8609,8 +9734,7 @@ typedef struct PgHdr DbPage; ** NOTE: These values must match the corresponding BTREE_ values in btree.h. */ #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ -#define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ -#define PAGER_MEMORY 0x0004 /* In-memory database */ +#define PAGER_MEMORY 0x0002 /* In-memory database */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). @@ -8630,6 +9754,24 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_GET_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_GET_READONLY 0x02 /* Read-only page is acceptable */ + +/* +** Flags for sqlite3PagerSetFlags() +*/ +#define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ +#define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ +#define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ +#define PAGER_SYNCHRONOUS_MASK 0x03 /* Mask for three values above */ +#define PAGER_FULLFSYNC 0x04 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x08 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x10 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x1c /* All above except SYNCHRONOUS */ + /* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for @@ -8654,7 +9796,9 @@ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); -SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); +SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); +SQLITE_PRIVATE void sqlite3PagerShrink(Pager*); +SQLITE_PRIVATE void sqlite3PagerSetFlags(Pager*,unsigned); SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int); SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *, int); SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); @@ -8668,6 +9812,7 @@ SQLITE_PRIVATE int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); SQLITE_PRIVATE void sqlite3PagerRef(DbPage*); SQLITE_PRIVATE void sqlite3PagerUnref(DbPage*); +SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage*); /* Operations on page references. */ SQLITE_PRIVATE int sqlite3PagerWrite(DbPage*); @@ -8682,30 +9827,39 @@ SQLITE_PRIVATE void sqlite3PagerPagecount(Pager*, int*); SQLITE_PRIVATE int sqlite3PagerBegin(Pager*, int exFlag, int); SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*); -SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster); SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*); SQLITE_PRIVATE int sqlite3PagerRollback(Pager*); SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n); SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); -SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); -SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); +SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +#endif + +#ifdef SQLITE_ENABLE_ZIPVFS +SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); +#endif /* Functions used to query pager state and configuration. */ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*); SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); -SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*); +SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); SQLITE_PRIVATE int sqlite3PagerNosync(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *); +SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); @@ -8760,11 +9914,12 @@ typedef struct PCache PCache; ** structure. */ struct PgHdr { - void *pData; /* Content of this page */ + sqlite3_pcache_page *pPage; /* Pcache object page handle */ + void *pData; /* Page data */ void *pExtra; /* Extra content */ PgHdr *pDirty; /* Transient list of dirty pages */ - Pgno pgno; /* Page number for this page */ Pager *pPager; /* The pager this page is part of */ + Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES u32 pageHash; /* Hash of page content */ #endif @@ -8789,6 +9944,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void); SQLITE_PRIVATE void sqlite3PcacheShutdown(void); @@ -8802,7 +9959,7 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *, int sz, int n); ** Under memory stress, invoke xStress to try to make pages clean. ** Only clean and unpinned pages can be reclaimed. */ -SQLITE_PRIVATE void sqlite3PcacheOpen( +SQLITE_PRIVATE int sqlite3PcacheOpen( int szPage, /* Size of every page */ int szExtra, /* Extra space associated with each page */ int bPurgeable, /* True if pages are on backing store */ @@ -8812,7 +9969,7 @@ SQLITE_PRIVATE void sqlite3PcacheOpen( ); /* Modify the page-size after the cache has been created. */ -SQLITE_PRIVATE void sqlite3PcacheSetPageSize(PCache *, int); +SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *, int); /* Return the size in bytes of a PCache object. Used to preallocate ** storage space. @@ -8822,7 +9979,9 @@ SQLITE_PRIVATE int sqlite3PcacheSize(void); /* One release per successful fetch. Page is pinned until released. ** Reference counted. */ -SQLITE_PRIVATE int sqlite3PcacheFetch(PCache*, Pgno, int createFlag, PgHdr**); +SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch(PCache*, Pgno, int createFlag); +SQLITE_PRIVATE int sqlite3PcacheFetchStress(PCache*, Pgno, sqlite3_pcache_page**); +SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish(PCache*, Pgno, sqlite3_pcache_page *pPage); SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr*); SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ @@ -8878,6 +10037,9 @@ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *, int); SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *); #endif +/* Free up as much memory as possible from the page cache */ +SQLITE_PRIVATE void sqlite3PcacheShrink(PCache*); + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* Try to return memory used by the pcache module to the main memory heap */ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int); @@ -8919,84 +10081,71 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); #define _SQLITE_OS_H_ /* -** Figure out if we are dealing with Unix, Windows, or some other -** operating system. After the following block of preprocess macros, -** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, SQLITE_OS_OS2, and SQLITE_OS_OTHER -** will defined to either 1 or 0. One of the four will be 1. The other -** three will be 0. +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. +*/ +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef _OS_SETUP_H_ +#define _OS_SETUP_H_ + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of SQLITE_OS_UNIX, +** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of +** the three will be 1. The other two will be 0. */ #if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# undef SQLITE_OS_OS2 -# define SQLITE_OS_OS2 0 -# else -# undef SQLITE_OS_OTHER -# endif +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# else +# undef SQLITE_OS_OTHER +# endif #endif #if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# define SQLITE_OS_OS2 0 -# elif defined(__EMX__) || defined(_OS2) || defined(OS2) || defined(_OS2_) || defined(__OS2__) -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 0 -# define SQLITE_OS_OS2 1 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# define SQLITE_OS_OS2 0 +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +# else +# define SQLITE_OS_UNIX 0 # endif -# else -# define SQLITE_OS_UNIX 0 -# define SQLITE_OS_OS2 0 -# endif #else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif #endif -/* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. -*/ -#if defined(_WIN32_WCE) -# define SQLITE_OS_WINCE 1 -#else -# define SQLITE_OS_WINCE 0 -#endif +#endif /* _OS_SETUP_H_ */ - -/* -** Define the maximum size of a temporary filename -*/ -#if SQLITE_OS_WIN -# include -# define SQLITE_TEMPNAME_SIZE (MAX_PATH+50) -#elif SQLITE_OS_OS2 -# if (__GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 3) && defined(OS2_HIGH_MEMORY) -# include /* has to be included before os2.h for linking to work */ -# endif -# define INCL_DOSDATETIME -# define INCL_DOSFILEMGR -# define INCL_DOSERRORS -# define INCL_DOSMISC -# define INCL_DOSPROCESS -# define INCL_DOSMODULEMGR -# define INCL_DOSSEMAPHORES -# include -# include -# define SQLITE_TEMPNAME_SIZE (CCHMAXPATHCOMP) -#else -# define SQLITE_TEMPNAME_SIZE 200 -#endif +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op @@ -9009,7 +10158,7 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); ** The default size of a disk sector */ #ifndef SQLITE_DEFAULT_SECTOR_SIZE -# define SQLITE_DEFAULT_SECTOR_SIZE 512 +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 #endif /* @@ -9092,7 +10241,7 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); ** shared locks begins at SHARED_FIRST. ** ** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possiblity of having +** byte ranges are used for Unix. This leaves open the possibility of having ** clients on win95, winNT, and unix all talking to the same shared file ** and all locking correctly. To do so would require that samba (or whatever ** tool is being used for file sharing) implements locks correctly between @@ -9142,6 +10291,7 @@ SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); #define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); @@ -9149,6 +10299,9 @@ SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); + /* ** Functions for accessing sqlite3_vfs methods @@ -9207,7 +10360,7 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); ** Figure out what version of the code to use. The choices are ** ** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The -** mutexes implemention cannot be overridden +** mutexes implementation cannot be overridden ** at start-time. ** ** SQLITE_MUTEX_NOOP For single-threaded applications. No @@ -9218,8 +10371,6 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); ** SQLITE_MUTEX_PTHREADS For multi-threaded applications on Unix. ** ** SQLITE_MUTEX_W32 For multi-threaded applications on Win32. -** -** SQLITE_MUTEX_OS2 For multi-threaded applications on OS/2. */ #if !SQLITE_THREADSAFE # define SQLITE_MUTEX_OMIT @@ -9229,8 +10380,6 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); # define SQLITE_MUTEX_PTHREADS # elif SQLITE_OS_WIN # define SQLITE_MUTEX_W32 -# elif SQLITE_OS_OS2 -# define SQLITE_MUTEX_OS2 # else # define SQLITE_MUTEX_NOOP # endif @@ -9242,14 +10391,17 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); */ #define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) #define sqlite3_mutex_free(X) -#define sqlite3_mutex_enter(X) +#define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK -#define sqlite3_mutex_leave(X) +#define sqlite3_mutex_leave(X) #define sqlite3_mutex_held(X) ((void)(X),1) #define sqlite3_mutex_notheld(X) ((void)(X),1) #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) #define sqlite3MutexInit() SQLITE_OK #define sqlite3MutexEnd() +#define MUTEX_LOGIC(X) +#else +#define MUTEX_LOGIC(X) X #endif /* defined(SQLITE_MUTEX_OMIT) */ /************** End of mutex.h ***********************************************/ @@ -9266,7 +10418,6 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); struct Db { char *zName; /* Name of this database */ Btree *pBt; /* The B*Tree structure for this database file */ - u8 inTrans; /* 0: not writable. 1: Transaction. 2: Checkpoint */ u8 safety_level; /* How aggressive at syncing data to disk */ Schema *pSchema; /* Pointer to database schema (possibly shared) */ }; @@ -9298,7 +10449,7 @@ struct Schema { Table *pSeqTab; /* The sqlite_sequence table used by AUTOINCREMENT */ u8 file_format; /* Schema format version for this file */ u8 enc; /* Text encoding used by this database */ - u16 flags; /* Flags associated with this schema */ + u16 schemaFlags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ }; @@ -9306,10 +10457,10 @@ struct Schema { ** These macros can be used to test, set, or clear bits in the ** Db.pSchema->flags field. */ -#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))==(P)) -#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))!=0) -#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->flags|=(P) -#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->flags&=~(P) +#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) +#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))!=0) +#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags|=(P) +#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags&=~(P) /* ** Allowed values for the DB.pSchema->flags field. @@ -9329,7 +10480,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_TRIGGER_DEPTH+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -9376,40 +10527,62 @@ struct FuncDefHash { FuncDef *a[23]; /* Hash table for functions */ }; +#ifdef SQLITE_USER_AUTHENTICATION +/* +** Information held in the "sqlite3" database connection object and used +** to manage user authentication. +*/ +typedef struct sqlite3_userauth sqlite3_userauth; +struct sqlite3_userauth { + u8 authLevel; /* Current authentication level */ + int nAuthPW; /* Size of the zAuthPW in bytes */ + char *zAuthPW; /* Password used to authenticate */ + char *zAuthUser; /* User name used to authenticate */ +}; + +/* Allowed values for sqlite3_userauth.authLevel */ +#define UAUTH_Unknown 0 /* Authentication not yet checked */ +#define UAUTH_Fail 1 /* User authentication failed */ +#define UAUTH_User 2 /* Authenticated as a normal user */ +#define UAUTH_Admin 3 /* Authenticated as an administrator */ + +/* Functions used only by user authorization logic */ +SQLITE_PRIVATE int sqlite3UserAuthTable(const char*); +SQLITE_PRIVATE int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); +SQLITE_PRIVATE void sqlite3UserAuthInit(sqlite3*); +SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); + +#endif /* SQLITE_USER_AUTHENTICATION */ + +/* +** typedef for the authorization callback function. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*, const char*); +#else + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*); +#endif + + /* ** Each database connection is an instance of the following structure. -** -** The sqlite.lastRowid records the last insert rowid generated by an -** insert statement. Inserts on views do not affect its value. Each -** trigger has its own context, so that lastRowid can be updated inside -** triggers as usual. The previous value will be restored once the trigger -** exits. Upon entering a before or instead of trigger, lastRowid is no -** longer (since after version 2.8.12) reset to -1. -** -** The sqlite.nChange does not count changes within triggers and keeps no -** context. It is reset at start of sqlite3_exec. -** The sqlite.lsChange represents the number of changes made by the last -** insert, update, or delete statement. It remains constant throughout the -** length of a statement and is then updated by OP_SetCounts. It keeps a -** context stack just like lastRowid so that the count of changes -** within a trigger is not seen outside the trigger. Changes to views do not -** affect the value of lsChange. -** The sqlite.csChange keeps track of the number of current changes (since -** the last statement) and is used to update sqlite_lsChange. -** -** The member variables sqlite.errCode, sqlite.zErrMsg and sqlite.zErrMsg16 -** store the most recent error code and, if applicable, string. The -** internal function sqlite3Error() is used to set these variables -** consistently. */ struct sqlite3 { sqlite3_vfs *pVfs; /* OS Interface */ - int nDb; /* Number of backends currently in use */ + struct Vdbe *pVdbe; /* List of active virtual machines */ + CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ + sqlite3_mutex *mutex; /* Connection mutex */ Db *aDb; /* All backends */ + int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ + i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 szMmap; /* Default mmap_size setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -9417,27 +10590,25 @@ struct sqlite3 { signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ + u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ int nextPagesize; /* Pagesize after VACUUM if >0 */ - int nTable; /* Number of tables in the database */ - CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ - i64 lastRowid; /* ROWID of most recent insert (see above) */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ - sqlite3_mutex *mutex; /* Connection mutex */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ + int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ - int iDb; /* When back is being initialized */ int newTnum; /* Rootpage of table being initialized */ + u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */ } init; + int nVdbeActive; /* Number of VDBEs currently running */ + int nVdbeRead; /* Number of active VDBEs that read or write */ + int nVdbeWrite; /* Number of active VDBEs that read and write */ + int nVdbeExec; /* Number of nested calls to VdbeExec() */ int nExtension; /* Number of loaded extensions */ void **aExtension; /* Array of shared library handles */ - struct Vdbe *pVdbe; /* List of active virtual machines */ - int activeVdbeCnt; /* Number of VDBEs currently executing */ - int writeVdbeCnt; /* Number of active VDBEs that are writing */ - int vdbeExecCnt; /* Number of nested calls to VdbeExec() */ void (*xTrace)(void*,const char*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ void (*xProfile)(void*,const char*,u64); /* Profiling function */ @@ -9456,42 +10627,38 @@ struct sqlite3 { void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*); void *pCollNeededArg; sqlite3_value *pErr; /* Most recent error message */ - char *zErrMsg; /* Most recent error message (UTF-8 encoded) */ - char *zErrMsg16; /* Most recent error message (UTF-16 encoded) */ union { volatile int isInterrupted; /* True if sqlite3_interrupt has been called */ double notUsed1; /* Spacer */ } u1; Lookaside lookaside; /* Lookaside malloc configuration */ #ifndef SQLITE_OMIT_AUTHORIZATION - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); - /* Access authorization function */ + sqlite3_xauth xAuth; /* Access authorization function */ void *pAuthArg; /* 1st argument to the access auth function */ #endif #ifndef SQLITE_OMIT_PROGRESS_CALLBACK int (*xProgress)(void *); /* The progress callback */ void *pProgressArg; /* Argument to the progress callback */ - int nProgressOps; /* Number of opcodes for progress callback */ + unsigned nProgressOps; /* Number of opcodes for progress callback */ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE + int nVTrans; /* Allocated size of aVTrans */ Hash aModule; /* populated by sqlite3_create_module() */ VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ VTable **aVTrans; /* Virtual tables with open transactions */ - int nVTrans; /* Allocated size of aVTrans */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ #endif FuncDefHash aFunc; /* Hash table of connection functions */ Hash aCollSeq; /* All collating sequences */ BusyHandler busyHandler; /* Busy callback */ - int busyTimeout; /* Busy handler timeout, in msec */ Db aDbStatic[2]; /* Static space for the 2 default backends */ Savepoint *pSavepoint; /* List of active savepoints */ + int busyTimeout; /* Busy handler timeout, in msec */ int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ - u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ + i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ - #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MASTER ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -9509,6 +10676,9 @@ struct sqlite3 { void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ sqlite3 *pNextBlocked; /* Next in list of all blocked connections */ #endif +#ifdef SQLITE_USER_AUTHENTICATION + sqlite3_userauth auth; /* User authentication information */ +#endif }; /* @@ -9519,48 +10689,73 @@ struct sqlite3 { /* ** Possible values for the sqlite3.flags. */ -#define SQLITE_VdbeTrace 0x00000100 /* True to trace VDBE execution */ -#define SQLITE_InternChanges 0x00000200 /* Uncommitted Hash table changes */ -#define SQLITE_FullColNames 0x00000400 /* Show full column names on SELECT */ -#define SQLITE_ShortColNames 0x00000800 /* Show short columns names */ -#define SQLITE_CountRows 0x00001000 /* Count rows changed by INSERT, */ +#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ +#define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ +#define SQLITE_FullFSync 0x00000004 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00000008 /* Use full fsync for checkpoint */ +#define SQLITE_CacheSpill 0x00000010 /* OK to spill pager cache */ +#define SQLITE_FullColNames 0x00000020 /* Show full column names on SELECT */ +#define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ +#define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ /* the count using a callback. */ -#define SQLITE_NullCallback 0x00002000 /* Invoke the callback once if the */ +#define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ -#define SQLITE_SqlTrace 0x00004000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ -#define SQLITE_WriteSchema 0x00010000 /* OK to update SQLITE_MASTER */ -#define SQLITE_NoReadlock 0x00020000 /* Readlocks are omitted when - ** accessing read-only databases */ -#define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ -#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ -#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */ -#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ -#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ -#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ -#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ -#define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ -#define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ -#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x40000000 /* True to enable triggers */ +#define SQLITE_SqlTrace 0x00000200 /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing 0x00000400 /* Debug listings of VDBE programs */ +#define SQLITE_WriteSchema 0x00000800 /* OK to update SQLITE_MASTER */ +#define SQLITE_VdbeAddopTrace 0x00001000 /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_IgnoreChecks 0x00002000 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommitted 0x0004000 /* For shared-cache mode */ +#define SQLITE_LegacyFileFmt 0x00008000 /* Create new databases in format 1 */ +#define SQLITE_RecoveryMode 0x00010000 /* Ignore schema errors */ +#define SQLITE_ReverseOrder 0x00020000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00040000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00080000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00100000 /* Enable automatic indexes */ +#define SQLITE_PreferBuiltin 0x00200000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ +#define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ +#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ + /* -** Bits of the sqlite3.flags field that are used by the -** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface. -** These must be the low-order bits of the flags field. +** Bits of the sqlite3.dbOptFlags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to +** selectively disable various optimizations. */ -#define SQLITE_QueryFlattener 0x01 /* Disable query flattening */ -#define SQLITE_ColumnCache 0x02 /* Disable the column cache */ -#define SQLITE_IndexSort 0x04 /* Disable indexes for sorting */ -#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ -#define SQLITE_IndexCover 0x10 /* Disable index covering table */ -#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ -#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ -#define SQLITE_OptMask 0xff /* Mask of all disablable opts */ +#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ +#define SQLITE_ColumnCache 0x0002 /* Column cache */ +#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ +/* not used 0x0010 // Was: SQLITE_IdxRealAsInt */ +#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ +#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ +#define SQLITE_Transitive 0x0200 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ +#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ +#define SQLITE_AllOpts 0xffff /* All optimizations */ + +/* +** Macros for testing whether or not optimizations are enabled or disabled. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) +#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) +#else +#define OptimizationDisabled(db, mask) 0 +#define OptimizationEnabled(db, mask) 1 +#endif + +/* +** Return true if it OK to factor constant expressions into the initialization +** code. The argument is a Parse object for the code generator. +*/ +#define ConstFactorOk(P) ((P)->okConstFactor) /* ** Possible values for the sqlite.magic field. @@ -9572,6 +10767,7 @@ struct sqlite3 { #define SQLITE_MAGIC_SICK 0x4b771290 /* Error and awaiting close */ #define SQLITE_MAGIC_BUSY 0xf03b7906 /* Database currently in use */ #define SQLITE_MAGIC_ERROR 0xb5357930 /* An SQLITE_MISUSE error occurred */ +#define SQLITE_MAGIC_ZOMBIE 0x64cffc7f /* Close with last statement close */ /* ** Each SQL function is defined by an instance of the following @@ -9581,8 +10777,7 @@ struct sqlite3 { */ struct FuncDef { i16 nArg; /* Number of arguments. -1 means unlimited */ - u8 iPrefEnc; /* Preferred text encoding (SQLITE_UTF8, 16LE, 16BE) */ - u8 flags; /* Some combination of SQLITE_FUNC_* */ + u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ @@ -9614,15 +10809,22 @@ struct FuncDestructor { }; /* -** Possible values for FuncDef.flags +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** are assert() statements in the code to verify this. */ -#define SQLITE_FUNC_LIKE 0x01 /* Candidate for the LIKE optimization */ -#define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */ -#define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */ -#define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */ -#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ +#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -9635,6 +10837,9 @@ struct FuncDestructor { ** as the user-data (sqlite3_user_data()) for the function. If ** argument bNC is true, then the SQLITE_FUNC_NEEDCOLL flag is set. ** +** VFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. +** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters @@ -9650,15 +10855,25 @@ struct FuncDestructor { ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ - {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ + (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ - {nArg, SQLITE_UTF8, nc*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} +#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} /* @@ -9670,6 +10885,7 @@ struct FuncDestructor { struct Savepoint { char *zName; /* Savepoint name (nul-terminated) */ i64 nDeferredCons; /* Number of deferred fk violations */ + i64 nDeferredImmCons; /* Number of deferred imm fk. */ Savepoint *pNext; /* Parent savepoint (if any) */ }; @@ -9704,52 +10920,34 @@ struct Column { char *zDflt; /* Original text of the default value */ char *zType; /* Data type for this column */ char *zColl; /* Collating sequence. If NULL, use the default */ - u8 notNull; /* True if there is a NOT NULL constraint */ - u8 isPrimKey; /* True if this column is part of the PRIMARY KEY */ + u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - u8 isHidden; /* True if this column is 'hidden' */ -#endif + u8 szEst; /* Estimated size of this column. INT==1 */ + u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; +/* Allowed values for Column.colFlags: +*/ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ + /* ** A "Collating Sequence" is defined by an instance of the following ** structure. Conceptually, a collating sequence consists of a name and ** a comparison routine that defines the order of that sequence. ** -** There may two separate implementations of the collation function, one -** that processes text in UTF-8 encoding (CollSeq.xCmp) and another that -** processes text encoded in UTF-16 (CollSeq.xCmp16), using the machine -** native byte order. When a collation sequence is invoked, SQLite selects -** the version that will require the least expensive encoding -** translations, if any. -** -** The CollSeq.pUser member variable is an extra parameter that passed in -** as the first argument to the UTF-8 comparison function, xCmp. -** CollSeq.pUser16 is the equivalent for the UTF-16 comparison function, -** xCmp16. -** -** If both CollSeq.xCmp and CollSeq.xCmp16 are NULL, it means that the +** If CollSeq.xCmp is NULL, it means that the ** collating sequence is undefined. Indices built on an undefined ** collating sequence may not be read or written. */ struct CollSeq { char *zName; /* Name of the collating sequence, UTF-8 encoded */ u8 enc; /* Text encoding handled by xCmp() */ - u8 type; /* One of the SQLITE_COLL_... values below */ void *pUser; /* First argument to xCmp() */ int (*xCmp)(void*,int, const void*, int, const void*); void (*xDel)(void*); /* Destructor for pUser */ }; -/* -** Allowed values of CollSeq.type: -*/ -#define SQLITE_COLL_BINARY 1 /* The default memcmp() collating sequence */ -#define SQLITE_COLL_NOCASE 2 /* The built-in NOCASE collating sequence */ -#define SQLITE_COLL_REVERSE 3 /* The built-in REVERSE collating sequence */ -#define SQLITE_COLL_USER 0 /* Any other user-defined collating sequence */ - /* ** A sort order can be either ASC or DESC. */ @@ -9763,18 +10961,18 @@ struct CollSeq { ** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve ** the speed a little by numbering the values consecutively. ** -** But rather than start with 0 or 1, we begin with 'a'. That way, +** But rather than start with 0 or 1, we begin with 'A'. That way, ** when multiple affinity types are concatenated into a string and ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing -** for a numeric type is a single comparison. +** for a numeric type is a single comparison. And the NONE type is first. */ -#define SQLITE_AFF_TEXT 'a' -#define SQLITE_AFF_NONE 'b' -#define SQLITE_AFF_NUMERIC 'c' -#define SQLITE_AFF_INTEGER 'd' -#define SQLITE_AFF_REAL 'e' +#define SQLITE_AFF_NONE 'A' +#define SQLITE_AFF_TEXT 'B' +#define SQLITE_AFF_NUMERIC 'C' +#define SQLITE_AFF_INTEGER 'D' +#define SQLITE_AFF_REAL 'E' #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -9782,15 +10980,21 @@ struct CollSeq { ** The SQLITE_AFF_MASK values masks off the significant bits of an ** affinity value. */ -#define SQLITE_AFF_MASK 0x67 +#define SQLITE_AFF_MASK 0x47 /* ** Additional bit values that can be ORed with an affinity without ** changing the affinity. +** +** The SQLITE_NOTNULL flag is a combination of NULLEQ and JUMPIFNULL. +** It causes an assert() to fire if either operand to a comparison +** operator is NULL. It is added to certain comparison operators to +** prove that the operands are always NOT NULL. */ -#define SQLITE_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ -#define SQLITE_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ +#define SQLITE_JUMPIFNULL 0x10 /* jumps if either operand is NULL */ +#define SQLITE_STOREP2 0x20 /* Store result in reg[P2] rather than jump */ #define SQLITE_NULLEQ 0x80 /* NULL=NULL */ +#define SQLITE_NOTNULL 0x90 /* Assert that operands are never NULL */ /* ** An object of this type is created for each virtual table present in @@ -9876,28 +11080,32 @@ struct VTable { */ struct Table { char *zName; /* Name of the table or view */ - int iPKey; /* If not negative, use aCol[iPKey] as the primary key */ - int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ - int tnum; /* Root BTree node for this table (see note above) */ - unsigned nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ - u16 nRef; /* Number of pointers to this Table */ - u8 tabFlags; /* Mask of TF_* values */ - u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK - Expr *pCheck; /* The AND of all CHECK constraints */ + ExprList *pCheck; /* All CHECK constraints */ #endif + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ + int tnum; /* Root BTree node for this table (see note above) */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + i16 nCol; /* Number of columns in this table */ + u16 nRef; /* Number of pointers to this Table */ + LogEst szTabRow; /* Estimated size of each table row in bytes */ +#ifdef SQLITE_ENABLE_COSTMULT + LogEst costMult; /* Cost multiplier for using this table */ +#endif + u8 tabFlags; /* Mask of TF_* values */ + u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - VTable *pVTable; /* List of VTable objects. */ int nModuleArg; /* Number of arguments to the module */ char **azModuleArg; /* Text of all module args. [0] is module name */ + VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ Schema *pSchema; /* Schema that contains this table */ @@ -9905,15 +11113,14 @@ struct Table { }; /* -** Allowed values for Tabe.tabFlags. +** Allowed values for Table.tabFlags. */ #define TF_Readonly 0x01 /* Read-only system table */ #define TF_Ephemeral 0x02 /* An ephemeral table */ #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_NeedMetadata 0x20 /* aCol[].zType and aCol[].pColl missing */ - +#define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ /* @@ -9923,12 +11130,15 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) -# define IsHiddenColumn(X) ((X)->isHidden) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) #else # define IsVirtual(X) 0 # define IsHiddenColumn(X) 0 #endif +/* Does the table have a rowid */ +#define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) + /* ** Each foreign key constraint is an instance of the following structure. ** @@ -9943,26 +11153,35 @@ struct Table { ** ); ** ** For foreign key "fk1", the from-table is "ex1" and the to-table is "ex2". +** Equivalent names: +** +** from-table == child-table +** to-table == parent-table ** ** Each REFERENCES clause generates an instance of the following structure ** which is attached to the from-table. The to-table need not exist when ** the from-table is created. The existence of the to-table is not checked. +** +** The list of all parents for child Table X is held at X.pFKey. +** +** A list of all children for a table named Z (which might not even exist) +** is held in Schema.fkeyHash with a hash key of Z. */ struct FKey { Table *pFrom; /* Table containing the REFERENCES clause (aka: Child) */ - FKey *pNextFrom; /* Next foreign key in pFrom */ + FKey *pNextFrom; /* Next FKey with the same in pFrom. Next parent of pFrom */ char *zTo; /* Name of table that the key points to (aka: Parent) */ - FKey *pNextTo; /* Next foreign key on table named zTo */ - FKey *pPrevTo; /* Previous foreign key on table named zTo */ + FKey *pNextTo; /* Next with the same zTo. Next child of zTo. */ + FKey *pPrevTo; /* Previous with the same zTo */ int nCol; /* Number of columns in this key */ /* EV: R-30323-21917 */ - u8 isDeferred; /* True if constraint checking is deferred till COMMIT */ - u8 aAction[2]; /* ON DELETE and ON UPDATE actions, respectively */ - Trigger *apTrigger[2]; /* Triggers for aAction[] actions */ - struct sColMap { /* Mapping of columns in pFrom to columns in zTo */ - int iFrom; /* Index of column in pFrom */ - char *zCol; /* Name of column in zTo. If 0 use PRIMARY KEY */ - } aCol[1]; /* One entry for each of nCol column s */ + u8 isDeferred; /* True if constraint checking is deferred till COMMIT */ + u8 aAction[2]; /* ON DELETE and ON UPDATE actions, respectively */ + Trigger *apTrigger[2];/* Triggers for aAction[] actions */ + struct sColMap { /* Mapping of columns in pFrom to columns in zTo */ + int iFrom; /* Index of column in pFrom */ + char *zCol; /* Name of column in zTo. If NULL use PRIMARY KEY */ + } aCol[1]; /* One entry for each of nCol columns */ }; /* @@ -10002,19 +11221,25 @@ struct FKey { #define OE_SetDflt 8 /* Set the foreign key value to its default */ #define OE_Cascade 9 /* Cascade the changes */ -#define OE_Default 99 /* Do whatever the default action is */ +#define OE_Default 10 /* Do whatever the default action is */ /* ** An instance of the following structure is passed as the first ** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. +** +** Note that aSortOrder[] and aColl[] have nField+1 slots. There +** are nField slots for the columns of an index then one extra slot +** for the rowid at the end. */ struct KeyInfo { - sqlite3 *db; /* The database connection */ + u32 nRef; /* Number of references to this KeyInfo object */ u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ - u16 nField; /* Number of entries in aColl[] */ - u8 *aSortOrder; /* Sort order for each column. May be NULL */ + u16 nField; /* Number of key columns in the index */ + u16 nXField; /* Number of columns beyond the key columns */ + sqlite3 *db; /* The database connection */ + u8 *aSortOrder; /* Sort order for each column. */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; @@ -10031,24 +11256,20 @@ struct KeyInfo { ** ** This structure holds a record that has already been disassembled ** into its constituent fields. +** +** The r1 and r2 member variables are only used by the optimized comparison +** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ - u16 flags; /* Boolean settings. UNPACKED_... below */ - i64 rowid; /* Used by UNPACKED_PREFIX_SEARCH */ + i8 default_rc; /* Comparison result if keys are equal */ + u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ Mem *aMem; /* Values */ + int r1; /* Value to return if (lhs > rhs) */ + int r2; /* Value to return if (rhs < lhs) */ }; -/* -** Allowed values of UnpackedRecord.flags -*/ -#define UNPACKED_NEED_FREE 0x0001 /* Memory is from sqlite3Malloc() */ -#define UNPACKED_NEED_DESTROY 0x0002 /* apMem[]s should all be destroyed */ -#define UNPACKED_IGNORE_ROWID 0x0004 /* Ignore trailing rowid on key1 */ -#define UNPACKED_INCRKEY 0x0008 /* Make this key an epsilon larger */ -#define UNPACKED_PREFIX_MATCH 0x0010 /* A prefix match is considered OK */ -#define UNPACKED_PREFIX_SEARCH 0x0020 /* A prefix match is considered OK */ /* ** Each SQL index is represented in memory by an @@ -10077,34 +11298,60 @@ struct UnpackedRecord { ** element. */ struct Index { - char *zName; /* Name of this index */ - int nColumn; /* Number of columns in the table used by this index */ - int *aiColumn; /* Which columns are used by this index. 1st is 0 */ - unsigned *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ - Table *pTable; /* The SQL table being indexed */ - int tnum; /* Page containing root of this index in database file */ - u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ - u8 bUnordered; /* Use this index for == or IN queries only */ - char *zColAff; /* String defining the affinity of each column */ - Index *pNext; /* The next index associated with the same table */ - Schema *pSchema; /* Schema containing this index */ - u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ - char **azColl; /* Array of collation sequence names for index */ - IndexSample *aSample; /* Array of SQLITE_INDEX_SAMPLES samples */ + char *zName; /* Name of this index */ + i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ + Table *pTable; /* The SQL table being indexed */ + char *zColAff; /* String defining the affinity of each column */ + Index *pNext; /* The next index associated with the same table */ + Schema *pSchema; /* Schema containing this index */ + u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ + char **azColl; /* Array of collation sequence names for index */ + Expr *pPartIdxWhere; /* WHERE clause for partial indices */ + KeyInfo *pKeyInfo; /* A KeyInfo object suitable for this index */ + int tnum; /* DB Page containing root of this index */ + LogEst szIdxRow; /* Estimated average row size in bytes */ + u16 nKeyCol; /* Number of columns forming the key */ + u16 nColumn; /* Number of columns stored in the index */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + unsigned idxType:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned bUnordered:1; /* Use this index for == or IN queries only */ + unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ + unsigned isResized:1; /* True if resizeIndexObject() has been called */ + unsigned isCovering:1; /* True if this is a covering index */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int nSample; /* Number of elements in aSample[] */ + int nSampleCol; /* Size of IndexSample.anEq[] and so on */ + tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ + IndexSample *aSample; /* Samples of the left-most key */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this table */ +#endif }; /* -** Each sample stored in the sqlite_stat2 table is represented in memory -** using a structure of this type. +** Allowed values for Index.idxType +*/ +#define SQLITE_IDXTYPE_APPDEF 0 /* Created using CREATE INDEX */ +#define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */ +#define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */ + +/* Return true if index X is a PRIMARY KEY index */ +#define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) + +/* Return true if index X is a UNIQUE index */ +#define IsUniqueIndex(X) ((X)->onError!=OE_None) + +/* +** Each sample stored in the sqlite_stat3 table is represented in memory +** using a structure of this type. See documentation at the top of the +** analyze.c source file for additional information. */ struct IndexSample { - union { - char *z; /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */ - double r; /* Value if eType is SQLITE_FLOAT or SQLITE_INTEGER */ - } u; - u8 eType; /* SQLITE_NULL, SQLITE_INTEGER ... etc. */ - u8 nByte; /* Size in byte of text or blob. */ + void *p; /* Pointer to sampled record */ + int n; /* Size of record in bytes */ + tRowcnt *anEq; /* Est. number of rows where the key equals this sample */ + tRowcnt *anLt; /* Est. number of rows where key is less than this sample */ + tRowcnt *anDLt; /* Est. number of distinct keys less than this sample */ }; /* @@ -10139,8 +11386,10 @@ struct AggInfo { u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ int sortingIdx; /* Cursor number of the sorting index */ - ExprList *pGroupBy; /* The group by clause */ + int sortingIdxPTab; /* Cursor number of pseudo-table */ int nSortingColumn; /* Number of columns in the sorting index */ + int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ int iTable; /* Cursor number of the source table */ @@ -10150,7 +11399,6 @@ struct AggInfo { Expr *pExpr; /* The original expression */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ - int nColumnAlloc; /* Number of slots allocated for aCol[] */ int nAccumulator; /* Number of columns that show through to the output. ** Additional columns are used only as parameters to ** aggregate functions */ @@ -10161,7 +11409,6 @@ struct AggInfo { int iDistinct; /* Ephemeral table used to enforce DISTINCT */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ - int nFuncAlloc; /* Number of slots allocated for aFunc[] */ }; /* @@ -10246,7 +11493,7 @@ typedef int ynVar; struct Expr { u8 op; /* Operation performed by this node */ char affinity; /* The affinity of the column or 0 if not a column */ - u16 flags; /* Various flags. EP_* See below */ + u32 flags; /* Various flags. EP_* See below */ union { char *zToken; /* Token value. Zero terminated and dequoted */ int iValue; /* Non-negative integer value if EP_IntValue */ @@ -10260,79 +11507,77 @@ struct Expr { Expr *pLeft; /* Left subnode */ Expr *pRight; /* Right subnode */ union { - ExprList *pList; /* Function arguments or in " IN ( IN ( ** -** If the prNotFound parameter is 0, then the b-tree will be used to iterate -** through the set members, skipping any duplicates. In this case an -** epheremal table must be used unless the selected is guaranteed +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephemeral table instead of an +** existing table. +** +** The inFlags parameter must contain exactly one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a +** fast membership test. When the IN_INDEX_LOOP bit is set, the +** IN index will be used to loop over all values of the RHS of the +** IN operator. +** +** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate +** through the set members) then the b-tree must not contain duplicates. +** An epheremal table must be used unless the selected is guaranteed ** to be unique - either because it is an INTEGER PRIMARY KEY or it ** has a UNIQUE constraint or UNIQUE index. ** -** If the prNotFound parameter is not 0, then the b-tree will be used -** for fast set membership tests. In this case an epheremal table must +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an epheremal table must ** be used unless is an INTEGER PRIMARY KEY or an index can ** be found with as its left-most column. ** +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and +** if the RHS of the IN operator is a list (not a subquery) then this +** routine might decide that creating an ephemeral b-tree for membership +** testing is too expensive and return IN_INDEX_NOOP. In that case, the +** calling routine should implement the IN operator using a sequence +** of Eq or Ne comparison operations. +** ** When the b-tree is being used for membership tests, the calling function -** needs to know whether or not the structure contains an SQL NULL -** value in order to correctly evaluate expressions like "X IN (Y, Z)". -** If there is any chance that the (...) might contain a NULL value at +** might need to know whether or not the RHS side of the IN operator +** contains a NULL. If prRhsHasNull is not a NULL pointer and +** if there is any chance that the (...) might contain a NULL value at ** runtime, then a register is allocated and the register number written -** to *prNotFound. If there is no chance that the (...) contains a -** NULL value, then *prNotFound is left unchanged. +** to *prRhsHasNull. If there is no chance that the (...) contains a +** NULL value, then *prRhsHasNull is left unchanged. ** -** If a register is allocated and its location stored in *prNotFound, then -** its initial value is NULL. If the (...) does not remain constant -** for the duration of the query (i.e. the SELECT within the (...) -** is a correlated subquery) then the value of the allocated register is -** reset to NULL each time the subquery is rerun. This allows the -** caller to use vdbe code equivalent to the following: -** -** if( register==NULL ){ -** has_null = -** register = 1 -** } -** -** in order to avoid running the -** test more often than is necessary. +** If a register is allocated and its location stored in *prRhsHasNull, then +** the value in that register will be NULL if the b-tree contains one or more +** NULL values, and it will be some non-NULL value if the b-tree contains no +** NULL values. */ #ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ - int mustBeUnique = (prNotFound==0); /* True if RHS must be unique */ + int mustBeUnique; /* True if RHS must be unique */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ assert( pX->op==TK_IN ); + mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; /* Check to see if an existing table or index can be used to ** satisfy the query. This is preferable to generating a new @@ -72658,13 +82276,20 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ sqlite3 *db = pParse->db; /* Database connection */ - Expr *pExpr = p->pEList->a[0].pExpr; /* Expression */ - int iCol = pExpr->iColumn; /* Index of column */ - Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ - Table *pTab = p->pSrc->a[0].pTab; /* Table
. */ - int iDb; /* Database idx for pTab */ + Table *pTab; /* Table
. */ + Expr *pExpr; /* Expression */ + i16 iCol; /* Index of column */ + i16 iDb; /* Database idx for pTab */ + + assert( p ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + iCol = (i16)pExpr->iColumn; - /* Code an OP_VerifyCookie and OP_TableLock for
. */ + /* Code an OP_Transaction and OP_TableLock for
. */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); @@ -72675,11 +82300,8 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ */ assert(v); if( iCol<0 ){ - int iMem = ++pParse->nMem; - int iAddr; - - iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); - sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); + int iAddr = sqlite3CodeOnce(pParse); + VdbeCoverage(v); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); eType = IN_INDEX_ROWID; @@ -72697,51 +82319,60 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ ** comparison is the same as the affinity of the column. If ** it is not, it is not possible to use any index. */ - char aff = comparisonAffinity(pX); - int affinity_ok = (pTab->aCol[iCol].affinity==aff||aff==SQLITE_AFF_NONE); + int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ if( (pIdx->aiColumn[0]==iCol) && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nColumn==1 && pIdx->onError!=OE_None)) + && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) ){ - int iMem = ++pParse->nMem; - int iAddr; - char *pKey; - - pKey = (char *)sqlite3IndexKeyinfo(pParse, pIdx); - iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); - sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); - - sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, - pKey,P4_KEYINFO_HANDOFF); + int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - eType = IN_INDEX_INDEX; + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - sqlite3VdbeJumpHere(v, iAddr); - if( prNotFound && !pTab->aCol[iCol].notNull ){ - *prNotFound = ++pParse->nMem; + if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ + *prRhsHasNull = ++pParse->nMem; + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); } + sqlite3VdbeJumpHere(v, iAddr); } } } } + /* If no preexisting index is available for the IN clause + ** and IN_INDEX_NOOP is an allowed reply + ** and the RHS of the IN operator is a list, not a subquery + ** and the RHS is not contant or has two or fewer terms, + ** then it is not worth creating an ephemeral table to evaluate + ** the IN operator so return IN_INDEX_NOOP. + */ + if( eType==0 + && (inFlags & IN_INDEX_NOOP_OK) + && !ExprHasProperty(pX, EP_xIsSelect) + && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + ){ + eType = IN_INDEX_NOOP; + } + + if( eType==0 ){ - /* Could not found an existing table or index to use as the RHS b-tree. + /* Could not find an existing table or index to use as the RHS b-tree. ** We will have to generate an ephemeral table to do the job. */ - double savedNQueryLoop = pParse->nQueryLoop; + u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; eType = IN_INDEX_EPH; - if( prNotFound ){ - *prNotFound = rMayHaveNull = ++pParse->nMem; - }else{ - testcase( pParse->nQueryLoop>(double)1 ); - pParse->nQueryLoop = (double)1; - if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){ + if( inFlags & IN_INDEX_LOOP ){ + pParse->nQueryLoop = 0; + if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; } + }else if( prRhsHasNull ){ + *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); pParse->nQueryLoop = savedNQueryLoop; @@ -72772,15 +82403,9 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ ** ** If rMayHaveNull is non-zero, that means that the operation is an IN ** (not a SELECT or EXISTS) and that the RHS might contains NULLs. -** Furthermore, the IN is in a WHERE clause and that we really want -** to iterate over the RHS of the IN operator in order to quickly locate -** all corresponding LHS elements. All this routine does is initialize -** the register given by rMayHaveNull to NULL. Calling routines will take -** care of changing this register value to non-NULL if the RHS is NULL-free. -** -** If rMayHaveNull is zero, that means that the subquery is being used -** for membership testing only. There is no need to initialize any -** registers to indicate the presense or absence of NULLs on the RHS. +** All this routine does is initialize the register given by rMayHaveNull +** to NULL. Calling routines will take care of changing this register +** value to non-NULL if the RHS is NULL-free. ** ** For a SELECT or EXISTS operator, return the register that holds the ** result. For IN operators or if an error occurs, the return value is 0. @@ -72789,10 +82414,10 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ SQLITE_PRIVATE int sqlite3CodeSubselect( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ - int rMayHaveNull, /* Register that records whether NULLs exist in RHS */ + int rHasNullFlag, /* Register that records whether NULLs exist in RHS */ int isRowid /* If true, LHS of IN operator is a rowid */ ){ - int testAddr = 0; /* One-time test address */ + int jmpIfDynamic = -1; /* One-time test address */ int rReg = 0; /* Register storing resulting */ Vdbe *v = sqlite3GetVdbe(pParse); if( NEVER(v==0) ) return 0; @@ -72808,17 +82433,14 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** If all of the above are false, then we can run this code just once ** save the results, and reuse the same result on subsequent invocations. */ - if( !ExprHasAnyProperty(pExpr, EP_VarSelect) && !pParse->pTriggerTab ){ - int mem = ++pParse->nMem; - sqlite3VdbeAddOp1(v, OP_If, mem); - testAddr = sqlite3VdbeAddOp2(v, OP_Integer, 1, mem); - assert( testAddr>0 || pParse->db->mallocFailed ); + if( !ExprHasProperty(pExpr, EP_VarSelect) ){ + jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN if( pParse->explain==2 ){ char *zMsg = sqlite3MPrintf( - pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr?"":"CORRELATED ", + pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ", pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId ); sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); @@ -72828,13 +82450,9 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( switch( pExpr->op ){ case TK_IN: { char affinity; /* Affinity of the LHS of the IN */ - KeyInfo keyInfo; /* Keyinfo for the generated table */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ - - if( rMayHaveNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); - } + KeyInfo *pKeyInfo = 0; /* Key information */ affinity = sqlite3ExprAffinity(pLeft); @@ -72853,9 +82471,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( */ pExpr->iTable = pParse->nTab++; addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); - if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - memset(&keyInfo, 0, sizeof(keyInfo)); - keyInfo.nField = 1; + pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -72863,22 +82479,28 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** Generate code to write the results of the select into the temporary ** table allocated and opened above. */ + Select *pSelect = pExpr->x.pSelect; SelectDest dest; ExprList *pEList; assert( !isRowid ); sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.affinity = (u8)affinity; + dest.affSdst = (u8)affinity; assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pExpr->x.pSelect->iLimit = 0; - if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3KeyInfoUnref(pKeyInfo); return 0; } - pEList = pExpr->x.pSelect->pEList; - if( ALWAYS(pEList!=0 && pEList->nExpr>0) ){ - keyInfo.aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, - pEList->a[0].pExpr); - } + pEList = pSelect->pEList; + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, + pEList->a[0].pExpr); }else if( ALWAYS(pExpr->x.pList!=0) ){ /* Case 2: expr IN (exprlist) ** @@ -72895,12 +82517,15 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( !affinity ){ affinity = SQLITE_AFF_NONE; } - keyInfo.aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + if( pKeyInfo ){ + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + } /* Loop through each expression in . */ r1 = sqlite3GetTempReg(pParse); r2 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Null, 0, r2); + if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2); for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ Expr *pE2 = pItem->pExpr; int iValToIns; @@ -72910,9 +82535,9 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( testAddr && !sqlite3ExprIsConstant(pE2) ){ - sqlite3VdbeChangeToNoop(v, testAddr-1, 2); - testAddr = 0; + if( jmpIfDynamic>=0 && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, jmpIfDynamic); + jmpIfDynamic = -1; } /* Evaluate the expression and insert it into the temp table */ @@ -72923,6 +82548,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( isRowid ){ sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); @@ -72934,8 +82560,8 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempReg(pParse, r2); } - if( !isRowid ){ - sqlite3VdbeChangeP4(v, addr, (void *)&keyInfo, P4_KEYINFO); + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } break; } @@ -72961,11 +82587,12 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; - sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iParm); + dest.iSdst = dest.iSDParm; + sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; - sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iParm); + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); VdbeComment((v, "Init EXISTS result")); } sqlite3ExprDelete(pParse->db, pSel->pLimit); @@ -72975,16 +82602,20 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( sqlite3Select(pParse, pSel, &dest) ){ return 0; } - rReg = dest.iParm; - ExprSetIrreducible(pExpr); + rReg = dest.iSDParm; + ExprSetVVAProperty(pExpr, EP_NoReduce); break; } } - if( testAddr ){ - sqlite3VdbeJumpHere(v, testAddr-1); + if( rHasNullFlag ){ + sqlite3SetHasNullFlag(v, pExpr->iTable, rHasNullFlag); } - sqlite3ExprCachePop(pParse, 1); + + if( jmpIfDynamic>=0 ){ + sqlite3VdbeJumpHere(v, jmpIfDynamic); + } + sqlite3ExprCachePop(pParse); return rReg; } @@ -73003,7 +82634,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** if the LHS is NULL or if the LHS is not contained within the RHS and the ** RHS contains one or more NULL values. ** -** This routine generates code will jump to destIfFalse if the LHS is not +** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. @@ -73026,7 +82657,9 @@ static void sqlite3ExprCodeIN( v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); - eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull); + eType = sqlite3FindInIndex(pParse, pExpr, + IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, + destIfFalse==destIfNull ? 0 : &rRhsHasNull); /* Figure out the affinity to use to create a key from the results ** of the expression. affinityStr stores a static string suitable for @@ -73040,86 +82673,118 @@ static void sqlite3ExprCodeIN( r1 = sqlite3GetTempReg(pParse); sqlite3ExprCode(pParse, pExpr->pLeft, r1); - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. + /* If sqlite3FindInIndex() did not find or create an index that is + ** suitable for evaluating the IN operator, then evaluate using a + ** sequence of comparisons. */ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); - }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - sqlite3VdbeJumpHere(v, addr1); - } - - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree - */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - if( rRhsHasNull==0 || destIfFalse==destIfNull ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. - */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); - - }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. - */ - int j1, j2, j3; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the presence of NULLs in the RHS does not matter, so jump - ** over all of the code that follows. - */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - - /* Here we begin generating code that runs if the LHS is not - ** contained within the RHS. Generate additional code that - ** tests the RHS for NULLs. If the RHS contains a NULL then - ** jump to destIfNull. If there are no NULLs in the RHS then - ** jump to destIfFalse. - */ - j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull); - j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); - sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull); - sqlite3VdbeJumpHere(v, j3); - sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1); - sqlite3VdbeJumpHere(v, j2); - - /* Jump to the appropriate target depending on whether or not - ** the RHS contains a NULL - */ - sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); + if( eType==IN_INDEX_NOOP ){ + ExprList *pList = pExpr->x.pList; + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + int labelOk = sqlite3VdbeMakeLabel(v); + int r2, regToFree; + int regCkNull = 0; + int ii; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + if( destIfNull!=destIfFalse ){ + regCkNull = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); + } + for(ii=0; iinExpr; ii++){ + r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ + sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); + } + if( iinExpr-1 || destIfNull!=destIfFalse ){ + sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, iinExpr-1); + VdbeCoverageIf(v, ii==pList->nExpr-1); + sqlite3VdbeChangeP5(v, affinity); + }else{ + assert( destIfNull==destIfFalse ); + sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); + } + sqlite3ReleaseTempReg(pParse, regToFree); + } + if( regCkNull ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - - /* The OP_Found at the top of this branch jumps here when true, - ** causing the overall IN expression evaluation to fall through. + } + sqlite3VdbeResolveLabel(v, labelOk); + sqlite3ReleaseTempReg(pParse, regCkNull); + }else{ + + /* If the LHS is NULL, then the result is either false or NULL depending + ** on whether the RHS is empty or not, respectively. + */ + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); + } + } + + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + VdbeCoverage(v); + }else{ + /* In this case, the RHS is an index b-tree. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); + + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. + */ + assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); + if( rRhsHasNull==0 ){ + /* This branch runs if it is known at compile time that the RHS + ** cannot contain NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. + ** + ** Also run this branch if NULL is equivalent to FALSE + ** for this particular IN operator. + */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + VdbeCoverage(v); + }else{ + /* In this branch, the RHS of the IN might contain a NULL and + ** the presence of a NULL on the RHS makes a difference in the + ** outcome. + */ + int j1; + + /* First check to see if the LHS is contained in the RHS. If so, + ** then the answer is TRUE the presence of NULLs in the RHS does + ** not matter. If the LHS is not contained in the RHS, then the + ** answer is NULL if the RHS contains NULLs and the answer is + ** FALSE if the RHS is NULL-free. + */ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeJumpHere(v, j1); + } } } sqlite3ReleaseTempReg(pParse, r1); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -73176,7 +82841,7 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ i64 value; const char *z = pExpr->u.zToken; assert( z!=0 ); - c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } @@ -73186,7 +82851,14 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); #else - codeReal(v, z, negFlag, iMem); +#ifndef SQLITE_OMIT_HEX_INTEGER + if( sqlite3_strnicmp(z,"0x",2)==0 ){ + sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); + }else +#endif + { + codeReal(v, z, negFlag, iMem); + } #endif } } @@ -73222,7 +82894,7 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int ** for testing only - to verify that SQLite always gets the same answer ** with and without the column cache. */ - if( pParse->db->flags & SQLITE_ColumnCache ) return; + if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; /* First replace any existing entry. ** @@ -73231,15 +82903,6 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int */ #ifndef NDEBUG for(i=0, p=pParse->aColCache; iiReg && p->iTable==iTab && p->iColumn==iCol ){ - cacheEntryClear(pParse, p); - p->iLevel = pParse->iCacheLevel; - p->iReg = iReg; - p->lru = pParse->iCacheCnt++; - return; - } -#endif assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } #endif @@ -73302,19 +82965,28 @@ SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ */ SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ pParse->iCacheLevel++; +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("PUSH to %d\n", pParse->iCacheLevel); + } +#endif } /* ** Remove from the column cache any entries that were added since the -** the previous N Push operations. In other words, restore the cache -** to the state it was in N Pushes ago. +** the previous sqlite3ExprCachePush operation. In other words, restore +** the cache to the state it was in prior the most recent Push. */ -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse, int N){ +SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ int i; struct yColCache *p; - assert( N>0 ); - assert( pParse->iCacheLevel>=N ); - pParse->iCacheLevel -= N; + assert( pParse->iCacheLevel>=1 ); + pParse->iCacheLevel--; +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("POP to %d\n", pParse->iCacheLevel); + } +#endif for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ cacheEntryClear(pParse, p); @@ -73345,15 +83017,19 @@ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( Vdbe *v, /* The VDBE under construction */ Table *pTab, /* The table containing the value */ - int iTabCur, /* The cursor for this table */ + int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ int iCol, /* Index of the column to extract */ - int regOut /* Extract the valud into this register */ + int regOut /* Extract the value into this register */ ){ if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); }else{ int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; - sqlite3VdbeAddOp3(v, op, iTabCur, iCol, regOut); + int x = iCol; + if( !HasRowid(pTab) ){ + x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + } + sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); } if( iCol>=0 ){ sqlite3ColumnDefault(v, pTab, iCol, regOut); @@ -73374,7 +83050,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column */ ){ Vdbe *v = pParse->pVdbe; int i; @@ -73389,7 +83066,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( } assert( v!=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + if( p5 ){ + sqlite3VdbeChangeP5(v, p5); + }else{ + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + } return iReg; } @@ -73400,6 +83081,11 @@ SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ int i; struct yColCache *p; +#if SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("CLEAR\n"); + } +#endif for(i=0, p=pParse->aColCache; iiReg ){ cacheEntryClear(pParse, p); @@ -73421,28 +83107,9 @@ SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, in ** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. */ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ - int i; - struct yColCache *p; - if( NEVER(iFrom==iTo) ) return; + assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - for(i=0, p=pParse->aColCache; iiReg; - if( x>=iFrom && xiReg += iTo-iFrom; - } - } -} - -/* -** Generate code to copy content from registers iFrom...iFrom+nReg-1 -** over to iTo..iTo+nReg-1. -*/ -SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, int iFrom, int iTo, int nReg){ - int i; - if( NEVER(iFrom==iTo) ) return; - for(i=0; ipVdbe, OP_Copy, iFrom+i, iTo+i); - } + sqlite3ExprCacheRemove(pParse, iFrom, nReg); } #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) @@ -73464,6 +83131,16 @@ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ +/* +** Convert an expression node to a TK_REGISTER +*/ +static void exprToRegister(Expr *p, int iReg){ + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); +} + /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -73483,6 +83160,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) int regFree2 = 0; /* If non-zero free this temporary register */ int r1, r2, r3, r4; /* Various register numbers */ sqlite3 *db = pParse->db; /* The database connection */ + Expr tempX; /* Temporary expression node */ assert( target>0 && target<=pParse->nMem ); if( v==0 ){ @@ -73504,21 +83182,27 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) inReg = pCol->iMem; break; }else if( pAggInfo->useSortingIdx ){ - sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx, + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); break; } /* Otherwise, fall thru into the TK_COLUMN case */ } case TK_COLUMN: { - if( pExpr->iTable<0 ){ - /* This only happens when coding check constraints */ - assert( pParse->ckBase>0 ); - inReg = pExpr->iColumn + pParse->ckBase; - }else{ - inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target); + int iTab = pExpr->iTable; + if( iTab<0 ){ + if( pParse->ckBase>0 ){ + /* Generating CHECK constraints or inserting into partial index */ + inReg = pExpr->iColumn + pParse->ckBase; + break; + }else{ + /* Deleting from a partial index */ + iTab = pParse->iPartIdxTab; + } } + inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + pExpr->iColumn, iTab, target, + pExpr->op2); break; } case TK_INTEGER: { @@ -73580,26 +83264,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - int aff, to_op; inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - aff = sqlite3AffinityType(pExpr->u.zToken); - to_op = aff - SQLITE_AFF_TEXT + OP_ToText; - assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); - assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); - assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); - assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); - assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); - testcase( to_op==OP_ToText ); - testcase( to_op==OP_ToBlob ); - testcase( to_op==OP_ToNumeric ); - testcase( to_op==OP_ToInt ); - testcase( to_op==OP_ToReal ); if( inReg!=target ){ sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); inReg = target; } - sqlite3VdbeAddOp1(v, to_op, inReg); + sqlite3VdbeAddOp2(v, OP_Cast, target, + sqlite3AffinityType(pExpr->u.zToken, 0)); testcase( usedAsColumnCache(pParse, inReg, inReg) ); sqlite3ExprCacheAffinityChange(pParse, inReg, 1); break; @@ -73611,22 +83282,16 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_GE: case TK_NE: case TK_EQ: { - assert( TK_LT==OP_Lt ); - assert( TK_LE==OP_Le ); - assert( TK_GT==OP_Gt ); - assert( TK_GE==OP_Ge ); - assert( TK_EQ==OP_Eq ); - assert( TK_NE==OP_Ne ); - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, inReg, SQLITE_STOREP2); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -73640,6 +83305,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) op = (op==TK_IS) ? TK_EQ : TK_NE; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -73656,28 +83323,17 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_LSHIFT: case TK_RSHIFT: case TK_CONCAT: { - assert( TK_AND==OP_And ); - assert( TK_OR==OP_Or ); - assert( TK_PLUS==OP_Add ); - assert( TK_MINUS==OP_Subtract ); - assert( TK_REM==OP_Remainder ); - assert( TK_BITAND==OP_BitAnd ); - assert( TK_BITOR==OP_BitOr ); - assert( TK_SLASH==OP_Divide ); - assert( TK_LSHIFT==OP_ShiftLeft ); - assert( TK_RSHIFT==OP_ShiftRight ); - assert( TK_CONCAT==OP_Concat ); - testcase( op==TK_AND ); - testcase( op==TK_OR ); - testcase( op==TK_PLUS ); - testcase( op==TK_MINUS ); - testcase( op==TK_REM ); - testcase( op==TK_BITAND ); - testcase( op==TK_BITOR ); - testcase( op==TK_SLASH ); - testcase( op==TK_LSHIFT ); - testcase( op==TK_RSHIFT ); - testcase( op==TK_CONCAT ); + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); + assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); + assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); + assert( TK_BITAND==OP_BitAnd ); testcase( op==TK_BITAND ); + assert( TK_BITOR==OP_BitOr ); testcase( op==TK_BITOR ); + assert( TK_SLASH==OP_Divide ); testcase( op==TK_SLASH ); + assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); + assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); + assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); sqlite3VdbeAddOp3(v, op, r2, r1, target); @@ -73696,8 +83352,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) codeReal(v, pLeft->u.zToken, 1, target); #endif }else{ - regFree1 = r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Integer, 0, r1); + tempX.op = TK_INTEGER; + tempX.flags = EP_IntValue|EP_TokenOnly; + tempX.u.iValue = 0; + r1 = sqlite3ExprCodeTemp(pParse, &tempX, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); testcase( regFree2==0 ); @@ -73707,10 +83365,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } case TK_BITNOT: case TK_NOT: { - assert( TK_BITNOT==OP_BitNot ); - assert( TK_NOT==OP_Not ); - testcase( op==TK_BITNOT ); - testcase( op==TK_NOT ); + assert( TK_BITNOT==OP_BitNot ); testcase( op==TK_BITNOT ); + assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); inReg = target; @@ -73720,15 +83376,15 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_ISNULL: case TK_NOTNULL: { int addr; - assert( TK_ISNULL==OP_IsNull ); - assert( TK_NOTNULL==OP_NotNull ); - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); sqlite3VdbeAddOp2(v, OP_Integer, 1, target); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); addr = sqlite3VdbeAddOp1(v, op, r1); - sqlite3VdbeAddOp2(v, OP_AddImm, target, -1); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); sqlite3VdbeJumpHere(v, addr); break; } @@ -73742,22 +83398,19 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } break; } - case TK_CONST_FUNC: case TK_FUNCTION: { ExprList *pFarg; /* List of function arguments */ int nFarg; /* Number of function arguments */ FuncDef *pDef; /* The function definition object */ int nId; /* Length of the function name in bytes */ const char *zId; /* The function name */ - int constMask = 0; /* Mask of function arguments that are constant */ + u32 constMask = 0; /* Mask of function arguments that are constant */ int i; /* Loop counter */ u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - testcase( op==TK_CONST_FUNC ); - testcase( op==TK_FUNCTION ); - if( ExprHasAnyProperty(pExpr, EP_TokenOnly) ){ + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ pFarg = 0; }else{ pFarg = pExpr->x.pList; @@ -73767,36 +83420,80 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); - if( pDef==0 ){ + if( pDef==0 || pDef->xFunc==0 ){ sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); break; } /* Attempt a direct implementation of the built-in COALESCE() and - ** IFNULL() functions. This avoids unnecessary evalation of + ** IFNULL() functions. This avoids unnecessary evaluation of ** arguments past the first non-NULL argument. */ - if( pDef->flags & SQLITE_FUNC_COALESCE ){ + if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){ int endCoalesce = sqlite3VdbeMakeLabel(v); assert( nFarg>=2 ); sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); for(i=1; ia[i].pExpr, target); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); } sqlite3VdbeResolveLabel(v, endCoalesce); break; } + /* The UNLIKELY() function is a no-op. The result is the value + ** of the first argument. + */ + if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ + assert( nFarg>=1 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + break; + } + for(i=0; ia[i].pExpr) ){ + testcase( i==31 ); + constMask |= MASKBIT32(i); + } + if( (pDef->funcFlags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pFarg->a[i].pExpr); + } + } if( pFarg ){ - r1 = sqlite3GetTempRange(pParse, nFarg); + if( constMask ){ + r1 = pParse->nMem+1; + pParse->nMem += nFarg; + }else{ + r1 = sqlite3GetTempRange(pParse, nFarg); + } + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 exprOp; + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); + pFarg->a[0].pExpr->op2 = + pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + } + } + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); - sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ + sqlite3ExprCodeExprList(pParse, pFarg, r1, + SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ r1 = 0; } @@ -73819,22 +83516,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); } #endif - for(i=0; ia[i].pExpr) ){ - constMask |= (1<flags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ - pColl = sqlite3ExprCollSeq(pParse, pFarg->a[i].pExpr); - } - } - if( pDef->flags & SQLITE_FUNC_NEEDCOLL ){ + if( pDef->funcFlags & SQLITE_FUNC_NEEDCOLL ){ if( !pColl ) pColl = db->pDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nFarg); - if( nFarg ){ + if( nFarg && constMask==0 ){ sqlite3ReleaseTempRange(pParse, r1, nFarg); } break; @@ -73884,18 +83573,20 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) r3 = sqlite3GetTempReg(pParse); r4 = sqlite3GetTempReg(pParse); codeCompare(pParse, pLeft, pRight, OP_Ge, - r1, r2, r3, SQLITE_STOREP2); + r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); pLItem++; pRight = pLItem->pExpr; sqlite3ReleaseTempReg(pParse, regFree2); r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); testcase( regFree2==0 ); codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); sqlite3ReleaseTempReg(pParse, r3); sqlite3ReleaseTempReg(pParse, r4); break; } + case TK_COLLATE: case TK_UPLUS: { inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; @@ -73967,9 +83658,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** WHEN x=eN THEN rN ELSE y END ** ** X (if it exists) is in pExpr->pLeft. - ** Y is in pExpr->pRight. The Y is also optional. If there is no - ** ELSE clause and no other term matches, then the result of the - ** exprssion is NULL. + ** Y is in the last element of pExpr->x.pList if pExpr->x.pList->nExpr is + ** odd. The Y is also optional. If the number of elements in x.pList + ** is even, then Y is omitted and the "otherwise" result is NULL. ** Ei is in pExpr->pList->a[i*2] and Ri is pExpr->pList->a[i*2+1]. ** ** The result of the expression is the Ri for the first matching Ei, @@ -73984,27 +83675,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ExprList *pEList; /* List of WHEN terms */ struct ExprList_item *aListelem; /* Array of WHEN terms */ Expr opCompare; /* The X==Ei expression */ - Expr cacheX; /* Cached expression X */ Expr *pX; /* The X expression */ Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ VVA_ONLY( int iCacheLevel = pParse->iCacheLevel; ) assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); - assert((pExpr->x.pList->nExpr % 2) == 0); assert(pExpr->x.pList->nExpr > 0); pEList = pExpr->x.pList; aListelem = pEList->a; nExpr = pEList->nExpr; endLabel = sqlite3VdbeMakeLabel(v); if( (pX = pExpr->pLeft)!=0 ){ - cacheX = *pX; + tempX = *pX; testcase( pX->op==TK_COLUMN ); - testcase( pX->op==TK_REGISTER ); - cacheX.iTable = sqlite3ExprCodeTemp(pParse, pX, ®Free1); + exprToRegister(&tempX, sqlite3ExprCodeTemp(pParse, pX, ®Free1)); testcase( regFree1==0 ); - cacheX.op = TK_REGISTER; opCompare.op = TK_EQ; - opCompare.pLeft = &cacheX; + opCompare.pLeft = &tempX; pTest = &opCompare; /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: ** The value in regFree1 might get SCopy-ed into the file result. @@ -74012,7 +83699,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** purposes and possibly overwritten. */ regFree1 = 0; } - for(i=0; iop==TK_COLUMN ); sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); - testcase( aListelem[i+1].pExpr->op==TK_REGISTER ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } - if( pExpr->pRight ){ + if( (nExpr&1)!=0 ){ sqlite3ExprCachePush(pParse); - sqlite3ExprCode(pParse, pExpr->pRight, target); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); + sqlite3ExprCachePop(pParse); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } @@ -74061,8 +83747,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( pExpr->affinity==OE_Ignore ){ sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + VdbeCoverage(v); }else{ - sqlite3HaltConstraint(pParse, pExpr->affinity, pExpr->u.zToken, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, + pExpr->affinity, pExpr->u.zToken, 0, 0); } break; @@ -74074,6 +83762,28 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) return inReg; } +/* +** Factor out the code of the given expression to initialization time. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeAtInit( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The expression to code when the VDBE initializes */ + int regDest, /* Store the value in this register */ + u8 reusable /* True if this expression is reusable */ +){ + ExprList *p; + assert( ConstFactorOk(pParse) ); + p = pParse->pConstExpr; + pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p = sqlite3ExprListAppend(pParse, p, pExpr); + if( p ){ + struct ExprList_item *pItem = &p->a[p->nExpr-1]; + pItem->u.iConstExprReg = regDest; + pItem->reusable = reusable; + } + pParse->pConstExpr = p; +} + /* ** Generate code to evaluate an expression and store the results ** into a register. Return the register number where the results @@ -74082,15 +83792,40 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** If the register is a temporary register that can be deallocated, ** then write its number into *pReg. If the result register is not ** a temporary, then set *pReg to zero. +** +** If pExpr is a constant, then this routine might generate this +** code to fill the register in the initialization section of the +** VDBE program, in order to factor it out of the evaluation loop. */ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ - int r1 = sqlite3GetTempReg(pParse); - int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); - if( r2==r1 ){ - *pReg = r1; + int r2; + pExpr = sqlite3ExprSkipCollate(pExpr); + if( ConstFactorOk(pParse) + && pExpr->op!=TK_REGISTER + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ + ExprList *p = pParse->pConstExpr; + int i; + *pReg = 0; + if( p ){ + struct ExprList_item *pItem; + for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ + if( pItem->reusable && sqlite3ExprCompare(pItem->pExpr,pExpr,-1)==0 ){ + return pItem->u.iConstExprReg; + } + } + } + r2 = ++pParse->nMem; + sqlite3ExprCodeAtInit(pParse, pExpr, r2, 1); }else{ - sqlite3ReleaseTempReg(pParse, r1); - *pReg = 0; + int r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( r2==r1 ){ + *pReg = r1; + }else{ + sqlite3ReleaseTempReg(pParse, r1); + *pReg = 0; + } } return r2; } @@ -74100,7 +83835,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ ** results in register target. The results are guaranteed to appear ** in register target. */ -SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ +SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ int inReg; assert( target>0 && target<=pParse->nMem ); @@ -74113,11 +83848,24 @@ SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); } } - return target; } /* -** Generate code that evalutes the given expression and puts the result +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. If the expression is constant, then this routine +** might choose to code the expression at initialization time. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ + if( pParse->okConstFactor && sqlite3ExprIsConstant(pExpr) ){ + sqlite3ExprCodeAtInit(pParse, pExpr, target, 0); + }else{ + sqlite3ExprCode(pParse, pExpr, target); + } +} + +/* +** Generate code that evaluates the given expression and puts the result ** in register target. ** ** Also make a copy of the expression results into another "cache" register @@ -74128,183 +83876,325 @@ SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ ** times. They are evaluated once and the results of the expression ** are reused. */ -SQLITE_PRIVATE int sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ +SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ Vdbe *v = pParse->pVdbe; - int inReg; - inReg = sqlite3ExprCode(pParse, pExpr, target); + int iMem; + assert( target>0 ); - /* This routine is called for terms to INSERT or UPDATE. And the only - ** other place where expressions can be converted into TK_REGISTER is - ** in WHERE clause processing. So as currently implemented, there is - ** no way for a TK_REGISTER to exist here. But it seems prudent to - ** keep the ALWAYS() in case the conditions above change with future - ** modifications or enhancements. */ - if( ALWAYS(pExpr->op!=TK_REGISTER) ){ - int iMem; - iMem = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Copy, inReg, iMem); - pExpr->iTable = iMem; - pExpr->op2 = pExpr->op; - pExpr->op = TK_REGISTER; - } - return inReg; + assert( pExpr->op!=TK_REGISTER ); + sqlite3ExprCode(pParse, pExpr, target); + iMem = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, target, iMem); + exprToRegister(pExpr, iMem); } +#ifdef SQLITE_DEBUG /* -** Return TRUE if pExpr is an constant expression that is appropriate -** for factoring out of a loop. Appropriate expressions are: -** -** * Any expression that evaluates to two or more opcodes. -** -** * Any OP_Integer, OP_Real, OP_String, OP_Blob, OP_Null, -** or OP_Variable that does not need to be placed in a -** specific register. -** -** There is no point in factoring out single-instruction constant -** expressions that need to be placed in a particular register. -** We could factor them out, but then we would end up adding an -** OP_SCopy instruction to move the value into the correct register -** later. We might as well just use the original instruction and -** avoid the OP_SCopy. +** Generate a human-readable explanation of an expression tree. */ -static int isAppropriateForFactoring(Expr *p){ - if( !sqlite3ExprIsConstantNotJoin(p) ){ - return 0; /* Only constant expressions are appropriate for factoring */ +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( pExpr==0 ){ + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; } - if( (p->flags & EP_FixedDest)==0 ){ - return 1; /* Any constant without a fixed destination is appropriate */ - } - while( p->op==TK_UPLUS ) p = p->pLeft; - switch( p->op ){ -#ifndef SQLITE_OMIT_BLOB_LITERAL - case TK_BLOB: -#endif - case TK_VARIABLE: - case TK_INTEGER: - case TK_FLOAT: - case TK_NULL: - case TK_STRING: { - testcase( p->op==TK_BLOB ); - testcase( p->op==TK_VARIABLE ); - testcase( p->op==TK_INTEGER ); - testcase( p->op==TK_FLOAT ); - testcase( p->op==TK_NULL ); - testcase( p->op==TK_STRING ); - /* Single-instruction constants with a fixed destination are - ** better done in-line. If we factor them, they will just end - ** up generating an OP_SCopy to move the value to the destination - ** register. */ - return 0; - } - case TK_UMINUS: { - if( p->pLeft->op==TK_FLOAT || p->pLeft->op==TK_INTEGER ){ - return 0; - } - break; - } - default: { - break; - } - } - return 1; -} - -/* -** If pExpr is a constant expression that is appropriate for -** factoring out of a loop, then evaluate the expression -** into a register and convert the expression into a TK_REGISTER -** expression. -*/ -static int evalConstExpr(Walker *pWalker, Expr *pExpr){ - Parse *pParse = pWalker->pParse; switch( pExpr->op ){ - case TK_IN: - case TK_REGISTER: { - return WRC_Prune; + case TK_AGG_COLUMN: { + sqlite3TreeViewLine(pView, "AGG{%d:%d}", + pExpr->iTable, pExpr->iColumn); + break; } - case TK_FUNCTION: - case TK_AGG_FUNCTION: - case TK_CONST_FUNC: { - /* The arguments to a function have a fixed destination. - ** Mark them this way to avoid generated unneeded OP_SCopy - ** instructions. - */ - ExprList *pList = pExpr->x.pList; - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - if( pList ){ - int i = pList->nExpr; - struct ExprList_item *pItem = pList->a; - for(; i>0; i--, pItem++){ - if( ALWAYS(pItem->pExpr) ) pItem->pExpr->flags |= EP_FixedDest; - } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + sqlite3TreeViewLine(pView, "COLUMN(%d)", pExpr->iColumn); + }else{ + sqlite3TreeViewLine(pView, "{%d:%d}", + pExpr->iTable, pExpr->iColumn); } break; } - } - if( isAppropriateForFactoring(pExpr) ){ - int r1 = ++pParse->nMem; - int r2; - r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); - if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1); - pExpr->op2 = pExpr->op; - pExpr->op = TK_REGISTER; - pExpr->iTable = r2; - return WRC_Prune; - } - return WRC_Continue; -} + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); + }else{ + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3TreeViewLine(pView,"NULL"); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_AS: { + sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID %Q", pExpr->u.zToken); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_COLLATE: { + sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", + pExpr->op2, pExpr->u.zToken); + }else{ + sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + } + if( pFarg ){ + sqlite3TreeViewExprList(pView, pFarg, 0, 0); + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3TreeViewLine(pView, "EXISTS-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_SELECT: { + sqlite3TreeViewLine(pView, "SELECT-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_IN: { + sqlite3TreeViewLine(pView, "IN"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + }else{ + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + } + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3TreeViewLine(pView, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affinity ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + break; + } +#endif + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } + } + if( zBinOp ){ + sqlite3TreeViewLine(pView, "%s", zBinOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + }else if( zUniOp ){ + sqlite3TreeViewLine(pView, "%s", zUniOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG /* -** Preevaluate constant subexpressions within pExpr and store the -** results in registers. Modify pExpr so that the constant subexpresions -** are TK_REGISTER opcodes that refer to the precomputed values. -** -** This routine is a no-op if the jump to the cookie-check code has -** already occur. Since the cookie-check jump is generated prior to -** any other serious processing, this check ensures that there is no -** way to accidently bypass the constant initializations. -** -** This routine is also a no-op if the SQLITE_FactorOutConst optimization -** is disabled via the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS) -** interface. This allows test logic to verify that the same answer is -** obtained for queries regardless of whether or not constants are -** precomputed into registers or if they are inserted in-line. +** Generate a human-readable explanation of an expression list. */ -SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ - Walker w; - if( pParse->cookieGoto ) return; - if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; - w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; - w.pParse = pParse; - sqlite3WalkExpr(&w, pExpr); +SQLITE_PRIVATE void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ + int i; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inExpr; i++){ + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); +#if 0 + if( pList->a[i].zName ){ + sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); + } + if( pList->a[i].bSpanIsTab ){ + sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); + } +#endif + } + } + sqlite3TreeViewPop(pView); } - +#endif /* SQLITE_DEBUG */ /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. ** ** Return the number of elements evaluated. +** +** The SQLITE_ECEL_DUP flag prevents the arguments from being +** filled using OP_SCopy. OP_Copy must be used instead. +** +** The SQLITE_ECEL_FACTOR argument allows constant arguments to be +** factored out into initialization code. */ SQLITE_PRIVATE int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* The expression list to be coded */ int target, /* Where to write results */ - int doHardCopy /* Make a hard copy of every element */ + u8 flags /* SQLITE_ECEL_* flags */ ){ struct ExprList_item *pItem; int i, n; + u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; assert( pList!=0 ); assert( target>0 ); assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ n = pList->nExpr; + if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); - if( inReg!=target+i ){ - sqlite3VdbeAddOp2(pParse->pVdbe, doHardCopy ? OP_Copy : OP_SCopy, - inReg, target+i); + if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); + }else{ + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + VdbeOp *pOp; + Vdbe *v = pParse->pVdbe; + if( copyOp==OP_Copy + && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && pOp->p1+pOp->p3+1==inReg + && pOp->p2+pOp->p3+1==target+i + ){ + pOp->p3++; + }else{ + sqlite3VdbeAddOp2(v, copyOp, inReg, target+i); + } + } } } return n; @@ -74320,7 +84210,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( ** x>=y AND x<=z ** ** Code it as such, taking care to do the common subexpression -** elementation of x. +** elimination of x. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ @@ -74346,8 +84236,7 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = &exprX; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); - exprX.op = TK_REGISTER; + exprToRegister(&exprX, sqlite3ExprCodeTemp(pParse, &exprX, ®Free1)); if( jumpIfTrue ){ sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); }else{ @@ -74388,24 +84277,26 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ op = pExpr->op; switch( op ){ case TK_AND: { int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_OR: { testcase( jumpIfNull==0 ); sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse); break; } case TK_NOT: { @@ -74419,23 +84310,17 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { - assert( TK_LT==OP_Lt ); - assert( TK_LE==OP_Le ); - assert( TK_GT==OP_Gt ); - assert( TK_GE==OP_Ge ); - assert( TK_EQ==OP_Eq ); - assert( TK_NE==OP_Ne ); - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -74449,18 +84334,20 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int op = (op==TK_IS) ? TK_EQ : TK_NE; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; } case TK_ISNULL: case TK_NOTNULL: { - assert( TK_ISNULL==OP_IsNull ); - assert( TK_NOTNULL==OP_NotNull ); - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); sqlite3VdbeAddOp2(v, op, r1, dest); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); testcase( regFree1==0 ); break; } @@ -74480,10 +84367,17 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int } #endif default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); + if( exprAlwaysTrue(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysFalse(pExpr) ){ + /* No-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } break; } } @@ -74508,7 +84402,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; /* The value of pExpr->op and op are related as follows: @@ -74546,17 +84440,19 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_AND: { testcase( jumpIfNull==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse); break; } case TK_OR: { int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_NOT: { @@ -74570,17 +84466,17 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -74594,16 +84490,18 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; } case TK_ISNULL: case TK_NOTNULL: { - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); + testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); testcase( regFree1==0 ); break; } @@ -74625,10 +84523,17 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int } #endif default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); + if( exprAlwaysFalse(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysTrue(pExpr) ){ + /* no-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } break; } } @@ -74642,6 +84547,12 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int ** by a COLLATE operator at the top level. Return 2 if there are differences ** other than the top-level COLLATE operator. ** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** +** The pA side might be using TK_REGISTER. If that is the case and pB is +** not using TK_REGISTER but is otherwise equivalent, then still return 0. +** ** Sometimes this routine will return 2 even if the two expressions ** really are equivalent. If we cannot prove that the expressions are ** identical, we return 2 just to be safe. So if this routine @@ -74652,33 +84563,44 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int ** just might result in some slightly slower code. But returning ** an incorrect 0 or 1 could lead to a malfunction. */ -SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ - if( pA==0||pB==0 ){ +SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ + u32 combinedFlags; + if( pA==0 || pB==0 ){ return pB==pA ? 0 : 2; } - assert( !ExprHasAnyProperty(pA, EP_TokenOnly|EP_Reduced) ); - assert( !ExprHasAnyProperty(pB, EP_TokenOnly|EP_Reduced) ); - if( ExprHasProperty(pA, EP_xIsSelect) || ExprHasProperty(pB, EP_xIsSelect) ){ + combinedFlags = pA->flags | pB->flags; + if( combinedFlags & EP_IntValue ){ + if( (pA->flags&pB->flags&EP_IntValue)!=0 && pA->u.iValue==pB->u.iValue ){ + return 0; + } return 2; } - if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; - if( pA->op!=pB->op ) return 2; - if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2; - if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2; - if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList) ) return 2; - if( pA->iTable!=pB->iTable || pA->iColumn!=pB->iColumn ) return 2; - if( ExprHasProperty(pA, EP_IntValue) ){ - if( !ExprHasProperty(pB, EP_IntValue) || pA->u.iValue!=pB->u.iValue ){ - return 2; + if( pA->op!=pB->op ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pA->pLeft, pB, iTab)<2 ){ + return 1; } - }else if( pA->op!=TK_COLUMN && pA->u.zToken ){ - if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; - if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ){ - return 2; + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pA, pB->pLeft, iTab)<2 ){ + return 1; + } + return 2; + } + if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ + if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + return pA->op==TK_COLLATE ? 1 : 2; + } + } + if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; + if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){ + if( combinedFlags & EP_xIsSelect ) return 2; + if( sqlite3ExprCompare(pA->pLeft, pB->pLeft, iTab) ) return 2; + if( sqlite3ExprCompare(pA->pRight, pB->pRight, iTab) ) return 2; + if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; + if( ALWAYS((combinedFlags & EP_Reduced)==0) ){ + if( pA->iColumn!=pB->iColumn ) return 2; + if( pA->iTable!=pB->iTable + && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; } } - if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1; - if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2; return 0; } @@ -74686,6 +84608,9 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ ** Compare two ExprList objects. Return 0 if they are identical and ** non-zero if they differ in any way. ** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** ** This routine might return non-zero for equivalent ExprLists. The ** only consequence will be disabled optimizations. But this routine ** must never return 0 if the two ExprList objects are different, or @@ -74694,7 +84619,7 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ ** Two NULL pointers are considered to be the same. But a NULL pointer ** always differs from a non-NULL pointer. */ -SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB){ +SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ int i; if( pA==0 && pB==0 ) return 0; if( pA==0 || pB==0 ) return 1; @@ -74703,11 +84628,107 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB){ Expr *pExprA = pA->a[i].pExpr; Expr *pExprB = pB->a[i].pExpr; if( pA->a[i].sortOrder!=pB->a[i].sortOrder ) return 1; - if( sqlite3ExprCompare(pExprA, pExprB) ) return 1; + if( sqlite3ExprCompare(pExprA, pExprB, iTab) ) return 1; } return 0; } +/* +** Return true if we can prove the pE2 will always be true if pE1 is +** true. Return false if we cannot complete the proof or if pE2 might +** be false. Examples: +** +** pE1: x==5 pE2: x==5 Result: true +** pE1: x>0 pE2: x==5 Result: false +** pE1: x=21 pE2: x=21 OR y=43 Result: true +** pE1: x!=123 pE2: x IS NOT NULL Result: true +** pE1: x!=?1 pE2: x IS NOT NULL Result: true +** pE1: x IS NULL pE2: x IS NOT NULL Result: false +** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** +** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has +** Expr.iTable<0 then assume a table number given by iTab. +** +** When in doubt, return false. Returning true might give a performance +** improvement. Returning false might cause a performance reduction, but +** it will always give the correct answer and is hence always safe. +*/ +SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ + if( sqlite3ExprCompare(pE1, pE2, iTab)==0 ){ + return 1; + } + if( pE2->op==TK_OR + && (sqlite3ExprImpliesExpr(pE1, pE2->pLeft, iTab) + || sqlite3ExprImpliesExpr(pE1, pE2->pRight, iTab) ) + ){ + return 1; + } + if( pE2->op==TK_NOTNULL + && sqlite3ExprCompare(pE1->pLeft, pE2->pLeft, iTab)==0 + && (pE1->op!=TK_ISNULL && pE1->op!=TK_IS) + ){ + return 1; + } + return 0; +} + +/* +** An instance of the following structure is used by the tree walker +** to count references to table columns in the arguments of an +** aggregate function, in order to implement the +** sqlite3FunctionThisSrc() routine. +*/ +struct SrcCount { + SrcList *pSrc; /* One particular FROM clause in a nested query */ + int nThis; /* Number of references to columns in pSrcList */ + int nOther; /* Number of references to columns in other FROM clauses */ +}; + +/* +** Count the number of references to columns. +*/ +static int exprSrcCount(Walker *pWalker, Expr *pExpr){ + /* The NEVER() on the second term is because sqlite3FunctionUsesThisSrc() + ** is always called before sqlite3ExprAnalyzeAggregates() and so the + ** TK_COLUMNs have not yet been converted into TK_AGG_COLUMN. If + ** sqlite3FunctionUsesThisSrc() is used differently in the future, the + ** NEVER() will need to be removed. */ + if( pExpr->op==TK_COLUMN || NEVER(pExpr->op==TK_AGG_COLUMN) ){ + int i; + struct SrcCount *p = pWalker->u.pSrcCount; + SrcList *pSrc = p->pSrc; + for(i=0; inSrc; i++){ + if( pExpr->iTable==pSrc->a[i].iCursor ) break; + } + if( inSrc ){ + p->nThis++; + }else{ + p->nOther++; + } + } + return WRC_Continue; +} + +/* +** Determine if any of the arguments to the pExpr Function reference +** pSrcList. Return true if they do. Also return true if the function +** has no arguments or has only constant arguments. Return false if pExpr +** references columns but not columns of tables found in pSrcList. +*/ +SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ + Walker w; + struct SrcCount cnt; + assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); + w.xExprCallback = exprSrcCount; + w.u.pSrcCount = &cnt; + cnt.pSrc = pSrcList; + cnt.nThis = 0; + cnt.nOther = 0; + sqlite3WalkExprList(&w, pExpr->x.pList); + return cnt.nThis>0 || cnt.nOther==0; +} + /* ** Add a new element to the pAggInfo->aCol[] array. Return the index of ** the new element. Return a negative number if malloc fails. @@ -74718,9 +84739,7 @@ static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ db, pInfo->aCol, sizeof(pInfo->aCol[0]), - 3, &pInfo->nColumn, - &pInfo->nColumnAlloc, &i ); return i; @@ -74736,9 +84755,7 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ db, pInfo->aFunc, sizeof(pInfo->aFunc[0]), - 3, &pInfo->nFunc, - &pInfo->nFuncAlloc, &i ); return i; @@ -74767,7 +84784,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ struct SrcList_item *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ struct AggInfo_col *pCol; - assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ /* If we reach this point, it means that pExpr refers to a table ** that is in the FROM clause of the aggregate query. @@ -74816,7 +84833,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ** Convert the pExpr to be a TK_AGG_COLUMN referring to that ** pAggInfo->aCol[] entry. */ - ExprSetIrreducible(pExpr); + ExprSetVVAProperty(pExpr, EP_NoReduce); pExpr->pAggInfo = pAggInfo; pExpr->op = TK_AGG_COLUMN; pExpr->iAgg = (i16)k; @@ -74827,15 +84844,15 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } case TK_AGG_FUNCTION: { - /* The pNC->nDepth==0 test causes aggregate functions in subqueries - ** to be ignored */ - if( pNC->nDepth==0 ){ + if( (pNC->ncFlags & NC_InAggFunc)==0 + && pWalker->walkerDepth==pExpr->op2 + ){ /* Check to see if pExpr is a duplicate of another aggregate ** function that is already in the pAggInfo structure */ struct AggInfo_func *pItem = pAggInfo->aFunc; for(i=0; inFunc; i++, pItem++){ - if( sqlite3ExprCompare(pItem->pExpr, pExpr)==0 ){ + if( sqlite3ExprCompare(pItem->pExpr, pExpr, -1)==0 ){ break; } } @@ -74862,38 +84879,36 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } /* Make pExpr point to the appropriate pAggInfo->aFunc[] entry */ - assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); - ExprSetIrreducible(pExpr); + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pExpr, EP_NoReduce); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; return WRC_Prune; + }else{ + return WRC_Continue; } } } return WRC_Continue; } static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ - NameContext *pNC = pWalker->u.pNC; - if( pNC->nDepth==0 ){ - pNC->nDepth++; - sqlite3WalkSelect(pWalker, pSelect); - pNC->nDepth--; - return WRC_Prune; - }else{ - return WRC_Continue; - } + UNUSED_PARAMETER(pWalker); + UNUSED_PARAMETER(pSelect); + return WRC_Continue; } /* -** Analyze the given expression looking for aggregate functions and -** for variables that need to be added to the pParse->aAgg[] array. -** Make additional entries to the pParse->aAgg[] array as necessary. +** Analyze the pExpr expression looking for aggregate functions and +** for variables that need to be added to AggInfo object that pNC->pAggInfo +** points to. Additional entries are made on the AggInfo object as +** necessary. ** ** This routine should only be called after the expression has been ** analyzed by sqlite3ResolveExprNames(). */ SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ Walker w; + memset(&w, 0, sizeof(w)); w.xExprCallback = analyzeAggregate; w.xSelectCallback = analyzeAggregatesInSelect; w.u.pNC = pNC; @@ -74932,7 +84947,7 @@ SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ ** purpose. ** ** If a register is currently being used by the column cache, then -** the dallocation is deferred until the column cache line that uses +** the deallocation is deferred until the column cache line that uses ** the register becomes stale. */ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ @@ -74974,6 +84989,14 @@ SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ } } +/* +** Mark all temporary registers as being unavailable for reuse. +*/ +SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ + pParse->nTempReg = 0; + pParse->nRangeReg = 0; +} + /************** End of expr.c ************************************************/ /************** Begin file alter.c *******************************************/ /* @@ -75054,8 +85077,8 @@ static void renameTableFunc( assert( len>0 ); } while( token!=TK_LP && token!=TK_USING ); - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, - zTableName, tname.z+tname.n); + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), + zSql, zTableName, tname.z+tname.n); sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } } @@ -75093,6 +85116,7 @@ static void renameParentFunc( int token; /* Type of token */ UNUSED_PARAMETER(NotUsed); + if( zInput==0 || zOld==0 ) return; for(z=zInput; *z; z=z+n){ n = sqlite3GetToken(z, &token); if( token==TK_REFERENCES ){ @@ -75107,7 +85131,7 @@ static void renameParentFunc( sqlite3Dequote(zParent); if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", - (zOutput?zOutput:""), z-zInput, zInput, (const char *)zNew + (zOutput?zOutput:""), (int)(z-zInput), zInput, (const char *)zNew ); sqlite3DbFree(db, zOutput); zOutput = zOut; @@ -75150,8 +85174,8 @@ static void renameTriggerFunc( UNUSED_PARAMETER(NotUsed); /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediatedly - ** preceded by either TK_ON or TK_DOT and immediatedly followed by one + ** statement is that the table name is the first token that is immediately + ** preceded by either TK_ON or TK_DOT and immediately followed by one ** of TK_WHEN, TK_BEGIN or TK_FOR. */ if( zSql ){ @@ -75193,8 +85217,8 @@ static void renameTriggerFunc( /* Variable tname now contains the token that is the old table-name ** in the CREATE TRIGGER statement. */ - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, - zTableName, tname.z+tname.n); + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), + zSql, zTableName, tname.z+tname.n); sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } } @@ -75391,7 +85415,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( assert( pSrc->nSrc==1 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); zDb = db->aDb[iDb].zName; @@ -75446,7 +85470,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( } #endif - /* Begin a transaction and code the VerifyCookie for database iDb. + /* Begin a transaction for database iDb. ** Then modify the schema cookie (since the ALTER TABLE modifies the ** schema). Open a statement transaction if the table is a virtual ** table. @@ -75507,7 +85531,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " "'sqlite_autoindex_' || %Q || substr(name,%d+18) " "ELSE name END " - "WHERE tbl_name=%Q AND " + "WHERE tbl_name=%Q COLLATE nocase AND " "(type='table' OR type='index' OR type='trigger');", zDb, SCHEMA_TABLE(iDb), zName, zName, zName, #ifndef SQLITE_OMIT_TRIGGER @@ -75582,6 +85606,7 @@ SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minForm sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); sqlite3VdbeJumpHere(v, j1); sqlite3ReleaseTempReg(pParse, r1); @@ -75641,7 +85666,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** If there is a NOT NULL constraint, then the default value for the ** column must not be NULL. */ - if( pCol->isPrimKey ){ + if( pCol->colFlags & COLFLAG_PRIMKEY ){ sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); return; } @@ -75664,7 +85689,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** can handle (i.e. not CURRENT_TIME etc.) */ if( pDflt ){ - sqlite3_value *pVal; + sqlite3_value *pVal = 0; if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ db->mallocFailed = 1; return; @@ -75734,7 +85759,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); if( db->mallocFailed ) goto exit_begin_add_column; - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -75805,7 +85830,7 @@ exit_begin_add_column: /************** End of alter.c ***********************************************/ /************** Begin file analyze.c *****************************************/ /* -** 2005 July 8 +** 2005-07-08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -75816,24 +85841,163 @@ exit_begin_add_column: ** ************************************************************************* ** This file contains code associated with the ANALYZE command. +** +** The ANALYZE command gather statistics about the content of tables +** and indices. These statistics are made available to the query planner +** to help it make better decisions about how to perform queries. +** +** The following system tables are or have been supported: +** +** CREATE TABLE sqlite_stat1(tbl, idx, stat); +** CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample); +** CREATE TABLE sqlite_stat3(tbl, idx, nEq, nLt, nDLt, sample); +** CREATE TABLE sqlite_stat4(tbl, idx, nEq, nLt, nDLt, sample); +** +** Additional tables might be added in future releases of SQLite. +** The sqlite_stat2 table is not created or used unless the SQLite version +** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled +** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. +** The sqlite_stat2 table is superseded by sqlite_stat3, which is only +** created and used by SQLite versions 3.7.9 and later and with +** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 +** is a superset of sqlite_stat2. The sqlite_stat4 is an enhanced +** version of sqlite_stat3 and is only available when compiled with +** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.1 and later. It is +** not possible to enable both STAT3 and STAT4 at the same time. If they +** are both enabled, then STAT4 takes precedence. +** +** For most applications, sqlite_stat1 provides all the statistics required +** for the query planner to make good choices. +** +** Format of sqlite_stat1: +** +** There is normally one row per index, with the index identified by the +** name in the idx column. The tbl column is the name of the table to +** which the index belongs. In each such row, the stat column will be +** a string consisting of a list of integers. The first integer in this +** list is the number of rows in the index. (This is the same as the +** number of rows in the table, except for partial indices.) The second +** integer is the average number of rows in the index that have the same +** value in the first column of the index. The third integer is the average +** number of rows in the index that have the same value for the first two +** columns. The N-th integer (for N>1) is the average number of rows in +** the index which have the same value for the first N-1 columns. For +** a K-column index, there will be K+1 integers in the stat column. If +** the index is unique, then the last integer will be 1. +** +** The list of integers in the stat column can optionally be followed +** by the keyword "unordered". The "unordered" keyword, if it is present, +** must be separated from the last integer by a single space. If the +** "unordered" keyword is present, then the query planner assumes that +** the index is unordered and will not use the index for a range query. +** +** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat +** column contains a single integer which is the (estimated) number of +** rows in the table identified by sqlite_stat1.tbl. +** +** Format of sqlite_stat2: +** +** The sqlite_stat2 is only created and is only used if SQLite is compiled +** with SQLITE_ENABLE_STAT2 and if the SQLite version number is between +** 3.6.18 and 3.7.8. The "stat2" table contains additional information +** about the distribution of keys within an index. The index is identified by +** the "idx" column and the "tbl" column is the name of the table to which +** the index belongs. There are usually 10 rows in the sqlite_stat2 +** table for each index. +** +** The sqlite_stat2 entries for an index that have sampleno between 0 and 9 +** inclusive are samples of the left-most key value in the index taken at +** evenly spaced points along the index. Let the number of samples be S +** (10 in the standard build) and let C be the number of rows in the index. +** Then the sampled rows are given by: +** +** rownumber = (i*C*2 + C)/(S*2) +** +** For i between 0 and S-1. Conceptually, the index space is divided into +** S uniform buckets and the samples are the middle row from each bucket. +** +** The format for sqlite_stat2 is recorded here for legacy reference. This +** version of SQLite does not support sqlite_stat2. It neither reads nor +** writes the sqlite_stat2 table. This version of SQLite only supports +** sqlite_stat3. +** +** Format for sqlite_stat3: +** +** The sqlite_stat3 format is a subset of sqlite_stat4. Hence, the +** sqlite_stat4 format will be described first. Further information +** about sqlite_stat3 follows the sqlite_stat4 description. +** +** Format for sqlite_stat4: +** +** As with sqlite_stat2, the sqlite_stat4 table contains histogram data +** to aid the query planner in choosing good indices based on the values +** that indexed columns are compared against in the WHERE clauses of +** queries. +** +** The sqlite_stat4 table contains multiple entries for each index. +** The idx column names the index and the tbl column is the table of the +** index. If the idx and tbl columns are the same, then the sample is +** of the INTEGER PRIMARY KEY. The sample column is a blob which is the +** binary encoding of a key from the index. The nEq column is a +** list of integers. The first integer is the approximate number +** of entries in the index whose left-most column exactly matches +** the left-most column of the sample. The second integer in nEq +** is the approximate number of entries in the index where the +** first two columns match the first two columns of the sample. +** And so forth. nLt is another list of integers that show the approximate +** number of entries that are strictly less than the sample. The first +** integer in nLt contains the number of entries in the index where the +** left-most column is less than the left-most column of the sample. +** The K-th integer in the nLt entry is the number of index entries +** where the first K columns are less than the first K columns of the +** sample. The nDLt column is like nLt except that it contains the +** number of distinct entries in the index that are less than the +** sample. +** +** There can be an arbitrary number of sqlite_stat4 entries per index. +** The ANALYZE command will typically generate sqlite_stat4 tables +** that contain between 10 and 40 samples which are distributed across +** the key space, though not uniformly, and which include samples with +** large nEq values. +** +** Format for sqlite_stat3 redux: +** +** The sqlite_stat3 table is like sqlite_stat4 except that it only +** looks at the left-most column of the index. The sqlite_stat3.sample +** column contains the actual value of the left-most column instead +** of a blob encoding of the complete index key as is found in +** sqlite_stat4.sample. The nEq, nLt, and nDLt entries of sqlite_stat3 +** all contain just a single integer which is the same as the first +** integer in the equivalent columns in sqlite_stat4. */ #ifndef SQLITE_OMIT_ANALYZE +#if defined(SQLITE_ENABLE_STAT4) +# define IsStat4 1 +# define IsStat3 0 +#elif defined(SQLITE_ENABLE_STAT3) +# define IsStat4 0 +# define IsStat3 1 +#else +# define IsStat4 0 +# define IsStat3 0 +# undef SQLITE_STAT4_SAMPLES +# define SQLITE_STAT4_SAMPLES 1 +#endif +#define IsStat34 (IsStat3+IsStat4) /* 1 for STAT3 or STAT4. 0 otherwise */ + /* -** This routine generates code that opens the sqlite_stat1 table for -** writing with cursor iStatCur. If the library was built with the -** SQLITE_ENABLE_STAT2 macro defined, then the sqlite_stat2 table is -** opened for writing using cursor (iStatCur+1) +** This routine generates code that opens the sqlite_statN tables. +** The sqlite_stat1 table is always relevant. sqlite_stat2 is now +** obsolete. sqlite_stat3 and sqlite_stat4 are only opened when +** appropriate compile-time options are provided. ** -** If the sqlite_stat1 tables does not previously exist, it is created. -** Similarly, if the sqlite_stat2 table does not exist and the library -** is compiled with SQLITE_ENABLE_STAT2 defined, it is created. +** If the sqlite_statN tables do not previously exist, it is created. ** ** Argument zWhere may be a pointer to a buffer containing a table name, ** or it may be a NULL pointer. If it is not NULL, then all entries in -** the sqlite_stat1 and (if applicable) sqlite_stat2 tables associated -** with the named table are deleted. If zWhere==0, then code is generated -** to delete all stat table entries. +** the sqlite_statN tables associated with the named table are deleted. +** If zWhere==0, then code is generated to delete all stat table entries. */ static void openStatTable( Parse *pParse, /* Parsing context */ @@ -75847,61 +86011,773 @@ static void openStatTable( const char *zCols; } aTable[] = { { "sqlite_stat1", "tbl,idx,stat" }, -#ifdef SQLITE_ENABLE_STAT2 - { "sqlite_stat2", "tbl,idx,sampleno,sample" }, +#if defined(SQLITE_ENABLE_STAT4) + { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, + { "sqlite_stat3", 0 }, +#elif defined(SQLITE_ENABLE_STAT3) + { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, + { "sqlite_stat4", 0 }, +#else + { "sqlite_stat3", 0 }, + { "sqlite_stat4", 0 }, #endif }; - - int aRoot[] = {0, 0}; - u8 aCreateTbl[] = {0, 0}; - int i; sqlite3 *db = pParse->db; Db *pDb; Vdbe *v = sqlite3GetVdbe(pParse); + int aRoot[ArraySize(aTable)]; + u8 aCreateTbl[ArraySize(aTable)]; + if( v==0 ) return; assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3VdbeDb(v)==db ); pDb = &db->aDb[iDb]; + /* Create new statistic tables if they do not exist, or clear them + ** if they do already exist. + */ for(i=0; izName))==0 ){ - /* The sqlite_stat[12] table does not exist. Create it. Note that a - ** side-effect of the CREATE TABLE statement is to leave the rootpage - ** of the new table in register pParse->regRoot. This is important - ** because the OpenWrite opcode below will be needing it. */ - sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols - ); - aRoot[i] = pParse->regRoot; - aCreateTbl[i] = 1; + if( aTable[i].zCols ){ + /* The sqlite_statN table does not exist. Create it. Note that a + ** side-effect of the CREATE TABLE statement is to leave the rootpage + ** of the new table in register pParse->regRoot. This is important + ** because the OpenWrite opcode below will be needing it. */ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols + ); + aRoot[i] = pParse->regRoot; + aCreateTbl[i] = OPFLAG_P2ISREG; + } }else{ /* The table already exists. If zWhere is not NULL, delete all entries ** associated with the table zWhere. If zWhere is NULL, delete the ** entire contents of the table. */ aRoot[i] = pStat->tnum; + aCreateTbl[i] = 0; sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); if( zWhere ){ sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE %s=%Q", pDb->zName, zTab, zWhereType, zWhere + "DELETE FROM %Q.%s WHERE %s=%Q", + pDb->zName, zTab, zWhereType, zWhere ); }else{ - /* The sqlite_stat[12] table already exists. Delete all rows. */ + /* The sqlite_stat[134] table already exists. Delete all rows. */ sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb); } } } - /* Open the sqlite_stat[12] tables for writing. */ - for(i=0; inRowid ){ + sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + } +} +#endif + +/* Initialize the BLOB value of a ROWID +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->u.aRowid = sqlite3DbMallocRaw(db, n); + if( p->u.aRowid ){ + p->nRowid = n; + memcpy(p->u.aRowid, pData, n); + }else{ + p->nRowid = 0; + } +} +#endif + +/* Initialize the INTEGER value of a ROWID. +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + p->u.iRowid = iRowid; +} +#endif + + +/* +** Copy the contents of object (*pFrom) into (*pTo). +*/ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ + pTo->isPSample = pFrom->isPSample; + pTo->iCol = pFrom->iCol; + pTo->iHash = pFrom->iHash; + memcpy(pTo->anEq, pFrom->anEq, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anLt, pFrom->anLt, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anDLt, pFrom->anDLt, sizeof(tRowcnt)*p->nCol); + if( pFrom->nRowid ){ + sampleSetRowid(p->db, pTo, pFrom->nRowid, pFrom->u.aRowid); + }else{ + sampleSetRowidInt64(p->db, pTo, pFrom->u.iRowid); + } +} +#endif + +/* +** Reclaim all memory of a Stat4Accum structure. +*/ +static void stat4Destructor(void *pOld){ + Stat4Accum *p = (Stat4Accum*)pOld; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int i; + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); + for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); + sampleClear(p->db, &p->current); +#endif + sqlite3DbFree(p->db, p); +} + +/* +** Implementation of the stat_init(N,K,C) SQL function. The three parameters +** are: +** N: The number of columns in the index including the rowid/pk (note 1) +** K: The number of columns in the index excluding the rowid/pk. +** C: The number of rows in the index (note 2) +** +** Note 1: In the special case of the covering index that implements a +** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the +** total number of columns in the table. +** +** Note 2: C is only used for STAT3 and STAT4. +** +** For indexes on ordinary rowid tables, N==K+1. But for indexes on +** WITHOUT ROWID tables, N=K+P where P is the number of columns in the +** PRIMARY KEY of the table. The covering index that implements the +** original WITHOUT ROWID table as N==K as a special case. +** +** This routine allocates the Stat4Accum object in heap memory. The return +** value is a pointer to the Stat4Accum object. The datatype of the +** return value is BLOB, but it is really just a pointer to the Stat4Accum +** object. +*/ +static void statInit( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stat4Accum *p; + int nCol; /* Number of columns in index being sampled */ + int nKeyCol; /* Number of key columns */ + int nColUp; /* nCol rounded up for alignment */ + int n; /* Bytes of space to allocate */ + sqlite3 *db; /* Database connection */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int mxSample = SQLITE_STAT4_SAMPLES; +#endif + + /* Decode the three function arguments */ + UNUSED_PARAMETER(argc); + nCol = sqlite3_value_int(argv[0]); + assert( nCol>0 ); + nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; + nKeyCol = sqlite3_value_int(argv[1]); + assert( nKeyCol<=nCol ); + assert( nKeyCol>0 ); + + /* Allocate the space required for the Stat4Accum object */ + n = sizeof(*p) + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anEq */ + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anDLt */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anLt */ + + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample) +#endif + ; + db = sqlite3_context_db_handle(context); + p = sqlite3DbMallocZero(db, n); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + p->db = db; + p->nRow = 0; + p->nCol = nCol; + p->nKeyCol = nKeyCol; + p->current.anDLt = (tRowcnt*)&p[1]; + p->current.anEq = &p->current.anDLt[nColUp]; + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + { + u8 *pSpace; /* Allocated space not yet assigned */ + int i; /* Used to iterate through p->aSample[] */ + + p->iGet = -1; + p->mxSample = mxSample; + p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); + p->current.anLt = &p->current.anEq[nColUp]; + p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565; + + /* Set up the Stat4Accum.a[] and aBest[] arrays */ + p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; + p->aBest = &p->a[mxSample]; + pSpace = (u8*)(&p->a[mxSample+nCol]); + for(i=0; i<(mxSample+nCol); i++){ + p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + } + assert( (pSpace - (u8*)p)==n ); + + for(i=0; iaBest[i].iCol = i; + } + } +#endif + + /* Return a pointer to the allocated object to the caller. Note that + ** only the pointer (the 2nd parameter) matters. The size of the object + ** (given by the 3rd parameter) is never used and can be any positive + ** value. */ + sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); +} +static const FuncDef statInitFuncdef = { + 2+IsStat34, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statInit, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_init", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +#ifdef SQLITE_ENABLE_STAT4 +/* +** pNew and pOld are both candidate non-periodic samples selected for +** the same column (pNew->iCol==pOld->iCol). Ignoring this column and +** considering only any trailing columns and the sample hash value, this +** function returns true if sample pNew is to be preferred over pOld. +** In other words, if we assume that the cardinalities of the selected +** column for pNew and pOld are equal, is pNew to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol+1] onwards are valid. +*/ +static int sampleIsBetterPost( + Stat4Accum *pAccum, + Stat4Sample *pNew, + Stat4Sample *pOld +){ + int nCol = pAccum->nCol; + int i; + assert( pNew->iCol==pOld->iCol ); + for(i=pNew->iCol+1; ianEq[i]>pOld->anEq[i] ) return 1; + if( pNew->anEq[i]anEq[i] ) return 0; + } + if( pNew->iHash>pOld->iHash ) return 1; + return 0; +} +#endif + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Return true if pNew is to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol] onwards are valid. +*/ +static int sampleIsBetter( + Stat4Accum *pAccum, + Stat4Sample *pNew, + Stat4Sample *pOld +){ + tRowcnt nEqNew = pNew->anEq[pNew->iCol]; + tRowcnt nEqOld = pOld->anEq[pOld->iCol]; + + assert( pOld->isPSample==0 && pNew->isPSample==0 ); + assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); + + if( (nEqNew>nEqOld) ) return 1; +#ifdef SQLITE_ENABLE_STAT4 + if( nEqNew==nEqOld ){ + if( pNew->iColiCol ) return 1; + return (pNew->iCol==pOld->iCol && sampleIsBetterPost(pAccum, pNew, pOld)); + } + return 0; +#else + return (nEqNew==nEqOld && pNew->iHash>pOld->iHash); +#endif +} + +/* +** Copy the contents of sample *pNew into the p->a[] array. If necessary, +** remove the least desirable sample from p->a[] to make room. +*/ +static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ + Stat4Sample *pSample = 0; + int i; + + assert( IsStat4 || nEqZero==0 ); + +#ifdef SQLITE_ENABLE_STAT4 + if( pNew->isPSample==0 ){ + Stat4Sample *pUpgrade = 0; + assert( pNew->anEq[pNew->iCol]>0 ); + + /* This sample is being added because the prefix that ends in column + ** iCol occurs many times in the table. However, if we have already + ** added a sample that shares this prefix, there is no need to add + ** this one. Instead, upgrade the priority of the highest priority + ** existing sample that shares this prefix. */ + for(i=p->nSample-1; i>=0; i--){ + Stat4Sample *pOld = &p->a[i]; + if( pOld->anEq[pNew->iCol]==0 ){ + if( pOld->isPSample ) return; + assert( pOld->iCol>pNew->iCol ); + assert( sampleIsBetter(p, pNew, pOld) ); + if( pUpgrade==0 || sampleIsBetter(p, pOld, pUpgrade) ){ + pUpgrade = pOld; + } + } + } + if( pUpgrade ){ + pUpgrade->iCol = pNew->iCol; + pUpgrade->anEq[pUpgrade->iCol] = pNew->anEq[pUpgrade->iCol]; + goto find_new_min; + } + } +#endif + + /* If necessary, remove sample iMin to make room for the new sample. */ + if( p->nSample>=p->mxSample ){ + Stat4Sample *pMin = &p->a[p->iMin]; + tRowcnt *anEq = pMin->anEq; + tRowcnt *anLt = pMin->anLt; + tRowcnt *anDLt = pMin->anDLt; + sampleClear(p->db, pMin); + memmove(pMin, &pMin[1], sizeof(p->a[0])*(p->nSample-p->iMin-1)); + pSample = &p->a[p->nSample-1]; + pSample->nRowid = 0; + pSample->anEq = anEq; + pSample->anDLt = anDLt; + pSample->anLt = anLt; + p->nSample = p->mxSample-1; + } + + /* The "rows less-than" for the rowid column must be greater than that + ** for the last sample in the p->a[] array. Otherwise, the samples would + ** be out of order. */ +#ifdef SQLITE_ENABLE_STAT4 + assert( p->nSample==0 + || pNew->anLt[p->nCol-1] > p->a[p->nSample-1].anLt[p->nCol-1] ); +#endif + + /* Insert the new sample */ + pSample = &p->a[p->nSample]; + sampleCopy(p, pSample, pNew); + p->nSample++; + + /* Zero the first nEqZero entries in the anEq[] array. */ + memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); + +#ifdef SQLITE_ENABLE_STAT4 + find_new_min: +#endif + if( p->nSample>=p->mxSample ){ + int iMin = -1; + for(i=0; imxSample; i++){ + if( p->a[i].isPSample ) continue; + if( iMin<0 || sampleIsBetter(p, &p->a[iMin], &p->a[i]) ){ + iMin = i; + } + } + assert( iMin>=0 ); + p->iMin = iMin; + } +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** Field iChng of the index being scanned has changed. So at this point +** p->current contains a sample that reflects the previous row of the +** index. The value of anEq[iChng] and subsequent anEq[] elements are +** correct at this point. +*/ +static void samplePushPrevious(Stat4Accum *p, int iChng){ +#ifdef SQLITE_ENABLE_STAT4 + int i; + + /* Check if any samples from the aBest[] array should be pushed + ** into IndexSample.a[] at this point. */ + for(i=(p->nCol-2); i>=iChng; i--){ + Stat4Sample *pBest = &p->aBest[i]; + pBest->anEq[i] = p->current.anEq[i]; + if( p->nSamplemxSample || sampleIsBetter(p, pBest, &p->a[p->iMin]) ){ + sampleInsert(p, pBest, i); + } + } + + /* Update the anEq[] fields of any samples already collected. */ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } + } +#endif + +#if defined(SQLITE_ENABLE_STAT3) && !defined(SQLITE_ENABLE_STAT4) + if( iChng==0 ){ + tRowcnt nLt = p->current.anLt[0]; + tRowcnt nEq = p->current.anEq[0]; + + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+nEq)/p->nPSample ){ + p->current.isPSample = 1; + sampleInsert(p, &p->current, 0); + p->current.isPSample = 0; + }else + + /* Or if it is a non-periodic sample. Add it in this case too. */ + if( p->nSamplemxSample + || sampleIsBetter(p, &p->current, &p->a[p->iMin]) + ){ + sampleInsert(p, &p->current, 0); + } + } +#endif + +#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 + UNUSED_PARAMETER( p ); + UNUSED_PARAMETER( iChng ); +#endif +} + +/* +** Implementation of the stat_push SQL function: stat_push(P,C,R) +** Arguments: +** +** P Pointer to the Stat4Accum object created by stat_init() +** C Index of left-most column to differ from previous row +** R Rowid for the current row. Might be a key record for +** WITHOUT ROWID tables. +** +** This SQL function always returns NULL. It's purpose it to accumulate +** statistical data and/or samples in the Stat4Accum object about the +** index being analyzed. The stat_get() SQL function will later be used to +** extract relevant information for constructing the sqlite_statN tables. +** +** The R parameter is only used for STAT3 and STAT4 +*/ +static void statPush( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + + /* The three function arguments */ + Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); + int iChng = sqlite3_value_int(argv[1]); + + UNUSED_PARAMETER( argc ); + UNUSED_PARAMETER( context ); + assert( p->nCol>0 ); + assert( iChngnCol ); + + if( p->nRow==0 ){ + /* This is the first call to this function. Do initialization. */ + for(i=0; inCol; i++) p->current.anEq[i] = 1; + }else{ + /* Second and subsequent calls get processed here */ + samplePushPrevious(p, iChng); + + /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply + ** to the current row of the index. */ + for(i=0; icurrent.anEq[i]++; + } + for(i=iChng; inCol; i++){ + p->current.anDLt[i]++; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + p->current.anLt[i] += p->current.anEq[i]; +#endif + p->current.anEq[i] = 1; + } + } + p->nRow++; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ + sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); + }else{ + sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), + sqlite3_value_blob(argv[2])); + } + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; +#endif + +#ifdef SQLITE_ENABLE_STAT4 + { + tRowcnt nLt = p->current.anLt[p->nCol-1]; + + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ + p->current.isPSample = 1; + p->current.iCol = 0; + sampleInsert(p, &p->current, p->nCol-1); + p->current.isPSample = 0; + } + + /* Update the aBest[] array. */ + for(i=0; i<(p->nCol-1); i++){ + p->current.iCol = i; + if( i>=iChng || sampleIsBetterPost(p, &p->current, &p->aBest[i]) ){ + sampleCopy(p, &p->aBest[i], &p->current); + } + } + } +#endif +} +static const FuncDef statPushFuncdef = { + 2+IsStat34, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statPush, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_push", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +#define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ +#define STAT_GET_ROWID 1 /* "rowid" column of stat[34] entry */ +#define STAT_GET_NEQ 2 /* "neq" column of stat[34] entry */ +#define STAT_GET_NLT 3 /* "nlt" column of stat[34] entry */ +#define STAT_GET_NDLT 4 /* "ndlt" column of stat[34] entry */ + +/* +** Implementation of the stat_get(P,J) SQL function. This routine is +** used to query statistical information that has been gathered into +** the Stat4Accum object by prior calls to stat_push(). The P parameter +** has type BLOB but it is really just a pointer to the Stat4Accum object. +** The content to returned is determined by the parameter J +** which is one of the STAT_GET_xxxx values defined above. +** +** If neither STAT3 nor STAT4 are enabled, then J is always +** STAT_GET_STAT1 and is hence omitted and this routine becomes +** a one-parameter function, stat_get(P), that always returns the +** stat1 table entry information. +*/ +static void statGet( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* STAT3 and STAT4 have a parameter on this routine. */ + int eCall = sqlite3_value_int(argv[1]); + assert( argc==2 ); + assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ + || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT + || eCall==STAT_GET_NDLT + ); + if( eCall==STAT_GET_STAT1 ) +#else + assert( argc==1 ); +#endif + { + /* Return the value to store in the "stat" column of the sqlite_stat1 + ** table for this index. + ** + ** The value is a string composed of a list of integers describing + ** the index. The first integer in the list is the total number of + ** entries in the index. There is one additional integer in the list + ** for each indexed column. This additional integer is an estimate of + ** the number of rows matched by a stabbing query on the index using + ** a key with the corresponding number of fields. In other words, + ** if the index is on columns (a,b) and the sqlite_stat1 value is + ** "100 10 2", then SQLite estimates that: + ** + ** * the index contains 100 rows, + ** * "WHERE a=?" matches 10 rows, and + ** * "WHERE a=? AND b=?" matches 2 rows. + ** + ** If D is the count of distinct values and K is the total number of + ** rows, then each estimate is computed as: + ** + ** I = (K+D-1)/D + */ + char *z; + int i; + + char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 ); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); + z = zRet + sqlite3Strlen30(zRet); + for(i=0; inKeyCol; i++){ + u64 nDistinct = p->current.anDLt[i] + 1; + u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; + sqlite3_snprintf(24, z, " %llu", iVal); + z += sqlite3Strlen30(z); + assert( p->current.anEq[i] ); + } + assert( z[0]=='\0' && z>zRet ); + + sqlite3_result_text(context, zRet, -1, sqlite3_free); + } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + else if( eCall==STAT_GET_ROWID ){ + if( p->iGet<0 ){ + samplePushPrevious(p, 0); + p->iGet = 0; + } + if( p->iGetnSample ){ + Stat4Sample *pS = p->a + p->iGet; + if( pS->nRowid==0 ){ + sqlite3_result_int64(context, pS->u.iRowid); + }else{ + sqlite3_result_blob(context, pS->u.aRowid, pS->nRowid, + SQLITE_TRANSIENT); + } + } + }else{ + tRowcnt *aCnt = 0; + + assert( p->iGetnSample ); + switch( eCall ){ + case STAT_GET_NEQ: aCnt = p->a[p->iGet].anEq; break; + case STAT_GET_NLT: aCnt = p->a[p->iGet].anLt; break; + default: { + aCnt = p->a[p->iGet].anDLt; + p->iGet++; + break; + } + } + + if( IsStat3 ){ + sqlite3_result_int64(context, (i64)aCnt[0]); + }else{ + char *zRet = sqlite3MallocZero(p->nCol * 25); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + char *z = zRet; + for(i=0; inCol; i++){ + sqlite3_snprintf(24, z, "%llu ", (u64)aCnt[i]); + z += sqlite3Strlen30(z); + } + assert( z[0]=='\0' && z>zRet ); + z[-1] = '\0'; + sqlite3_result_text(context, zRet, -1, sqlite3_free); + } + } + } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( argc ); +#endif +} +static const FuncDef statGetFuncdef = { + 1+IsStat34, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statGet, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_get", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ + assert( regOut!=regStat4 && regOut!=regStat4+1 ); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); +#elif SQLITE_DEBUG + assert( iParam==STAT_GET_STAT1 ); +#else + UNUSED_PARAMETER( iParam ); +#endif + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); + sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 1 + IsStat34); +} + /* ** Generate code to do an analysis of all indices associated with ** a single table. @@ -75911,34 +86787,31 @@ static void analyzeOneTable( Table *pTab, /* Table whose indices are to be analyzed */ Index *pOnlyIdx, /* If not NULL, only analyze this one index */ int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ - int iMem /* Available memory locations begin here */ + int iMem, /* Available memory locations begin here */ + int iTab /* Next available cursor */ ){ sqlite3 *db = pParse->db; /* Database handle */ Index *pIdx; /* An index to being analyzed */ int iIdxCur; /* Cursor open on index being analyzed */ + int iTabCur; /* Table cursor */ Vdbe *v; /* The virtual machine being built up */ int i; /* Loop counter */ - int topOfLoop; /* The top of the loop */ - int endOfLoop; /* The end of the loop */ int jZeroRows = -1; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ + u8 needTableCnt = 1; /* True to count the table */ + int regNewRowid = iMem++; /* Rowid for the inserted record */ + int regStat4 = iMem++; /* Register to hold Stat4Accum object */ + int regChng = iMem++; /* Index of changed index field */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int regRowid = iMem++; /* Rowid argument passed to stat_push() */ +#endif + int regTemp = iMem++; /* Temporary use register */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ - int regSampleno = iMem++; /* Register containing next sample number */ - int regCol = iMem++; /* Content of a column analyzed table */ - int regRec = iMem++; /* Register holding completed record */ - int regTemp = iMem++; /* Temporary use register */ - int regRowid = iMem++; /* Rowid for the inserted record */ - -#ifdef SQLITE_ENABLE_STAT2 - int addr = 0; /* Instruction address */ - int regTemp2 = iMem++; /* Temporary use register */ - int regSamplerecno = iMem++; /* Index of next sample to record */ - int regRecno = iMem++; /* Current sample index */ - int regLast = iMem++; /* Index of last sample to record */ - int regFirst = iMem++; /* Index of first sample to record */ -#endif + int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ + int regPrev = iMem; /* MUST BE LAST (see below) */ + pParse->nMem = MAX(pParse->nMem, iMem); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -75947,7 +86820,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -75962,214 +86835,276 @@ static void analyzeOneTable( } #endif - /* Establish a read-lock on the table at the shared-cache level. */ + /* Establish a read-lock on the table at the shared-cache level. + ** Open a read-only cursor on the table. Also allocate a cursor number + ** to use for scanning indexes (iIdxCur). No index cursor is opened at + ** this time though. */ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - - iIdxCur = pParse->nTab++; + iTabCur = iTab++; + iIdxCur = iTab++; + pParse->nTab = MAX(pParse->nTab, iTab); + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int nCol; - KeyInfo *pKey; + int nCol; /* Number of columns in pIdx. "N" */ + int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrNextRow; /* Address of "next_row:" */ + const char *zIdxName; /* Name of the index */ + int nColTest; /* Number of columns to test for changes */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; - nCol = pIdx->nColumn; - pKey = sqlite3IndexKeyinfo(pParse, pIdx); - if( iMem+1+(nCol*2)>pParse->nMem ){ - pParse->nMem = iMem+1+(nCol*2); + if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nCol = pIdx->nKeyCol; + zIdxName = pTab->zName; + nColTest = nCol - 1; + }else{ + nCol = pIdx->nColumn; + zIdxName = pIdx->zName; + nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; } - /* Open a cursor to the index to be analyzed. */ - assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); - sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb, - (char *)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIdx->zName)); - /* Populate the register containing the index name. */ - sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); -#ifdef SQLITE_ENABLE_STAT2 - - /* If this iteration of the loop is generating code to analyze the - ** first index in the pTab->pIndex list, then register regLast has - ** not been populated. In this case populate it now. */ - if( pTab->pIndex==pIdx ){ - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES, regSamplerecno); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2-1, regTemp); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2, regTemp2); - - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regLast); - sqlite3VdbeAddOp2(v, OP_Null, 0, regFirst); - addr = sqlite3VdbeAddOp3(v, OP_Lt, regSamplerecno, 0, regLast); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regFirst); - sqlite3VdbeAddOp3(v, OP_Multiply, regLast, regTemp, regLast); - sqlite3VdbeAddOp2(v, OP_AddImm, regLast, SQLITE_INDEX_SAMPLES*2-2); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regLast); - sqlite3VdbeJumpHere(v, addr); - } - - /* Zero the regSampleno and regRecno registers. */ - sqlite3VdbeAddOp2(v, OP_Integer, 0, regSampleno); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRecno); - sqlite3VdbeAddOp2(v, OP_Copy, regFirst, regSamplerecno); -#endif - - /* The block of memory cells initialized here is used as follows. + /* + ** Pseudo-code for loop that calls stat_push(): ** - ** iMem: - ** The total number of rows in the table. + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto chng_addr_0; ** - ** iMem+1 .. iMem+nCol: - ** Number of distinct entries in index considering the - ** left-most N columns only, where N is between 1 and nCol, - ** inclusive. + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto chng_addr_N ** - ** iMem+nCol+1 .. Mem+2*nCol: - ** Previous value of indexed columns, from left to right. + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... ** - ** Cells iMem through iMem+nCol are initialized to 0. The others are - ** initialized to contain an SQL NULL. + ** endDistinctTest: + ** regRowid = idx(rowid) + ** stat_push(P, regChng, regRowid) + ** Next csr + ** if !eof(csr) goto next_row; + ** + ** end_of_scan: */ - for(i=0; i<=nCol; i++){ - sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem+i); - } - for(i=0; inMem = MAX(pParse->nMem, regPrev+nColTest); - for(i=0; ipSchema) ); + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); - /* Calculate new values for regSamplerecno and regSampleno. - ** - ** sampleno = sampleno + 1 - ** samplerecno = samplerecno+(remaining records)/(remaining samples) - */ - sqlite3VdbeAddOp2(v, OP_AddImm, regSampleno, 1); - sqlite3VdbeAddOp3(v, OP_Subtract, regRecno, regLast, regTemp); - sqlite3VdbeAddOp2(v, OP_AddImm, regTemp, -1); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES, regTemp2); - sqlite3VdbeAddOp3(v, OP_Subtract, regSampleno, regTemp2, regTemp2); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regTemp, regTemp); - sqlite3VdbeAddOp3(v, OP_Add, regSamplerecno, regTemp, regSamplerecno); - - sqlite3VdbeJumpHere(v, ne); - sqlite3VdbeAddOp2(v, OP_AddImm, regRecno, 1); -#endif - - /* Always record the very first row */ - sqlite3VdbeAddOp1(v, OP_IfNot, iMem+1); - } - assert( pIdx->azColl!=0 ); - assert( pIdx->azColl[i]!=0 ); - pColl = sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); - sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1, - (char*)pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - } - if( db->mallocFailed ){ - /* If a malloc failure has occurred, then the result of the expression - ** passed as the second argument to the call to sqlite3VdbeJumpHere() - ** below may be negative. Which causes an assert() to fail (or an - ** out-of-bounds write if SQLITE_DEBUG is not defined). */ - return; - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop); - for(i=0; i0 then it is always the case the D>0 so division by zero - ** is never possible. + ** The third argument is only used for STAT3 and STAT4 */ - sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno); - if( jZeroRows<0 ){ - jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); +#endif + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); + sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2+IsStat34); + + /* Implementation of the following: + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto next_push_0; + ** + */ + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); + addrNextRow = sqlite3VdbeCurrentAddr(v); + + if( nColTest>0 ){ + int endDistinctTest = sqlite3VdbeMakeLabel(v); + int *aGotoChng; /* Array of jump instruction addresses */ + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + if( aGotoChng==0 ) continue; + + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto endDistinctTest + */ + sqlite3VdbeAddOp0(v, OP_Goto); + addrNextRow = sqlite3VdbeCurrentAddr(v); + if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){ + /* For a single-column UNIQUE index, once we have found a non-NULL + ** row, we know that all the rest will be distinct, so skip + ** subsequent distinctness tests. */ + sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); + VdbeCoverage(v); + } + for(i=0; iazColl[i]); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + + + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + */ + sqlite3VdbeJumpHere(v, addrNextRow-1); + for(i=0; ipTable); + int j, k, regKey; + regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); + VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); + sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); +#endif + assert( regChng==(regStat4+1) ); + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); + sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2+IsStat34); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + + /* Add the entry to the stat1 table. */ + callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + + /* Add the entries to the stat3 or stat4 table. */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + { + int regEq = regStat1; + int regLt = regStat1+1; + int regDLt = regStat1+2; + int regSample = regStat1+3; + int regCol = regStat1+4; + int regSampleRowid = regCol + nCol; + int addrNext; + int addrIsNull; + u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + + pParse->nMem = MAX(pParse->nMem, regCol+nCol); + + addrNext = sqlite3VdbeCurrentAddr(v); + callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); + VdbeCoverage(v); + callStatGet(v, regStat4, STAT_GET_NEQ, regEq); + callStatGet(v, regStat4, STAT_GET_NLT, regLt); + callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); + sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); + /* We know that the regSampleRowid row exists because it was read by + ** the previous loop. Thus the not-found jump of seekOp will never + ** be taken */ + VdbeCoverageNeverTaken(v); +#ifdef SQLITE_ENABLE_STAT3 + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, + pIdx->aiColumn[0], regSample); +#else + for(i=0; iaiColumn[i]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); +#endif + sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regTemp, regNewRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 1, addrNext); /* P1==1 for end-of-loop */ + sqlite3VdbeJumpHere(v, addrIsNull); + } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + + /* End of analysis */ + sqlite3VdbeJumpHere(v, addrRewind); } - /* If the table has no indices, create a single sqlite_stat1 entry - ** containing NULL as the index name and the row count as the content. + + /* Create a single sqlite_stat1 entry containing NULL as the index + ** name and the row count as the content. */ - if( pTab->pIndex==0 ){ - sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb); + if( pOnlyIdx==0 && needTableCnt ){ VdbeComment((v, "%s", pTab->zName)); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno); - sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); - jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regSampleno); - }else{ + sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); - jZeroRows = sqlite3VdbeAddOp0(v, OP_Goto); } - sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - if( pParse->nMemnMem = regRec; - sqlite3VdbeJumpHere(v, jZeroRows); } + /* ** Generate code that will cause the most recent index analysis to ** be loaded into internal hash tables where is can be used. @@ -76190,16 +87125,18 @@ static void analyzeDatabase(Parse *pParse, int iDb){ HashElem *k; int iStatCur; int iMem; + int iTab; sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; - pParse->nTab += 2; + pParse->nTab += 3; openStatTable(pParse, iDb, iStatCur, 0, 0); iMem = pParse->nMem+1; + iTab = pParse->nTab; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); } loadAnalysis(pParse, iDb); } @@ -76218,13 +87155,13 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; - pParse->nTab += 2; + pParse->nTab += 3; if( pOnlyIdx ){ openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx"); }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur, pParse->nMem+1); + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); loadAnalysis(pParse, iDb); } @@ -76248,6 +87185,7 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ Table *pTab; Index *pIdx; Token *pTableName; + Vdbe *v; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -76295,6 +87233,8 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* @@ -76307,6 +87247,66 @@ struct analysisInfo { const char *zDatabase; }; +/* +** The first argument points to a nul-terminated string containing a +** list of space separated integers. Read the first nOut of these into +** the array aOut[]. +*/ +static void decodeIntArray( + char *zIntArray, /* String containing int array to decode */ + int nOut, /* Number of slots in aOut[] */ + tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ + Index *pIndex /* Handle extra flags for this index, if not NULL */ +){ + char *z = zIntArray; + int c; + int i; + tRowcnt v; + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( z==0 ) z = ""; +#else + assert( z!=0 ); +#endif + for(i=0; *z && i='0' && c<='9' ){ + v = v*10 + c - '0'; + z++; + } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( aOut ) aOut[i] = v; + if( aLog ) aLog[i] = sqlite3LogEst(v); +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); + assert( aLog!=0 ); + aLog[i] = sqlite3LogEst(v); +#endif + if( *z==' ' ) z++; + } +#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 + assert( pIndex!=0 ); +#else + if( pIndex ) +#endif + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + } +#ifdef SQLITE_ENABLE_COSTMULT + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } +#endif + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } +} + /* ** This callback is invoked once for each index when reading the ** sqlite_stat1 table. @@ -76322,8 +87322,6 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ analysisInfo *pInfo = (analysisInfo*)pData; Index *pIndex; Table *pTable; - int i, c, n; - unsigned int v; const char *z; assert( argc==3 ); @@ -76336,28 +87334,38 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pTable==0 ){ return 0; } - if( argv[1] ){ - pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); - }else{ + if( argv[1]==0 ){ pIndex = 0; + }else if( sqlite3_stricmp(argv[0],argv[1])==0 ){ + pIndex = sqlite3PrimaryKeyIndex(pTable); + }else{ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); } - n = pIndex ? pIndex->nColumn : 0; z = argv[2]; - for(i=0; *z && i<=n; i++){ - v = 0; - while( (c=z[0])>='0' && c<='9' ){ - v = v*10 + c - '0'; - z++; - } - if( i==0 ) pTable->nRowEst = v; - if( pIndex==0 ) break; - pIndex->aiRowEst[i] = v; - if( *z==' ' ) z++; - if( memcmp(z, "unordered", 10)==0 ){ - pIndex->bUnordered = 1; - break; - } + + if( pIndex ){ + int nCol = pIndex->nKeyCol+1; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero( + sizeof(tRowcnt) * nCol + ); + if( aiRowEst==0 ) pInfo->db->mallocFailed = 1; +#else + tRowcnt * const aiRowEst = 0; +#endif + pIndex->bUnordered = 0; + decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + }else{ + Index fakeIdx; + fakeIdx.szIdxRow = pTable->szTabRow; +#ifdef SQLITE_ENABLE_COSTMULT + fakeIdx.pTable = pTable; +#endif + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); + pTable->szTabRow = fakeIdx.szIdxRow; } + return 0; } @@ -76366,36 +87374,275 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( pIdx->aSample ){ int j; - for(j=0; jnSample; j++){ IndexSample *p = &pIdx->aSample[j]; - if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ - sqlite3DbFree(db, p->u.z); - } + sqlite3DbFree(db, p->p); } sqlite3DbFree(db, pIdx->aSample); } + if( db && db->pnBytesFreed==0 ){ + pIdx->nSample = 0; + pIdx->aSample = 0; + } #else UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); -#endif +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Populate the pIdx->aAvgEq[] array based on the samples currently +** stored in pIdx->aSample[]. +*/ +static void initAvgEq(Index *pIdx){ + if( pIdx ){ + IndexSample *aSample = pIdx->aSample; + IndexSample *pFinal = &aSample[pIdx->nSample-1]; + int iCol; + int nCol = 1; + if( pIdx->nSampleCol>1 ){ + /* If this is stat4 data, then calculate aAvgEq[] values for all + ** sample columns except the last. The last is always set to 1, as + ** once the trailing PK fields are considered all index keys are + ** unique. */ + nCol = pIdx->nSampleCol-1; + pIdx->aAvgEq[nCol] = 1; + } + for(iCol=0; iColnSample; + int i; /* Used to iterate through samples */ + tRowcnt sumEq = 0; /* Sum of the nEq values */ + tRowcnt avgEq = 0; + tRowcnt nRow; /* Number of rows in index */ + i64 nSum100 = 0; /* Number of terms contributing to sumEq */ + i64 nDist100; /* Number of distinct values in index */ + + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ + nRow = pFinal->anLt[iCol]; + nDist100 = (i64)100 * pFinal->anDLt[iCol]; + nSample--; + }else{ + nRow = pIdx->aiRowEst[0]; + nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; + } + + /* Set nSum to the number of distinct (iCol+1) field prefixes that + ** occur in the stat4 table for this index. Set sumEq to the sum of + ** the nEq values for column iCol for the same set (adding the value + ** only once where there exist duplicate prefixes). */ + for(i=0; inSample-1) + || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] + ){ + sumEq += aSample[i].anEq[iCol]; + nSum100 += 100; + } + } + + if( nDist100>nSum100 ){ + avgEq = ((i64)100 * (nRow - sumEq))/(nDist100 - nSum100); + } + if( avgEq==0 ) avgEq = 1; + pIdx->aAvgEq[iCol] = avgEq; + } + } } /* -** Load the content of the sqlite_stat1 and sqlite_stat2 tables. The +** Look up an index by name. Or, if the name of a WITHOUT ROWID table +** is supplied instead, find the PRIMARY KEY index for that table. +*/ +static Index *findIndexOrPrimaryKey( + sqlite3 *db, + const char *zName, + const char *zDb +){ + Index *pIdx = sqlite3FindIndex(db, zName, zDb); + if( pIdx==0 ){ + Table *pTab = sqlite3FindTable(db, zName, zDb); + if( pTab && !HasRowid(pTab) ) pIdx = sqlite3PrimaryKeyIndex(pTab); + } + return pIdx; +} + +/* +** Load the content from either the sqlite_stat4 or sqlite_stat3 table +** into the relevant Index.aSample[] arrays. +** +** Arguments zSql1 and zSql2 must point to SQL statements that return +** data equivalent to the following (statements are different for stat3, +** see the caller of this function for details): +** +** zSql1: SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx +** zSql2: SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4 +** +** where %Q is replaced with the database name before the SQL is executed. +*/ +static int loadStatTbl( + sqlite3 *db, /* Database handle */ + int bStat3, /* Assume single column records only */ + const char *zSql1, /* SQL statement 1 (see above) */ + const char *zSql2, /* SQL statement 2 (see above) */ + const char *zDb /* Database name (e.g. "main") */ +){ + int rc; /* Result codes from subroutines */ + sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ + char *zSql; /* Text of the SQL statement */ + Index *pPrevIdx = 0; /* Previous index in the loop */ + IndexSample *pSample; /* A slot in pIdx->aSample[] */ + + assert( db->lookaside.bEnabled==0 ); + zSql = sqlite3MPrintf(db, zSql1, zDb); + if( !zSql ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int nIdxCol = 1; /* Number of columns in stat4 records */ + + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + int nByte; /* Bytes of space required */ + int i; /* Bytes of space required */ + tRowcnt *pSpace; + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + nSample = sqlite3_column_int(pStmt, 1); + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + assert( pIdx==0 || bStat3 || pIdx->nSample==0 ); + /* Index.nSample is non-zero at this point if data has already been + ** loaded from the stat4 table. In this case ignore stat3 data. */ + if( pIdx==0 || pIdx->nSample ) continue; + if( bStat3==0 ){ + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } + } + pIdx->nSampleCol = nIdxCol; + nByte = sizeof(IndexSample) * nSample; + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + + pIdx->aSample = sqlite3DbMallocZero(db, nByte); + if( pIdx->aSample==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; + } + pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; + } + assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); + } + rc = sqlite3_finalize(pStmt); + if( rc ) return rc; + + zSql = sqlite3MPrintf(db, zSql2, zDb); + if( !zSql ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nCol = 1; /* Number of columns in index */ + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + if( pIdx==0 ) continue; + /* This next condition is true if data has already been loaded from + ** the sqlite_stat4 table. In this case ignore stat3 data. */ + nCol = pIdx->nSampleCol; + if( bStat3 && nCol>1 ) continue; + if( pIdx!=pPrevIdx ){ + initAvgEq(pPrevIdx); + pPrevIdx = pIdx; + } + pSample = &pIdx->aSample[pIdx->nSample]; + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); + + /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + ** This is in case the sample record is corrupted. In that case, the + ** sqlite3VdbeRecordCompare() may read up to two varints past the + ** end of the allocated buffer before it realizes it is dealing with + ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a buffer overread. */ + pSample->n = sqlite3_column_bytes(pStmt, 4); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + if( pSample->p==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; + } + memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + pIdx->nSample++; + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) initAvgEq(pPrevIdx); + return rc; +} + +/* +** Load content from the sqlite_stat4 and sqlite_stat3 tables into +** the Index.aSample[] arrays of all indices. +*/ +static int loadStat4(sqlite3 *db, const char *zDb){ + int rc = SQLITE_OK; /* Result codes from subroutines */ + + assert( db->lookaside.bEnabled==0 ); + if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ + rc = loadStatTbl(db, 0, + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", + zDb + ); + } + + if( rc==SQLITE_OK && sqlite3FindTable(db, "sqlite_stat3", zDb) ){ + rc = loadStatTbl(db, 1, + "SELECT idx,count(*) FROM %Q.sqlite_stat3 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sqlite_record(sample) FROM %Q.sqlite_stat3", + zDb + ); + } + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** Load the content of the sqlite_stat1 and sqlite_stat3/4 tables. The ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat2 are used to populate the +** arrays. The contents of sqlite_stat3/4 are used to populate the ** Index.aSample[] arrays. ** ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT2 was defined -** during compilation and the sqlite_stat2 table is present, no data is +** is returned. In this case, even if SQLITE_ENABLE_STAT3/4 was defined +** during compilation and the sqlite_stat3/4 table is present, no data is ** read from it. ** -** If SQLITE_ENABLE_STAT2 was defined during compilation and the -** sqlite_stat2 table is not present in the database, SQLITE_ERROR is +** If SQLITE_ENABLE_STAT3/4 was defined during compilation and the +** sqlite_stat4 table is not present in the database, SQLITE_ERROR is ** returned. However, in this case, data is read from the sqlite_stat1 ** table (if it is present) before returning. ** @@ -76417,8 +87664,10 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; +#endif } /* Check to make sure the sqlite_stat1 table exists */ @@ -76430,7 +87679,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load new statistics out of the sqlite_stat1 table */ zSql = sqlite3MPrintf(db, - "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ @@ -76439,78 +87688,18 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ } - /* Load the statistics from the sqlite_stat2 table. */ -#ifdef SQLITE_ENABLE_STAT2 - if( rc==SQLITE_OK && !sqlite3FindTable(db, "sqlite_stat2", sInfo.zDatabase) ){ - rc = SQLITE_ERROR; - } + /* Load the statistics from the sqlite_stat4 table. */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - - zSql = sqlite3MPrintf(db, - "SELECT idx,sampleno,sample FROM %Q.sqlite_stat2", sInfo.zDatabase); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - sqlite3DbFree(db, zSql); - } - - if( rc==SQLITE_OK ){ - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - - zIndex = (char *)sqlite3_column_text(pStmt, 0); - pIdx = zIndex ? sqlite3FindIndex(db, zIndex, sInfo.zDatabase) : 0; - if( pIdx ){ - int iSample = sqlite3_column_int(pStmt, 1); - if( iSample=0 ){ - int eType = sqlite3_column_type(pStmt, 2); - - if( pIdx->aSample==0 ){ - static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES; - pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz); - if( pIdx->aSample==0 ){ - db->mallocFailed = 1; - break; - } - memset(pIdx->aSample, 0, sz); - } - - assert( pIdx->aSample ); - { - IndexSample *pSample = &pIdx->aSample[iSample]; - pSample->eType = (u8)eType; - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - pSample->u.r = sqlite3_column_double(pStmt, 2); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const char *z = (const char *)( - (eType==SQLITE_BLOB) ? - sqlite3_column_blob(pStmt, 2): - sqlite3_column_text(pStmt, 2) - ); - int n = sqlite3_column_bytes(pStmt, 2); - if( n>24 ){ - n = 24; - } - pSample->nByte = (u8)n; - if( n < 1){ - pSample->u.z = 0; - }else{ - pSample->u.z = sqlite3DbStrNDup(0, z, n); - if( pSample->u.z==0 ){ - db->mallocFailed = 1; - break; - } - } - } - } - } - } - } - rc = sqlite3_finalize(pStmt); - } + int lookasideEnabled = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; + rc = loadStat4(db, sInfo.zDatabase); + db->lookaside.bEnabled = lookasideEnabled; + } + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3_free(pIdx->aiRowEst); + pIdx->aiRowEst = 0; } #endif @@ -76564,10 +87753,6 @@ static int resolveAttachExpr(NameContext *pName, Expr *pExpr) if( pExpr ){ if( pExpr->op!=TK_ID ){ rc = sqlite3ResolveExprNames(pName, pExpr); - if( rc==SQLITE_OK && !sqlite3ExprIsConstant(pExpr) ){ - sqlite3ErrorMsg(pName->pParse, "invalid name: \"%s\"", pExpr->u.zToken); - return SQLITE_ERROR; - } }else{ pExpr->op = TK_STRING; } @@ -76635,7 +87820,7 @@ static void attachFunc( } } - /* Allocate the new entry in the db->aDb[] array and initialise the schema + /* Allocate the new entry in the db->aDb[] array and initialize the schema ** hash tables. */ if( db->aDb==db->aDbStatic ){ @@ -76652,7 +87837,7 @@ static void attachFunc( /* Open the database file. If the btree is successfully opened, use ** it to obtain the database schema. At this point the schema may - ** or may not be initialised. + ** or may not be initialized. */ flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); @@ -76684,6 +87869,9 @@ static void attachFunc( sqlite3PagerLockingMode(pPager, db->dfltLockMode); sqlite3BtreeSecureDelete(aNew->pBt, sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); +#endif } aNew->safety_level = 3; aNew->zName = sqlite3DbStrDup(db, zName); @@ -76734,6 +87922,15 @@ static void attachFunc( rc = sqlite3Init(db, &zErrDyn); sqlite3BtreeLeaveAll(db); } +#ifdef SQLITE_USER_AUTHENTICATION + if( rc==SQLITE_OK ){ + u8 newAuth = 0; + rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); + if( newAuthauth.authLevel ){ + rc = SQLITE_AUTH_USER; + } + } +#endif if( rc ){ int iDb = db->nDb - 1; assert( iDb>=2 ); @@ -76742,7 +87939,7 @@ static void attachFunc( db->aDb[iDb].pBt = 0; db->aDb[iDb].pSchema = 0; } - sqlite3ResetInternalSchema(db, -1); + sqlite3ResetAllSchemasOfConnection(db); db->nDb = iDb; if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ db->mallocFailed = 1; @@ -76814,7 +88011,7 @@ static void detachFunc( sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; - sqlite3ResetInternalSchema(db, -1); + sqlite3ResetAllSchemasOfConnection(db); return; detach_error: @@ -76902,8 +88099,7 @@ attach_end: SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ static const FuncDef detach_func = { 1, /* nArg */ - SQLITE_UTF8, /* iPrefEnc */ - 0, /* flags */ + SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ detachFunc, /* xFunc */ @@ -76924,8 +88120,7 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ static const FuncDef attach_func = { 3, /* nArg */ - SQLITE_UTF8, /* iPrefEnc */ - 0, /* flags */ + SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ attachFunc, /* xFunc */ @@ -76942,11 +88137,8 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p /* ** Initialize a DbFixer structure. This routine must be called prior ** to passing the structure to one of the sqliteFixAAAA() routines below. -** -** The return value indicates whether or not fixation is required. TRUE -** means we do need to fix the database references, FALSE means we do not. */ -SQLITE_PRIVATE int sqlite3FixInit( +SQLITE_PRIVATE void sqlite3FixInit( DbFixer *pFix, /* The fixer to be initialized */ Parse *pParse, /* Error messages will be written here */ int iDb, /* This is the database that must be used */ @@ -76955,14 +88147,14 @@ SQLITE_PRIVATE int sqlite3FixInit( ){ sqlite3 *db; - if( NEVER(iDb<0) || iDb==1 ) return 0; db = pParse->db; assert( db->nDb>iDb ); pFix->pParse = pParse; pFix->zDb = db->aDb[iDb].zName; + pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; - return 1; + pFix->bVarOnly = (iDb==1); } /* @@ -76990,13 +88182,16 @@ SQLITE_PRIVATE int sqlite3FixSrcList( if( NEVER(pList==0) ) return 0; zDb = pFix->zDb; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->zDatabase==0 ){ - pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ - sqlite3ErrorMsg(pFix->pParse, - "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); - return 1; + if( pFix->bVarOnly==0 ){ + if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ + sqlite3ErrorMsg(pFix->pParse, + "%s %T cannot reference objects in database %s", + pFix->zType, pFix->pName, pItem->zDatabase); + return 1; + } + sqlite3DbFree(pFix->pParse->db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; @@ -77020,9 +88215,21 @@ SQLITE_PRIVATE int sqlite3FixSelect( if( sqlite3FixExpr(pFix, pSelect->pWhere) ){ return 1; } + if( sqlite3FixExprList(pFix, pSelect->pGroupBy) ){ + return 1; + } if( sqlite3FixExpr(pFix, pSelect->pHaving) ){ return 1; } + if( sqlite3FixExprList(pFix, pSelect->pOrderBy) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pSelect->pOffset) ){ + return 1; + } pSelect = pSelect->pPrior; } return 0; @@ -77032,7 +88239,15 @@ SQLITE_PRIVATE int sqlite3FixExpr( Expr *pExpr /* The expression to be fixed to one database */ ){ while( pExpr ){ - if( ExprHasAnyProperty(pExpr, EP_TokenOnly) ) break; + if( pExpr->op==TK_VARIABLE ){ + if( pFix->pParse->db->init.busy ){ + pExpr->op = TK_NULL; + }else{ + sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); + return 1; + } + } + if( ExprHasProperty(pExpr, EP_TokenOnly) ) break; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; }else{ @@ -77158,7 +88373,7 @@ SQLITE_API int sqlite3_set_authorizer( void *pArg ){ sqlite3_mutex_enter(db->mutex); - db->xAuth = xAuth; + db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; sqlite3ExpirePreparedStatements(db); sqlite3_mutex_leave(db->mutex); @@ -77193,7 +88408,11 @@ SQLITE_PRIVATE int sqlite3AuthReadCol( char *zDb = db->aDb[iDb].zName; /* Name of attached database */ int rc; /* Auth callback return code */ - rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ if( db->nDb>2 || iDb!=0 ){ sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); @@ -77293,7 +88512,11 @@ SQLITE_PRIVATE int sqlite3AuthCheck( if( db->xAuth==0 ){ return SQLITE_OK; } - rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext); + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ sqlite3ErrorMsg(pParse, "not authorized"); pParse->rc = SQLITE_AUTH; @@ -77449,6 +88672,19 @@ static void codeTableLocks(Parse *pParse){ #define codeTableLocks(x) #endif +/* +** Return TRUE if the given yDbMask object is empty - if it contains no +** 1 bits. This routine is used by the DbMaskAllZero() and DbMaskNotZero() +** macros when SQLITE_MAX_ATTACHED is greater than 30. +*/ +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ + int i; + for(i=0; ipToplevel==0 ); db = pParse->db; if( db->mallocFailed ) return; if( pParse->nested ) return; @@ -77475,38 +88712,50 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ + while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); +#if SQLITE_USER_AUTHENTICATION + if( pParse->nTableLock>0 && db->init.busy==0 ){ + sqlite3UserAuthInit(db); + if( db->auth.authLevelrc = SQLITE_AUTH_USER; + sqlite3ErrorMsg(pParse, "user not authenticated"); + return; + } + } +#endif + /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are ** set for each database that is used. Generate code to start a ** transaction on each used database and to verify the schema cookie ** on each used database. */ - if( pParse->cookieGoto>0 ){ - yDbMask mask; - int iDb; - sqlite3VdbeJumpHere(v, pParse->cookieGoto-1); - for(iDb=0, mask=1; iDbnDb; mask<<=1, iDb++){ - if( (mask & pParse->cookieMask)==0 ) continue; + if( db->mallocFailed==0 + && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) + ){ + int iDb, i; + assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + for(iDb=0; iDbnDb; iDb++){ + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v,OP_Transaction, iDb, (mask & pParse->writeMask)!=0); - if( db->init.busy==0 ){ - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - sqlite3VdbeAddOp3(v, OP_VerifyCookie, - iDb, pParse->cookieValue[iDb], - db->aDb[iDb].pSchema->iGeneration); - } + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pParse->cookieValue[iDb], /* P3 */ + db->aDb[iDb].pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); } #ifndef SQLITE_OMIT_VIRTUALTABLE - { - int i; - for(i=0; inVtabLock; i++){ - char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); - sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); - } - pParse->nVtabLock = 0; + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); } + pParse->nVtabLock = 0; #endif /* Once all the cookies have been verified and transactions opened, @@ -77519,8 +88768,17 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ */ sqlite3AutoincrementBegin(pParse); + /* Code constant expressions that where factored out of inner loops */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); + } + } + /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pParse->cookieGoto); + sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); } } @@ -77528,10 +88786,6 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ /* Get the VDBE program ready for execution */ if( v && ALWAYS(pParse->nErr==0) && !db->mallocFailed ){ -#ifdef SQLITE_DEBUG - FILE *trace = (db->flags & SQLITE_VdbeTrace)!=0 ? stdout : 0; - sqlite3VdbeTrace(v, trace); -#endif assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ @@ -77546,8 +88800,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ pParse->nMem = 0; pParse->nSet = 0; pParse->nVar = 0; - pParse->cookieMask = 0; - pParse->cookieGoto = 0; + DbMaskZero(pParse->cookieMask); } /* @@ -77588,6 +88841,16 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ pParse->nested--; } +#if SQLITE_USER_AUTHENTICATION +/* +** Return TRUE if zTable is the name of the system table that stores the +** list of users and their access credentials. +*/ +SQLITE_PRIVATE int sqlite3UserAuthTable(const char *zTable){ + return sqlite3_stricmp(zTable, "sqlite_user")==0; +} +#endif + /* ** Locate the in-memory structure that describes a particular database ** table given the name of that table and (optionally) the name of the @@ -77603,16 +88866,21 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - int nName; assert( zName!=0 ); - nName = sqlite3Strlen30(zName); /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); +#if SQLITE_USER_AUTHENTICATION + /* Only the admin user is allowed to know that the sqlite_user table + ** exists */ + if( db->auth.authLevelnDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); if( p ) break; } return p; @@ -77652,9 +88920,40 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( } pParse->checkSchema = 1; } +#if SQLITE_USER_AUTHENICATION + else if( pParse->db->auth.authLevelpSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +SQLITE_PRIVATE Table *sqlite3LocateTableItem( + Parse *pParse, + int isView, + struct SrcList_item *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, isView, p->zName, zDb); +} + /* ** Locate the in-memory structure that describes ** a particular index given the name of that index @@ -77670,7 +88969,6 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ Index *p = 0; int i; - int nName = sqlite3Strlen30(zName); /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ @@ -77679,7 +88977,7 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha assert( pSchema ); if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&pSchema->idxHash, zName, nName); + p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; } return p; @@ -77692,7 +88990,13 @@ static void freeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE sqlite3DeleteIndexSamples(db, p); #endif + if( db==0 || db->pnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo); + sqlite3ExprDelete(db, p->pPartIdxWhere); sqlite3DbFree(db, p->zColAff); + if( p->isResized ) sqlite3DbFree(db, p->azColl); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3_free(p->aiRowEst); +#endif sqlite3DbFree(db, p); } @@ -77704,13 +89008,11 @@ static void freeIndex(sqlite3 *db, Index *p){ */ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ Index *pIndex; - int len; Hash *pHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &db->aDb[iDb].pSchema->idxHash; - len = sqlite3Strlen30(zIdxName); - pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0); + pIndex = sqlite3HashInsert(pHash, zIdxName, 0); if( ALWAYS(pIndex) ){ if( pIndex->pTable->pIndex==pIndex ){ pIndex->pTable->pIndex = pIndex->pNext; @@ -77730,58 +89032,15 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char } /* -** Erase all schema information from the in-memory hash tables of -** a single database. This routine is called to reclaim memory -** before the database closes. It is also called during a rollback -** if there were schema changes during the transaction or if a -** schema-cookie mismatch occurs. +** Look through the list of open database files in db->aDb[] and if +** any have been closed, remove them from the list. Reallocate the +** db->aDb[] structure to a smaller size, if possible. ** -** If iDb<0 then reset the internal schema tables for all database -** files. If iDb>=0 then reset the internal schema for only the -** single file indicated. +** Entry 0 (the "main" database) and entry 1 (the "temp" database) +** are never candidates for being collapsed. */ -SQLITE_PRIVATE void sqlite3ResetInternalSchema(sqlite3 *db, int iDb){ +SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3 *db){ int i, j; - assert( iDbnDb ); - - if( iDb>=0 ){ - /* Case 1: Reset the single schema identified by iDb */ - Db *pDb = &db->aDb[iDb]; - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - assert( pDb->pSchema!=0 ); - sqlite3SchemaClear(pDb->pSchema); - - /* If any database other than TEMP is reset, then also reset TEMP - ** since TEMP might be holding triggers that reference tables in the - ** other database. - */ - if( iDb!=1 ){ - pDb = &db->aDb[1]; - assert( pDb->pSchema!=0 ); - sqlite3SchemaClear(pDb->pSchema); - } - return; - } - /* Case 2 (from here to the end): Reset all schemas for all attached - ** databases. */ - assert( iDb<0 ); - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Db *pDb = &db->aDb[i]; - if( pDb->pSchema ){ - sqlite3SchemaClear(pDb->pSchema); - } - } - db->flags &= ~SQLITE_InternChanges; - sqlite3VtabUnlockList(db); - sqlite3BtreeLeaveAll(db); - - /* If one or more of the auxiliary database files has been closed, - ** then remove them from the auxiliary database list. We take the - ** opportunity to do this here since we have just deleted all of the - ** schema hash tables and therefore do not have to make any changes - ** to any of those tables. - */ for(i=j=2; inDb; i++){ struct Db *pDb = &db->aDb[i]; if( pDb->pBt==0 ){ @@ -77803,6 +89062,51 @@ SQLITE_PRIVATE void sqlite3ResetInternalSchema(sqlite3 *db, int iDb){ } } +/* +** Reset the schema for the database at index iDb. Also reset the +** TEMP schema. +*/ +SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ + Db *pDb; + assert( iDbnDb ); + + /* Case 1: Reset the single schema identified by iDb */ + pDb = &db->aDb[iDb]; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( pDb->pSchema!=0 ); + sqlite3SchemaClear(pDb->pSchema); + + /* If any database other than TEMP is reset, then also reset TEMP + ** since TEMP might be holding triggers that reference tables in the + ** other database. + */ + if( iDb!=1 ){ + pDb = &db->aDb[1]; + assert( pDb->pSchema!=0 ); + sqlite3SchemaClear(pDb->pSchema); + } + return; +} + +/* +** Erase all schema information from all attached databases (including +** "main" and "temp") for a single database connection. +*/ +SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pSchema ){ + sqlite3SchemaClear(pDb->pSchema); + } + } + db->flags &= ~SQLITE_InternChanges; + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); + sqlite3CollapseDatabaseArray(db); +} + /* ** This routine is called when a commit occurs. */ @@ -77838,9 +89142,16 @@ static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ ** the table data structure from the hash table. But it does destroy ** memory structures of the indices and foreign keys associated with ** the table. +** +** The db parameter is optional. It is needed if the Table object +** contains lookaside memory. (Table objects in the schema do not use +** lookaside memory, but some ephemeral Table objects do.) Or the +** db parameter can be used with db->pnBytesFreed to measure the memory +** used by the Table object. */ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; + TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ assert( !pTable || pTable->nRef>0 ); @@ -77848,6 +89159,12 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ if( !pTable ) return; if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + /* Record the number of outstanding lookaside allocations in schema Tables + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. */ + TESTONLY( nLookaside = (db && (pTable->tabFlags & TF_Ephemeral)==0) ? + db->lookaside.nOut : 0 ); + /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; @@ -77855,7 +89172,7 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ if( !db || db->pnBytesFreed==0 ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( - &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + &pIndex->pSchema->idxHash, zName, 0 ); assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); @@ -77873,12 +89190,15 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK - sqlite3ExprDelete(db, pTable->pCheck); + sqlite3ExprListDelete(db, pTable->pCheck); #endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif sqlite3DbFree(db, pTable); + + /* Verify that no lookaside memory was used by schema tables */ + assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); } /* @@ -77895,8 +89215,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ pDb = &db->aDb[iDb]; - p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, - sqlite3Strlen30(zTabName),0); + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); sqlite3DeleteTable(db, p); db->flags |= SQLITE_InternChanges; } @@ -77932,8 +89251,7 @@ SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *p, int iDb){ Vdbe *v = sqlite3GetVdbe(p); sqlite3TableLock(p, iDb, MASTER_ROOT, 1, SCHEMA_TABLE(iDb)); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 0, MASTER_ROOT, iDb); - sqlite3VdbeChangeP4(v, -1, (char *)5, P4_INT32); /* 5 column table */ + sqlite3VdbeAddOp4Int(v, OP_OpenWrite, 0, MASTER_ROOT, iDb, 5); if( p->nTab==0 ){ p->nTab = 1; } @@ -78038,6 +89356,27 @@ SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ return SQLITE_OK; } +/* +** Return the PRIMARY KEY index of a table +*/ +SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table *pTab){ + Index *p; + for(p=pTab->pIndex; p && !IsPrimaryKeyIndex(p); p=p->pNext){} + return p; +} + +/* +** Return the column of index pIdx that corresponds to table +** column iCol. Return -1 if not found. +*/ +SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index *pIdx, i16 iCol){ + int i; + for(i=0; inColumn; i++){ + if( iCol==pIdx->aiColumn[i] ) return i; + } + return -1; +} + /* ** Begin constructing a new table representation in memory. This is ** the first of several action routines that get called in response @@ -78170,7 +89509,7 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1000000; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -78213,7 +89552,7 @@ SQLITE_PRIVATE void sqlite3StartTable( reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); + j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); @@ -78237,7 +89576,7 @@ SQLITE_PRIVATE void sqlite3StartTable( }else #endif { - sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); + pParse->addrCrTab = sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); @@ -78317,6 +89656,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ ** be called next to set pCol->affinity correctly. */ pCol->affinity = SQLITE_AFF_NONE; + pCol->szEst = 1; p->nCol++; } @@ -78358,15 +89698,18 @@ SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ ** If none of the substrings in the above table are found, ** SQLITE_AFF_NUMERIC is returned. */ -SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn){ +SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ u32 h = 0; char aff = SQLITE_AFF_NUMERIC; + const char *zChar = 0; - if( zIn ) while( zIn[0] ){ + if( zIn==0 ) return aff; + while( zIn[0] ){ h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; zIn++; if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ - aff = SQLITE_AFF_TEXT; + aff = SQLITE_AFF_TEXT; + zChar = zIn; }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){ /* CLOB */ aff = SQLITE_AFF_TEXT; }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){ /* TEXT */ @@ -78374,6 +89717,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn){ }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ aff = SQLITE_AFF_NONE; + if( zIn[0]=='(' ) zChar = zIn; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ && aff==SQLITE_AFF_NUMERIC ){ @@ -78391,6 +89735,28 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn){ } } + /* If pszEst is not NULL, store an estimate of the field size. The + ** estimate is scaled so that the size of an integer is 1. */ + if( pszEst ){ + *pszEst = 1; /* default size is approx 4 bytes */ + if( aff255 ) v = 255; + *pszEst = v; /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */ + break; + } + zChar++; + } + }else{ + *pszEst = 5; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ + } + } + } return aff; } @@ -78412,7 +89778,7 @@ SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){ pCol = &p->aCol[p->nCol-1]; assert( pCol->zType==0 ); pCol->zType = sqlite3NameFromToken(pParse->db, pType); - pCol->affinity = sqlite3AffinityType(pCol->zType); + pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); } /* @@ -78432,7 +89798,7 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ p = pParse->pNewTable; if( p!=0 ){ pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ + if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", pCol->zName); }else{ @@ -78478,6 +89844,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( Table *pTab = pParse->pNewTable; char *zType = 0; int iCol = -1, i; + int nTerm; if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite3ErrorMsg(pParse, @@ -78487,39 +89854,44 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; + nTerm = 1; }else{ - for(i=0; inExpr; i++){ + nTerm = pList->nExpr; + for(i=0; inCol; iCol++){ if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; break; } } - if( iColnCol ){ - pTab->aCol[iCol].isPrimKey = 1; - } } - if( pList->nExpr>1 ) iCol = -1; } - if( iCol>=0 && iColnCol ){ - zType = pTab->aCol[iCol].zType; - } - if( zType && sqlite3StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE_SO_ASC ){ + if( nTerm==1 + && zType && sqlite3StrICmp(zType, "INTEGER")==0 + && sortOrder==SQLITE_SO_ASC + ){ pTab->iPKey = iCol; pTab->keyConf = (u8)onError; assert( autoInc==0 || autoInc==1 ); pTab->tabFlags |= autoInc*TF_Autoincrement; + if( pList ) pParse->iPkSortOrder = pList->a[0].sortOrder; }else if( autoInc ){ #ifndef SQLITE_OMIT_AUTOINCREMENT sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " "INTEGER PRIMARY KEY"); #endif }else{ + Vdbe *v = pParse->pVdbe; Index *p; - p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0); + if( v ) pParse->addrSkipPK = sqlite3VdbeAddOp0(v, OP_Noop); + p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0); if( p ){ - p->autoIndex = 2; + p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + if( v ) sqlite3VdbeJumpHere(v, pParse->addrSkipPK); } pList = 0; } @@ -78536,15 +89908,20 @@ SQLITE_PRIVATE void sqlite3AddCheckConstraint( Parse *pParse, /* Parsing context */ Expr *pCheckExpr /* The check expression */ ){ - sqlite3 *db = pParse->db; #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; - if( pTab && !IN_DECLARE_VTAB ){ - pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, pCheckExpr); + sqlite3 *db = pParse->db; + if( pTab && !IN_DECLARE_VTAB + && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) + ){ + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + } }else #endif { - sqlite3ExprDelete(db, pCheckExpr); + sqlite3ExprDelete(pParse->db, pCheckExpr); } } @@ -78566,6 +89943,7 @@ SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ if( sqlite3LocateCollSeq(pParse, zColl) ){ Index *pIdx; + sqlite3DbFree(db, p->aCol[i].zColl); p->aCol[i].zColl = zColl; /* If the column is declared as " PRIMARY KEY COLLATE ", @@ -78573,7 +89951,7 @@ SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ ** collation type was added. Correct this if it is the case. */ for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ - assert( pIdx->nColumn==1 ); + assert( pIdx->nKeyCol==1 ); if( pIdx->aiColumn[0]==i ){ pIdx->azColl[0] = p->aCol[i].zColl; } @@ -78611,10 +89989,7 @@ SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(db, enc, pColl, zName); - if( !pColl ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - } + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); } return pColl; @@ -78684,10 +90059,10 @@ static void identPut(char *z, int *pIdx, char *zSignedIdent){ for(j=0; zIdent[j]; j++){ if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; } - needQuote = sqlite3Isdigit(zIdent[0]) || sqlite3KeywordCode(zIdent, j)!=TK_ID; - if( !needQuote ){ - needQuote = zIdent[j]; - } + needQuote = sqlite3Isdigit(zIdent[0]) + || sqlite3KeywordCode(zIdent, j)!=TK_ID + || zIdent[j]!=0 + || j==0; if( needQuote ) z[i++] = '"'; for(j=0; zIdent[j]; j++){ @@ -78735,8 +90110,8 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ static const char * const azType[] = { - /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NONE */ "", + /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", /* SQLITE_AFF_REAL */ " REAL" @@ -78748,18 +90123,18 @@ static char *createTableStmt(sqlite3 *db, Table *p){ k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 ); - assert( pCol->affinity-SQLITE_AFF_TEXT < ArraySize(azType) ); - testcase( pCol->affinity==SQLITE_AFF_TEXT ); + assert( pCol->affinity-SQLITE_AFF_NONE >= 0 ); + assert( pCol->affinity-SQLITE_AFF_NONE < ArraySize(azType) ); testcase( pCol->affinity==SQLITE_AFF_NONE ); + testcase( pCol->affinity==SQLITE_AFF_TEXT ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; + zType = azType[pCol->affinity - SQLITE_AFF_NONE]; len = sqlite3Strlen30(zType); assert( pCol->affinity==SQLITE_AFF_NONE - || pCol->affinity==sqlite3AffinityType(zType) ); + || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; assert( k<=n ); @@ -78768,6 +90143,191 @@ static char *createTableStmt(sqlite3 *db, Table *p){ return zStmt; } +/* +** Resize an Index object to hold N columns total. Return SQLITE_OK +** on success and SQLITE_NOMEM on an OOM error. +*/ +static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ + char *zExtra; + int nByte; + if( pIdx->nColumn>=N ) return SQLITE_OK; + assert( pIdx->isResized==0 ); + nByte = (sizeof(char*) + sizeof(i16) + 1)*N; + zExtra = sqlite3DbMallocZero(db, nByte); + if( zExtra==0 ) return SQLITE_NOMEM; + memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); + pIdx->azColl = (char**)zExtra; + zExtra += sizeof(char*)*N; + memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); + pIdx->aiColumn = (i16*)zExtra; + zExtra += sizeof(i16)*N; + memcpy(zExtra, pIdx->aSortOrder, pIdx->nColumn); + pIdx->aSortOrder = (u8*)zExtra; + pIdx->nColumn = N; + pIdx->isResized = 1; + return SQLITE_OK; +} + +/* +** Estimate the total row width for a table. +*/ +static void estimateTableWidth(Table *pTab){ + unsigned wTable = 0; + const Column *pTabCol; + int i; + for(i=pTab->nCol, pTabCol=pTab->aCol; i>0; i--, pTabCol++){ + wTable += pTabCol->szEst; + } + if( pTab->iPKey<0 ) wTable++; + pTab->szTabRow = sqlite3LogEst(wTable*4); +} + +/* +** Estimate the average size of a row for an index. +*/ +static void estimateIndexWidth(Index *pIdx){ + unsigned wIndex = 0; + int i; + const Column *aCol = pIdx->pTable->aCol; + for(i=0; inColumn; i++){ + i16 x = pIdx->aiColumn[i]; + assert( xpTable->nCol ); + wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + } + pIdx->szIdxRow = sqlite3LogEst(wIndex*4); +} + +/* Return true if value x is found any of the first nCol entries of aiCol[] +*/ +static int hasColumn(const i16 *aiCol, int nCol, int x){ + while( nCol-- > 0 ) if( x==*(aiCol++) ) return 1; + return 0; +} + +/* +** This routine runs at the end of parsing a CREATE TABLE statement that +** has a WITHOUT ROWID clause. The job of this routine is to convert both +** internal schema data structures and the generated VDBE code so that they +** are appropriate for a WITHOUT ROWID table instead of a rowid table. +** Changes include: +** +** (1) Convert the OP_CreateTable into an OP_CreateIndex. There is +** no rowid btree for a WITHOUT ROWID. Instead, the canonical +** data storage is a covering index btree. +** (2) Bypass the creation of the sqlite_master table entry +** for the PRIMARY KEY as the primary key index is now +** identified by the sqlite_master table entry of the table itself. +** (3) Set the Index.tnum of the PRIMARY KEY Index object in the +** schema to the rootpage from the main table. +** (4) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (5) Add all table columns to the PRIMARY KEY Index object +** so that the PRIMARY KEY is a covering index. The surplus +** columns are part of KeyInfo.nXField and are not used for +** sorting or lookup or uniqueness checks. +** (6) Replace the rowid tail on all automatically generated UNIQUE +** indices with the PRIMARY KEY columns. +*/ +static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ + Index *pIdx; + Index *pPk; + int nPk; + int i, j; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + + /* Convert the OP_CreateTable opcode that would normally create the + ** root-page for the table into an OP_CreateIndex opcode. The index + ** created will become the PRIMARY KEY index. + */ + if( pParse->addrCrTab ){ + assert( v ); + sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex; + } + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master + ** table entry. + */ + if( pParse->addrSkipPK ){ + assert( v ); + sqlite3VdbeGetOp(v, pParse->addrSkipPK)->opcode = OP_Goto; + } + + /* Locate the PRIMARY KEY index. Or, if this table was originally + ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. + */ + if( pTab->iPKey>=0 ){ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse, 0, 0); + if( pList==0 ) return; + pList->a[0].zName = sqlite3DbStrDup(pParse->db, + pTab->aCol[pTab->iPKey].zName); + pList->a[0].sortOrder = pParse->iPkSortOrder; + assert( pParse->pNewTable==pTab ); + pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); + if( pPk==0 ) return; + pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + pTab->iPKey = -1; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + } + pPk->isCovering = 1; + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + + /* Make sure every column of the PRIMARY KEY is NOT NULL */ + for(i=0; iaCol[pPk->aiColumn[i]].notNull = 1; + } + pPk->uniqNotNull = 1; + + /* The root page of the PRIMARY KEY is the table root page */ + pPk->tnum = pTab->tnum; + + /* Update the in-memory representation of all UNIQUE indices by converting + ** the final rowid column into one or more columns of the PRIMARY KEY. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int n; + if( IsPrimaryKeyIndex(pIdx) ) continue; + for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; + } + if( n==0 ){ + /* This index is a superset of the primary key */ + pIdx->nColumn = pIdx->nKeyCol; + continue; + } + if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; + for(i=0, j=pIdx->nKeyCol; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ){ + pIdx->aiColumn[j] = pPk->aiColumn[i]; + pIdx->azColl[j] = pPk->azColl[i]; + j++; + } + } + assert( pIdx->nColumn>=pIdx->nKeyCol+n ); + assert( pIdx->nColumn>=j ); + } + + /* Add all table columns to the PRIMARY KEY index + */ + if( nPknCol ){ + if( resizeIndexObject(db, pPk, pTab->nCol) ) return; + for(i=0, j=nPk; inCol; i++){ + if( !hasColumn(pPk->aiColumn, j, i) ){ + assert( jnColumn ); + pPk->aiColumn[j] = i; + pPk->azColl[j] = "BINARY"; + j++; + } + } + assert( pPk->nColumn==j ); + assert( pTab->nCol==j ); + }else{ + pPk->nColumn = pTab->nCol; + } +} + /* ** This routine is called to report the final ")" that terminates ** a CREATE TABLE statement. @@ -78791,12 +90351,14 @@ static char *createTableStmt(sqlite3 *db, Table *p){ SQLITE_PRIVATE void sqlite3EndTable( Parse *pParse, /* Parse context */ Token *pCons, /* The ',' token after the last column defn. */ - Token *pEnd, /* The final ')' token in the CREATE TABLE */ + Token *pEnd, /* The ')' before options in the CREATE TABLE */ + u8 tabOpts, /* Extra table options. Usually 0. */ Select *pSelect /* Select from a "CREATE ... AS SELECT" */ ){ - Table *p; - sqlite3 *db = pParse->db; - int iDb; + Table *p; /* The new table */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* Database in which the table lives */ + Index *pIdx; /* An implied index of the table */ if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ return; @@ -78806,30 +90368,6 @@ SQLITE_PRIVATE void sqlite3EndTable( assert( !db->init.busy || !pSelect ); - iDb = sqlite3SchemaToIndex(db, p->pSchema); - -#ifndef SQLITE_OMIT_CHECK - /* Resolve names in all CHECK constraint expressions. - */ - if( p->pCheck ){ - SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ - NameContext sNC; /* Name context for pParse->pNewTable */ - - memset(&sNC, 0, sizeof(sNC)); - memset(&sSrc, 0, sizeof(sSrc)); - sSrc.nSrc = 1; - sSrc.a[0].zName = p->zName; - sSrc.a[0].pTab = p; - sSrc.a[0].iCursor = -1; - sNC.pParse = pParse; - sNC.pSrcList = &sSrc; - sNC.isCheck = 1; - if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ - return; - } - } -#endif /* !defined(SQLITE_OMIT_CHECK) */ - /* If the db->init.busy is 1 it means we are reading the SQL off the ** "sqlite_master" or "sqlite_temp_master" table on the disk. ** So do not write to the disk again. Extract the root page number @@ -78840,6 +90378,37 @@ SQLITE_PRIVATE void sqlite3EndTable( p->tnum = db->init.newTnum; } + /* Special processing for WITHOUT ROWID Tables */ + if( tabOpts & TF_WithoutRowid ){ + if( (p->tabFlags & TF_Autoincrement) ){ + sqlite3ErrorMsg(pParse, + "AUTOINCREMENT not allowed on WITHOUT ROWID tables"); + return; + } + if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ + sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); + }else{ + p->tabFlags |= TF_WithoutRowid; + convertToWithoutRowidTable(pParse, p); + } + } + + iDb = sqlite3SchemaToIndex(db, p->pSchema); + +#ifndef SQLITE_OMIT_CHECK + /* Resolve names in all CHECK constraint expressions. + */ + if( p->pCheck ){ + sqlite3ResolveSelfReference(pParse, p, NC_IsCheck, 0, p->pCheck); + } +#endif /* !defined(SQLITE_OMIT_CHECK) */ + + /* Estimate the average row size for the table and for all implied indices */ + estimateTableWidth(p); + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + estimateIndexWidth(pIdx); + } + /* If not initializing, then create a record for the new table ** in the SQLITE_MASTER table of the database. ** @@ -78892,7 +90461,7 @@ SQLITE_PRIVATE void sqlite3EndTable( assert(pParse->nTab==1); sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); - sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); pParse->nTab = 2; sqlite3SelectDestInit(&dest, SRT_Table, 1); sqlite3Select(pParse, pSelect, &dest); @@ -78913,7 +90482,9 @@ SQLITE_PRIVATE void sqlite3EndTable( if( pSelect ){ zStmt = createTableStmt(db, p); }else{ - n = (int)(pEnd->z - pParse->sNameToken.z) + 1; + Token *pEnd2 = tabOpts ? &pParse->sLastToken : pEnd; + n = (int)(pEnd2->z - pParse->sNameToken.z); + if( pEnd2->z[0]!=';' ) n += pEnd2->n; zStmt = sqlite3MPrintf(db, "CREATE %s %.*s", zType2, n, pParse->sNameToken.z ); @@ -78956,7 +90527,7 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Reparse everything to update our internal data structures */ sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "tbl_name='%q'", p->zName)); + sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName)); } @@ -78966,15 +90537,13 @@ SQLITE_PRIVATE void sqlite3EndTable( Table *pOld; Schema *pSchema = p->pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, - sqlite3Strlen30(p->zName),p); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ db->mallocFailed = 1; return; } pParse->pNewTable = 0; - db->nTable++; db->flags |= SQLITE_InternChanges; #ifndef SQLITE_OMIT_ALTERTABLE @@ -79010,7 +90579,7 @@ SQLITE_PRIVATE void sqlite3CreateView( const char *z; Token sEnd; DbFixer sFix; - Token *pName; + Token *pName = 0; int iDb; sqlite3 *db = pParse->db; @@ -79027,9 +90596,8 @@ SQLITE_PRIVATE void sqlite3CreateView( } sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); - if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName) - && sqlite3FixSelect(&sFix, pSelect) - ){ + sqlite3FixInit(&sFix, pParse, iDb, "view", pName); + if( sqlite3FixSelect(&sFix, pSelect) ){ sqlite3SelectDelete(db, pSelect); return; } @@ -79063,7 +90631,7 @@ SQLITE_PRIVATE void sqlite3CreateView( sEnd.n = 1; /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ - sqlite3EndTable(pParse, 0, &sEnd, 0); + sqlite3EndTable(pParse, 0, &sEnd, 0, 0); return; } #endif /* SQLITE_OMIT_VIEW */ @@ -79080,7 +90648,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + sqlite3_xauth xAuth; /* Saved xAuth pointer */ assert( pTable ); @@ -79151,7 +90719,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pSelTab->aCol = 0; sqlite3DeleteTable(db, pSelTab); assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->flags |= DB_UnresetViews; + pTable->pSchema->schemaFlags |= DB_UnresetViews; }else{ pTable->nCol = 0; nErr++; @@ -79309,6 +90877,7 @@ static void destroyTable(Parse *pParse, Table *pTab){ return; }else{ int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 && iDbdb->nDb ); destroyRootPage(pParse, iLargest, iDb); iDestroyed = iLargest; } @@ -79316,6 +90885,100 @@ static void destroyTable(Parse *pParse, Table *pTab){ #endif } +/* +** Remove entries from the sqlite_statN tables (for N in (1,2,3)) +** after a DROP INDEX or DROP TABLE command. +*/ +static void sqlite3ClearStatTables( + Parse *pParse, /* The parsing context */ + int iDb, /* The database number */ + const char *zType, /* "idx" or "tbl" */ + const char *zName /* Name of index or table */ +){ + int i; + const char *zDbName = pParse->db->aDb[iDb].zName; + for(i=1; i<=4; i++){ + char zTab[24]; + sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); + if( sqlite3FindTable(pParse->db, zTab, zDbName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE %s=%Q", + zDbName, zTab, zType, zName + ); + } + } +} + +/* +** Generate code to drop a table. +*/ +SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ + Vdbe *v; + sqlite3 *db = pParse->db; + Trigger *pTrigger; + Db *pDb = &db->aDb[iDb]; + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* Drop all triggers associated with the table being dropped. Code + ** is generated to remove entries from sqlite_master and/or + ** sqlite_temp_master if required. + */ + pTrigger = sqlite3TriggerList(pParse, pTab); + while( pTrigger ){ + assert( pTrigger->pSchema==pTab->pSchema || + pTrigger->pSchema==db->aDb[1].pSchema ); + sqlite3DropTriggerPtr(pParse, pTrigger); + pTrigger = pTrigger->pNext; + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Remove any entries of the sqlite_sequence table associated with + ** the table being dropped. This is done before the table is dropped + ** at the btree level, in case the sqlite_sequence table needs to + ** move as a result of the drop (can happen in auto-vacuum mode). + */ + if( pTab->tabFlags & TF_Autoincrement ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", + pDb->zName, pTab->zName + ); + } +#endif + + /* Drop all SQLITE_MASTER table and index entries that refer to the + ** table. The program name loops through the master table and deletes + ** every row that refers to a table of the same name as the one being + ** dropped. Triggers are handled separately because a trigger can be + ** created in the temp database that refers to a table in another + ** database. + */ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", + pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + if( !isView && !IsVirtual(pTab) ){ + destroyTable(pParse, pTab); + } + + /* Remove the table entry from SQLite's internal schema and modify + ** the schema cookie. + */ + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + } + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + sqlite3ChangeCookie(pParse, iDb); + sqliteViewResetAll(db, iDb); +} + /* ** This routine is called to do the work of a DROP TABLE statement. ** pName is the name of the table to be dropped. @@ -79332,8 +90995,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); if( noErr ) db->suppressErr++; - pTab = sqlite3LocateTable(pParse, isView, - pName->a[0].zName, pName->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; if( pTab==0 ){ @@ -79384,7 +91046,8 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } } #endif - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && sqlite3StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } @@ -79408,75 +91071,11 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, */ v = sqlite3GetVdbe(pParse); if( v ){ - Trigger *pTrigger; - Db *pDb = &db->aDb[iDb]; sqlite3BeginWriteOperation(pParse, 1, iDb); - -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp0(v, OP_VBegin); - } -#endif + sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); sqlite3FkDropTable(pParse, pName, pTab); - - /* Drop all triggers associated with the table being dropped. Code - ** is generated to remove entries from sqlite_master and/or - ** sqlite_temp_master if required. - */ - pTrigger = sqlite3TriggerList(pParse, pTab); - while( pTrigger ){ - assert( pTrigger->pSchema==pTab->pSchema || - pTrigger->pSchema==db->aDb[1].pSchema ); - sqlite3DropTriggerPtr(pParse, pTrigger); - pTrigger = pTrigger->pNext; - } - -#ifndef SQLITE_OMIT_AUTOINCREMENT - /* Remove any entries of the sqlite_sequence table associated with - ** the table being dropped. This is done before the table is dropped - ** at the btree level, in case the sqlite_sequence table needs to - ** move as a result of the drop (can happen in auto-vacuum mode). - */ - if( pTab->tabFlags & TF_Autoincrement ){ - sqlite3NestedParse(pParse, - "DELETE FROM %s.sqlite_sequence WHERE name=%Q", - pDb->zName, pTab->zName - ); - } -#endif - - /* Drop all SQLITE_MASTER table and index entries that refer to the - ** table. The program name loops through the master table and deletes - ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be - ** created in the temp database that refers to a table in another - ** database. - */ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); - - /* Drop any statistics from the sqlite_stat1 table, if it exists */ - if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.sqlite_stat1 WHERE tbl=%Q", pDb->zName, pTab->zName - ); - } - - if( !isView && !IsVirtual(pTab) ){ - destroyTable(pParse, pTab); - } - - /* Remove the table entry from SQLite's internal schema and modify - ** the schema cookie. - */ - if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); - } - sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); - sqlite3ChangeCookie(pParse, iDb); + sqlite3CodeDropTable(pParse, pTab, iDb, isView); } - sqliteViewResetAll(db, iDb); exit_drop_table: sqlite3SrcListDelete(db, pName); @@ -79487,8 +91086,8 @@ exit_drop_table: ** currently under construction. pFromCol determines which columns ** in the current table point to the foreign key. If pFromCol==0 then ** connect the key to the last column inserted. pTo is the name of -** the table referred to. pToCol is a list of tables in the other -** pTo table that the foreign key points to. flags contains all +** the table referred to (a.k.a the "parent" table). pToCol is a list +** of tables in the parent pTo table. flags contains all ** information about the conflict resolution algorithms specified ** in the ON DELETE, ON UPDATE and ON INSERT clauses. ** @@ -79588,7 +91187,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, - pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey + pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ db->mallocFailed = 1; @@ -79644,12 +91243,14 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ Table *pTab = pIndex->pTable; /* The table that is indexed */ int iTab = pParse->nTab++; /* Btree cursor used for pTab */ int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ + int iSorter; /* Cursor opened by OpenSorter (if in use) */ int addr1; /* Address of top of loop */ + int addr2; /* Address to jump to for next iteration */ int tnum; /* Root page of index */ + int iPartIdxLabel; /* Jump to this label to skip a row */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ - int regIdxKey; /* Registers containing the index key */ - int regRecord; /* Register holding assemblied index record */ + int regRecord; /* Register holding assembled index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -79669,43 +91270,87 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ tnum = memRootPage; }else{ tnum = pIndex->tnum; - sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); } - pKey = sqlite3IndexKeyinfo(pParse, pIndex); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, - (char *)pKey, P4_KEYINFO_HANDOFF); - if( memRootPage>=0 ){ - sqlite3VdbeChangeP5(v, 1); - } - sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); - regRecord = sqlite3GetTempReg(pParse); - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - if( pIndex->onError!=OE_None ){ - const int regRowid = regIdxKey + pIndex->nColumn; - const int j2 = sqlite3VdbeCurrentAddr(v) + 2; - void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); + pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); - /* The registers accessed by the OP_IsUnique opcode were allocated - ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() - ** call above. Just before that function was freed they were released - ** (made available to the compiler for reuse) using - ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique - ** opcode use the values stored within seems dangerous. However, since - ** we can be sure that no other temp registers have been allocated - ** since sqlite3ReleaseTempRange() was called, it is safe to do so. - */ - sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); + /* Open the sorter cursor if we are to use one. */ + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) + sqlite3KeyInfoRef(pKey), P4_KEYINFO); + + /* Open the table. Loop through all rows of the table, inserting index + ** records into the sorter. */ + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); VdbeCoverage(v); + regRecord = sqlite3GetTempReg(pParse); + + sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, + (char *)pKey, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); + + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); + if( IsUniqueIndex(pIndex) && pKey!=0 ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, + pIndex->nKeyCol); VdbeCoverage(v); + sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); } - sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); + sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp1(v, OP_Close, iTab); sqlite3VdbeAddOp1(v, OP_Close, iIdx); + sqlite3VdbeAddOp1(v, OP_Close, iSorter); +} + +/* +** Allocate heap space to hold an Index object with nCol columns. +** +** Increase the allocation size to provide an extra nExtra bytes +** of 8-byte aligned space after the Index object and return a +** pointer to this extra space in *ppExtra. +*/ +SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( + sqlite3 *db, /* Database connection */ + i16 nCol, /* Total number of columns in the index */ + int nExtra, /* Number of bytes of extra space to alloc */ + char **ppExtra /* Pointer to the "extra" space */ +){ + Index *p; /* Allocated index object */ + int nByte; /* Bytes of space for Index object + arrays */ + + nByte = ROUND8(sizeof(Index)) + /* Index structure */ + ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ + sizeof(i16)*nCol + /* Index.aiColumn */ + sizeof(u8)*nCol); /* Index.aSortOrder */ + p = sqlite3DbMallocZero(db, nByte + nExtra); + if( p ){ + char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->aSortOrder = (u8*)pExtra; + p->nColumn = nCol; + p->nKeyCol = nCol - 1; + *ppExtra = ((char*)p) + nByte; + } + return p; } /* @@ -79722,7 +91367,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ ** ** If the index is created successfully, return a pointer to the new Index ** structure. This is used by sqlite3AddPrimaryKey() to mark the index -** as the tables primary key (Index.autoIndex==2). +** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) */ SQLITE_PRIVATE Index *sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ @@ -79732,7 +91377,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ExprList *pList, /* A list of columns to be indexed */ int onError, /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ Token *pStart, /* The CREATE token that begins this statement */ - Token *pEnd, /* The ")" that closes the CREATE INDEX statement */ + Expr *pPIWhere, /* WHERE clause for partial indices */ int sortOrder, /* Sort order of primary key when pList==NULL */ int ifNotExist /* Omit error if index already exists */ ){ @@ -79742,7 +91387,6 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( char *zName = 0; /* Name of the index */ int nName; /* Number of characters in zName */ int i, j; - Token nullId; /* Fake token for an empty ID list */ DbFixer sFix; /* For assigning database names to pTable */ int sortOrderMask; /* 1 to honor DESC in index. 0 to ignore. */ sqlite3 *db = pParse->db; @@ -79750,11 +91394,12 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - int nCol; - int nExtra = 0; - char *zExtra; + const Column *pTabCol; /* A column in the table */ + int nExtra = 0; /* Space allocated for zExtra[] */ + int nExtraCol; /* Number of extra columns needed */ + char *zExtra = 0; /* Extra space after the Index object */ + Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - assert( pStart==0 || pEnd!=0 ); /* pEnd must be non-NULL if pStart is */ assert( pParse->nErr==0 ); /* Never called with prior errors */ if( db->mallocFailed || IN_DECLARE_VTAB ){ goto exit_create_index; @@ -79775,9 +91420,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( assert( pName1 && pName2 ); iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ) goto exit_create_index; + assert( pName && pName->z ); #ifndef SQLITE_OMIT_TEMPDB - /* If the index name was unqualified, check if the the table + /* If the index name was unqualified, check if the table ** is a temp table. If so, set the database to 1. Do not do this ** if initialising a database schema. */ @@ -79789,19 +91435,25 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( } #endif - if( sqlite3FixInit(&sFix, pParse, iDb, "index", pName) && - sqlite3FixSrcList(&sFix, pTblName) - ){ + sqlite3FixInit(&sFix, pParse, iDb, "index", pName); + if( sqlite3FixSrcList(&sFix, pTblName) ){ /* Because the parser constructs pTblName from a single identifier, ** sqlite3FixSrcList can never fail. */ assert(0); } - pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, - pTblName->a[0].zDatabase); - if( !pTab || db->mallocFailed ) goto exit_create_index; - assert( db->aDb[iDb].pSchema==pTab->pSchema ); + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; + if( iDb==1 && db->aDb[iDb].pSchema!=pTab->pSchema ){ + sqlite3ErrorMsg(pParse, + "cannot create a TEMP index on non-TEMP table \"%s\"", + pTab->zName); + goto exit_create_index; + } + if( !HasRowid(pTab) ) pPk = sqlite3PrimaryKeyIndex(pTab); }else{ assert( pName==0 ); + assert( pStart==0 ); pTab = pParse->pNewTable; if( !pTab ) goto exit_create_index; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -79811,7 +91463,11 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + && db->init.busy==0 +#if SQLITE_USER_AUTHENTICATION + && sqlite3UserAuthTable(pTab->zName)==0 +#endif + && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; } @@ -79844,6 +91500,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( pName ){ zName = sqlite3NameFromToken(db, pName); if( zName==0 ) goto exit_create_index; + assert( pName->z!=0 ); if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto exit_create_index; } @@ -79893,11 +91550,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** So create a fake list to simulate this. */ if( pList==0 ){ - nullId.z = pTab->aCol[pTab->nCol-1].zName; - nullId.n = sqlite3Strlen30((char*)nullId.z); pList = sqlite3ExprListAppend(pParse, 0, 0); if( pList==0 ) goto exit_create_index; - sqlite3ExprListSetName(pParse, pList, &nullId, 0); + pList->a[0].zName = sqlite3DbStrDup(pParse->db, + pTab->aCol[pTab->nCol-1].zName); pList->a[0].sortOrder = (u8)sortOrder; } @@ -79907,12 +91563,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; if( pExpr ){ - CollSeq *pColl = pExpr->pColl; - /* Either pColl!=0 or there was an OOM failure. But if an OOM - ** failure we have quit before reaching this point. */ - if( ALWAYS(pColl) ){ - nExtra += (1 + sqlite3Strlen30(pColl->zName)); - } + assert( pExpr->op==TK_COLLATE ); + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -79920,31 +91572,28 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** Allocate the index structure. */ nName = sqlite3Strlen30(zName); - nCol = pList->nExpr; - pIndex = sqlite3DbMallocZero(db, - sizeof(Index) + /* Index structure */ - sizeof(int)*nCol + /* Index.aiColumn */ - sizeof(int)*(nCol+1) + /* Index.aiRowEst */ - sizeof(char *)*nCol + /* Index.azColl */ - sizeof(u8)*nCol + /* Index.aSortOrder */ - nName + 1 + /* Index.zName */ - nExtra /* Collation sequence names */ - ); + nExtraCol = pPk ? pPk->nKeyCol : 1; + pIndex = sqlite3AllocateIndexObject(db, pList->nExpr + nExtraCol, + nName + nExtra + 1, &zExtra); if( db->mallocFailed ){ goto exit_create_index; } - pIndex->azColl = (char**)(&pIndex[1]); - pIndex->aiColumn = (int *)(&pIndex->azColl[nCol]); - pIndex->aiRowEst = (unsigned *)(&pIndex->aiColumn[nCol]); - pIndex->aSortOrder = (u8 *)(&pIndex->aiRowEst[nCol+1]); - pIndex->zName = (char *)(&pIndex->aSortOrder[nCol]); - zExtra = (char *)(&pIndex->zName[nName+1]); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); + pIndex->zName = zExtra; + zExtra += nName + 1; memcpy(pIndex->zName, zName, nName+1); pIndex->pTable = pTab; - pIndex->nColumn = pList->nExpr; pIndex->onError = (u8)onError; - pIndex->autoIndex = (u8)(pName==0); + pIndex->uniqNotNull = onError!=OE_None; + pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; pIndex->pSchema = db->aDb[iDb].pSchema; + pIndex->nKeyCol = pList->nExpr; + if( pPIWhere ){ + sqlite3ResolveSelfReference(pParse, pTab, NC_PartIdx, pPIWhere, 0); + pIndex->pPartIdxWhere = pPIWhere; + pPIWhere = 0; + } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); /* Check to see if we should honor DESC requests on index columns @@ -79967,7 +91616,6 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ const char *zColName = pListItem->zName; - Column *pTabCol; int requestedSortOrder; char *zColl; /* Collation sequence name */ @@ -79980,15 +91628,12 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pParse->checkSchema = 1; goto exit_create_index; } - pIndex->aiColumn[i] = j; - /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of - ** the way the "idxlist" non-terminal is constructed by the parser, - ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl - ** must exist or else there must have been an OOM error. But if there - ** was an OOM error, we would never reach this point. */ - if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ + assert( j<=0x7fff ); + pIndex->aiColumn[i] = (i16)j; + if( pListItem->pExpr ){ int nColl; - zColl = pListItem->pExpr->pColl->zName; + assert( pListItem->pExpr->op==TK_COLLATE ); + zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -79997,9 +91642,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( nExtra -= nColl; }else{ zColl = pTab->aCol[j].zColl; - if( !zColl ){ - zColl = db->pDfltColl->zName; - } + if( !zColl ) zColl = "BINARY"; } if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; @@ -80007,8 +91650,27 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; + if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; + } + if( pPk ){ + for(j=0; jnKeyCol; j++){ + int x = pPk->aiColumn[j]; + if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ + pIndex->nColumn--; + }else{ + pIndex->aiColumn[i] = x; + pIndex->azColl[i] = pPk->azColl[j]; + pIndex->aSortOrder[i] = pPk->aSortOrder[j]; + i++; + } + } + assert( i==pIndex->nColumn ); + }else{ + pIndex->aiColumn[i] = -1; + pIndex->azColl[i] = "BINARY"; } sqlite3DefaultRowEst(pIndex); + if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); if( pTab==pParse->pNewTable ){ /* This routine has been called to create an automatic index as a @@ -80035,12 +91697,12 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( Index *pIdx; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int k; - assert( pIdx->onError!=OE_None ); - assert( pIdx->autoIndex ); - assert( pIndex->onError!=OE_None ); + assert( IsUniqueIndex(pIdx) ); + assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF ); + assert( IsUniqueIndex(pIndex) ); - if( pIdx->nColumn!=pIndex->nColumn ) continue; - for(k=0; knColumn; k++){ + if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue; + for(k=0; knKeyCol; k++){ const char *z1; const char *z2; if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; @@ -80048,14 +91710,14 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( z2 = pIndex->azColl[k]; if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; } - if( k==pIdx->nColumn ){ + if( k==pIdx->nKeyCol ){ if( pIdx->onError!=pIndex->onError ){ /* This constraint creates the same index as a previous ** constraint specified somewhere in the CREATE TABLE statement. ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the - ** explicitly specified behaviour for the index. + ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ sqlite3ErrorMsg(pParse, @@ -80077,8 +91739,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, sqlite3Strlen30(pIndex->zName), - pIndex); + pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ db->mallocFailed = 1; @@ -80090,22 +91751,20 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( } } - /* If the db->init.busy is 0 then create the index on disk. This - ** involves writing the index into the master table and filling in the - ** index with the current table contents. + /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the + ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then + ** emit code to allocate the index rootpage on disk and make an entry for + ** the index in the sqlite_master table and populate the index with + ** content. But, do not do this if we are simply reading the sqlite_master + ** table to parse the schema, or if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table. ** - ** The db->init.busy is 0 when the user first enters a CREATE INDEX - ** command. db->init.busy is 1 when a database is opened and - ** CREATE INDEX statements are read out of the master table. In - ** the latter case the index already exists on disk, which is why - ** we don't want to recreate it. - ** - ** If pTblName==0 it means this index is generated as a primary key - ** or UNIQUE constraint of a CREATE TABLE statement. Since the table + ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY + ** or UNIQUE index in a CREATE TABLE statement. Since the table ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else{ /* if( db->init.busy==0 ) */ + else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; @@ -80123,12 +91782,11 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** the zStmt variable */ if( pStart ){ - assert( pEnd!=0 ); + int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; + if( pName->z[n-1]==';' ) n--; /* A named index with an explicit CREATE INDEX statement */ zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", - onError==OE_None ? "" : " UNIQUE", - pEnd->z - pName->z + 1, - pName->z); + onError==OE_None ? "" : " UNIQUE", n, pName->z); }else{ /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ /* zStmt = sqlite3MPrintf(""); */ @@ -80184,10 +91842,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* Clean up before exiting */ exit_create_index: - if( pIndex ){ - sqlite3DbFree(db, pIndex->zColAff); - sqlite3DbFree(db, pIndex); - } + if( pIndex ) freeIndex(db, pIndex); + sqlite3ExprDelete(db, pPIWhere); sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); sqlite3DbFree(db, zName); @@ -80198,11 +91854,11 @@ exit_create_index: ** Fill the Index.aiRowEst[] array with default information - information ** to be used when we have not run the ANALYZE command. ** -** aiRowEst[0] is suppose to contain the number of elements in the index. +** aiRowEst[0] is supposed to contain the number of elements in the index. ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns +** of rows that match any particular combination of the first 2 columns ** of the index. And so forth. It must always be the case that * ** aiRowEst[N]<=aiRowEst[N-1] @@ -80213,20 +91869,27 @@ exit_create_index: ** are based on typical values found in actual indices. */ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ - unsigned *a = pIdx->aiRowEst; + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; - unsigned n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nColumn; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nColumn] = 1; + + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ + a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); } + + assert( 0==sqlite3LogEst(1) ); + if( IsUniqueIndex(pIdx) ) a[pIdx->nKeyCol] = 0; } /* @@ -80257,7 +91920,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists pParse->checkSchema = 1; goto exit_drop_index; } - if( pIndex->autoIndex ){ + if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){ sqlite3ErrorMsg(pParse, "index associated with UNIQUE " "or PRIMARY KEY constraint cannot be dropped", 0); goto exit_drop_index; @@ -80285,15 +91948,9 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - pIndex->zName + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName ); - if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.sqlite_stat1 WHERE idx=%Q", - db->aDb[iDb].zName, pIndex->zName - ); - } + sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); sqlite3ChangeCookie(pParse, iDb); destroyRootPage(pParse, pIndex->tnum, iDb); sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); @@ -80304,45 +91961,43 @@ exit_drop_index: } /* -** pArray is a pointer to an array of objects. Each object in the -** array is szEntry bytes in size. This routine allocates a new -** object on the end of the array. +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. ** -** *pnEntry is the number of entries already in use. *pnAlloc is -** the previously allocated size of the array. initSize is the -** suggested initial array size allocation. +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). ** -** The index of the new entry is returned in *pIdx. +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. ** -** This routine returns a pointer to the array of objects. This -** might be the same as the pArray parameter or it might be a different -** pointer if the array was resized. +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. */ SQLITE_PRIVATE void *sqlite3ArrayAllocate( sqlite3 *db, /* Connection to notify of malloc failures */ void *pArray, /* Array of objects. Might be reallocated */ int szEntry, /* Size of each object in the array */ - int initSize, /* Suggested initial allocation, in elements */ int *pnEntry, /* Number of objects currently in use */ - int *pnAlloc, /* Current size of the allocation, in elements */ int *pIdx /* Write the index of a new slot here */ ){ char *z; - if( *pnEntry >= *pnAlloc ){ - void *pNew; - int newSize; - newSize = (*pnAlloc)*2 + initSize; - pNew = sqlite3DbRealloc(db, pArray, newSize*szEntry); + int n = *pnEntry; + if( (n & (n-1))==0 ){ + int sz = (n==0) ? 1 : 2*n; + void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); if( pNew==0 ){ *pIdx = -1; return pArray; } - *pnAlloc = sqlite3DbMallocSize(db, pNew)/szEntry; pArray = pNew; } z = (char*)pArray; - memset(&z[*pnEntry * szEntry], 0, szEntry); - *pIdx = *pnEntry; + memset(&z[n * szEntry], 0, szEntry); + *pIdx = n; ++*pnEntry; return pArray; } @@ -80358,15 +92013,12 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pT if( pList==0 ){ pList = sqlite3DbMallocZero(db, sizeof(IdList) ); if( pList==0 ) return 0; - pList->nAlloc = 0; } pList->a = sqlite3ArrayAllocate( db, pList->a, sizeof(pList->a[0]), - 5, &pList->nId, - &pList->nAlloc, &i ); if( i<0 ){ @@ -80437,7 +92089,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( assert( iStart<=pSrc->nSrc ); /* Allocate additional space if needed */ - if( pSrc->nSrc+nExtra>pSrc->nAlloc ){ + if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; int nAlloc = pSrc->nSrc+nExtra; int nGot; @@ -80449,7 +92101,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( } pSrc = pNew; nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; - pSrc->nAlloc = (u16)nGot; + pSrc->nAlloc = nGot; } /* Move existing slots that come after the newly inserted slots @@ -80457,7 +92109,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( for(i=pSrc->nSrc-1; i>=iStart; i--){ pSrc->a[i+nExtra] = pSrc->a[i]; } - pSrc->nSrc += (i16)nExtra; + pSrc->nSrc += nExtra; /* Zero the newly allocated slots */ memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); @@ -80581,7 +92233,7 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ ** if this is the first term of the FROM clause. pTable and pDatabase ** are the name of the table and database named in the FROM clause term. ** pDatabase is NULL if the database name qualifier is missing - the -** usual case. If the term has a alias, then pAlias points to the +** usual case. If the term has an alias, then pAlias points to the ** alias token. If the term is a subquery, then pSubquery is the ** SELECT statement that the subquery encodes. The pTable and ** pDatabase parameters are NULL for subqueries. The pOn and pUsing @@ -80665,8 +92317,9 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI ** operator with A. This routine shifts that operator over to B. */ SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ - if( p && p->a ){ + if( p ){ int i; + assert( p->a || p->nSrc==0 ); for(i=p->nSrc-1; i>0; i--){ p->a[i].jointype = p->a[i-1].jointype; } @@ -80704,13 +92357,10 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ ** Commit a transaction */ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ - sqlite3 *db; Vdbe *v; assert( pParse!=0 ); - db = pParse->db; - assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ + assert( pParse->db!=0 ); if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "COMMIT", 0, 0) ){ return; } @@ -80724,13 +92374,10 @@ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ ** Rollback a transaction */ SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ - sqlite3 *db; Vdbe *v; assert( pParse!=0 ); - db = pParse->db; - assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ + assert( pParse->db!=0 ); if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "ROLLBACK", 0, 0) ){ return; } @@ -80794,50 +92441,24 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ } /* -** Generate VDBE code that will verify the schema cookie and start -** a read-transaction for all named database files. -** -** It is important that all schema cookies be verified and all -** read transactions be started before anything else happens in -** the VDBE program. But this routine can be called after much other -** code has been generated. So here is what we do: -** -** The first time this routine is called, we code an OP_Goto that -** will jump to a subroutine at the end of the program. Then we -** record every database that needs its schema verified in the -** pParse->cookieMask field. Later, after all other code has been -** generated, the subroutine that does the cookie verifications and -** starts the transactions will be coded and the OP_Goto P2 value -** will be made to point to that subroutine. The generation of the -** cookie verification subroutine code happens in sqlite3FinishCoding(). -** -** If iDb<0 then code the OP_Goto only - don't set flag to verify the -** schema on any databases. This can be used to position the OP_Goto -** early in the code, before we know if any database tables will be used. +** Record the fact that the schema cookie will need to be verified +** for database iDb. The code to actually verify the schema cookie +** will occur at the end of the top-level VDBE and will be generated +** later, by sqlite3FinishCoding(). */ SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3 *db = pToplevel->db; - if( pToplevel->cookieGoto==0 ){ - Vdbe *v = sqlite3GetVdbe(pToplevel); - if( v==0 ) return; /* This only happens if there was a prior error */ - pToplevel->cookieGoto = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0)+1; - } - if( iDb>=0 ){ - sqlite3 *db = pToplevel->db; - yDbMask mask; - - assert( iDbnDb ); - assert( db->aDb[iDb].pBt!=0 || iDb==1 ); - assert( iDbcookieMask & mask)==0 ){ - pToplevel->cookieMask |= mask; - pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; - if( !OMIT_TEMPDB && iDb==1 ){ - sqlite3OpenTempDatabase(pToplevel); - } + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDbcookieMask, iDb)==0 ){ + DbMaskSet(pToplevel->cookieMask, iDb); + pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; + if( !OMIT_TEMPDB && iDb==1 ){ + sqlite3OpenTempDatabase(pToplevel); } } } @@ -80873,7 +92494,7 @@ SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb) SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3CodeVerifySchema(pParse, iDb); - pToplevel->writeMask |= ((yDbMask)1)<writeMask, iDb); pToplevel->isMultiWrite |= setStatement; } @@ -80915,12 +92536,73 @@ SQLITE_PRIVATE void sqlite3MayAbort(Parse *pParse){ ** error. The onError parameter determines which (if any) of the statement ** and/or current transaction is rolled back. */ -SQLITE_PRIVATE void sqlite3HaltConstraint(Parse *pParse, int onError, char *p4, int p4type){ +SQLITE_PRIVATE void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + i8 p4type, /* P4_STATIC or P4_TRANSIENT */ + u8 p5Errmsg /* P5_ErrMsg type */ +){ Vdbe *v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } - sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, p4, p4type); + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); + if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg); +} + +/* +** Code an OP_Halt due to UNIQUE or PRIMARY KEY constraint violation. +*/ +SQLITE_PRIVATE void sqlite3UniqueConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Constraint type */ + Index *pIdx /* The index that triggers the constraint */ +){ + char *zErr; + int j; + StrAccum errMsg; + Table *pTab = pIdx->pTable; + + sqlite3StrAccumInit(&errMsg, 0, 0, 200); + errMsg.db = pParse->db; + for(j=0; jnKeyCol; j++){ + char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3StrAccumAppendAll(&errMsg, pTab->zName); + sqlite3StrAccumAppend(&errMsg, ".", 1); + sqlite3StrAccumAppendAll(&errMsg, zCol); + } + zErr = sqlite3StrAccumFinish(&errMsg); + sqlite3HaltConstraint(pParse, + IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY + : SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, P4_DYNAMIC, P5_ConstraintUnique); +} + + +/* +** Code an OP_Halt due to non-unique rowid. +*/ +SQLITE_PRIVATE void sqlite3RowidConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Conflict resolution algorithm */ + Table *pTab /* The table with the non-unique rowid */ +){ + char *zMsg; + int rc; + if( pTab->iPKey>=0 ){ + zMsg = sqlite3MPrintf(pParse->db, "%s.%s", pTab->zName, + pTab->aCol[pTab->iPKey].zName); + rc = SQLITE_CONSTRAINT_PRIMARYKEY; + }else{ + zMsg = sqlite3MPrintf(pParse->db, "%s.rowid", pTab->zName); + rc = SQLITE_CONSTRAINT_ROWID; + } + sqlite3HaltConstraint(pParse, rc, onError, zMsg, P4_DYNAMIC, + P5_ConstraintUnique); } /* @@ -80933,8 +92615,8 @@ static int collationMatch(const char *zColl, Index *pIndex){ assert( zColl!=0 ); for(i=0; inColumn; i++){ const char *z = pIndex->azColl[i]; - assert( z!=0 ); - if( 0==sqlite3StrICmp(z, zColl) ){ + assert( z!=0 || pIndex->aiColumn[i]<0 ); + if( pIndex->aiColumn[i]>=0 && 0==sqlite3StrICmp(z, zColl) ){ return 1; } } @@ -81053,42 +92735,122 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ #endif /* -** Return a dynamicly allocated KeyInfo structure that can be used -** with OP_OpenRead or OP_OpenWrite to access database index pIdx. +** Return a KeyInfo structure that is appropriate for the given Index. ** -** If successful, a pointer to the new structure is returned. In this case -** the caller is responsible for calling sqlite3DbFree(db, ) on the returned -** pointer. If an error occurs (out of memory or missing collation -** sequence), NULL is returned and the state of pParse updated to reflect -** the error. +** The KeyInfo structure for an index is cached in the Index object. +** So there might be multiple references to the returned pointer. The +** caller should not try to modify the KeyInfo object. +** +** The caller should invoke sqlite3KeyInfoUnref() on the returned object +** when it has finished using it. */ -SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){ - int i; - int nCol = pIdx->nColumn; - int nBytes = sizeof(KeyInfo) + (nCol-1)*sizeof(CollSeq*) + nCol; - sqlite3 *db = pParse->db; - KeyInfo *pKey = (KeyInfo *)sqlite3DbMallocZero(db, nBytes); - - if( pKey ){ - pKey->db = pParse->db; - pKey->aSortOrder = (u8 *)&(pKey->aColl[nCol]); - assert( &pKey->aSortOrder[nCol]==&(((u8 *)pKey)[nBytes]) ); - for(i=0; iazColl[i]; - assert( zColl ); - pKey->aColl[i] = sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + if( pParse->nErr ) return 0; +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){ + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } +#endif + if( pIdx->pKeyInfo==0 ){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + } + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + assert( zColl!=0 ); + pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortOrder[i] = pIdx->aSortOrder[i]; + } + if( pParse->nErr ){ + sqlite3KeyInfoUnref(pKey); + }else{ + pIdx->pKeyInfo = pKey; + } + } + } + return sqlite3KeyInfoRef(pIdx->pKeyInfo); +} + +#ifndef SQLITE_OMIT_CTE +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. +*/ +SQLITE_PRIVATE With *sqlite3WithAdd( + Parse *pParse, /* Parsing context */ + With *pWith, /* Existing WITH clause, or NULL */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ + Select *pQuery /* Query used to initialize the table */ +){ + sqlite3 *db = pParse->db; + With *pNew; + char *zName; + + /* Check that the CTE name is unique within this WITH clause. If + ** not, store an error in the Parse structure. */ + zName = sqlite3NameFromToken(pParse->db, pName); + if( zName && pWith ){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName); + } } - pKey->nField = (u16)nCol; } - if( pParse->nErr ){ - sqlite3DbFree(db, pKey); - pKey = 0; + if( pWith ){ + int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + pNew = sqlite3DbRealloc(db, pWith, nByte); + }else{ + pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); } - return pKey; + assert( zName!=0 || pNew==0 ); + assert( db->mallocFailed==0 || pNew==0 ); + + if( pNew==0 ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + sqlite3DbFree(db, zName); + pNew = pWith; + }else{ + pNew->a[pNew->nCte].pSelect = pQuery; + pNew->a[pNew->nCte].pCols = pArglist; + pNew->a[pNew->nCte].zName = zName; + pNew->a[pNew->nCte].zErr = 0; + pNew->nCte++; + } + + return pNew; } +/* +** Free the contents of the With object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ + if( pWith ){ + int i; + for(i=0; inCte; i++){ + struct Cte *pCte = &pWith->a[i]; + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); + } + sqlite3DbFree(db, pWith); + } +} +#endif /* !defined(SQLITE_OMIT_CTE) */ + /************** End of build.c ***********************************************/ /************** Begin file callback.c ****************************************/ /* @@ -81167,17 +92929,18 @@ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ ** ** The return value is either the collation sequence to be used in database ** db for collation type name zName, length nName, or NULL, if no collation -** sequence can be found. +** sequence can be found. If no collation is found, leave an error message. ** ** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() */ SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( - sqlite3* db, /* The database connection */ + Parse *pParse, /* Parsing context */ u8 enc, /* The desired encoding for the collating sequence */ CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ const char *zName /* Collating sequence name */ ){ CollSeq *p; + sqlite3 *db = pParse->db; p = pColl; if( !p ){ @@ -81194,6 +92957,9 @@ SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( p = 0; } assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + } return p; } @@ -81212,10 +92978,8 @@ SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ if( pColl ){ const char *zName = pColl->zName; sqlite3 *db = pParse->db; - CollSeq *p = sqlite3GetCollSeq(db, ENC(db), pColl, zName); + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); if( !p ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - pParse->nErr++; return SQLITE_ERROR; } assert( p==pColl ); @@ -81232,7 +92996,7 @@ SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ ** ** Each pointer stored in the sqlite3.aCollSeq hash table contains an ** array of three CollSeq structures. The first is the collation sequence -** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. ** ** Stored immediately after the three collation sequences is a copy of ** the collation sequence name. A pointer to this string is stored in @@ -81244,11 +93008,11 @@ static CollSeq *findCollSeqEntry( int create /* Create a new entry if true */ ){ CollSeq *pColl; - int nName = sqlite3Strlen30(zName); - pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + pColl = sqlite3HashFind(&db->aCollSeq, zName); if( 0==pColl && create ){ - pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); + int nName = sqlite3Strlen30(zName); + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1); if( pColl ){ CollSeq *pDel = 0; pColl[0].zName = (char*)&pColl[3]; @@ -81259,7 +93023,7 @@ static CollSeq *findCollSeqEntry( pColl[2].enc = SQLITE_UTF16BE; memcpy(pColl[0].zName, zName, nName); pColl[0].zName[nName] = 0; - pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl); + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); /* If a malloc() failure occurred in sqlite3HashInsert(), it will ** return the pColl pointer to be deleted (because it wasn't added @@ -81315,38 +93079,57 @@ SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( ** that uses encoding enc. The value returned indicates how well the ** request is matched. A higher value indicates a better match. ** +** If nArg is -1 that means to only return a match (non-zero) if p->nArg +** is also -1. In other words, we are searching for a function that +** takes a variable number of arguments. +** +** If nArg is -2 that means that we are searching for any function +** regardless of the number of arguments it uses, so return a positive +** match score for any +** ** The returned value is always between 0 and 6, as follows: ** -** 0: Not a match, or if nArg<0 and the function is has no implementation. -** 1: A variable arguments function that prefers UTF-8 when a UTF-16 -** encoding is requested, or vice versa. -** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is -** requested, or vice versa. -** 3: A variable arguments function using the same text encoding. -** 4: A function with the exact number of arguments requested that -** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. -** 5: A function with the exact number of arguments requested that -** prefers UTF-16LE when UTF-16BE is requested, or vice versa. -** 6: An exact match. +** 0: Not a match. +** 1: UTF8/16 conversion required and function takes any number of arguments. +** 2: UTF16 byte order change required and function takes any number of args. +** 3: encoding matches and function takes any number of arguments +** 4: UTF8/16 conversion required - argument count matches exactly +** 5: UTF16 byte order conversion required - argument count matches exactly +** 6: Perfect match: encoding and argument count match exactly. ** +** If nArg==(-2) then any function with a non-null xStep or xFunc is +** a perfect match and any function with both xStep and xFunc NULL is +** a non-match. */ -static int matchQuality(FuncDef *p, int nArg, u8 enc){ - int match = 0; - if( p->nArg==-1 || p->nArg==nArg - || (nArg==-1 && (p->xFunc!=0 || p->xStep!=0)) - ){ +#define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ +static int matchQuality( + FuncDef *p, /* The function we are evaluating for match quality */ + int nArg, /* Desired number of arguments. (-1)==any */ + u8 enc /* Desired text encoding */ +){ + int match; + + /* nArg of -2 is a special case */ + if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; + + /* Wrong number of arguments means "no match" */ + if( p->nArg!=nArg && p->nArg>=0 ) return 0; + + /* Give a better score to a function with a specific number of arguments + ** than to function that accepts any number of arguments. */ + if( p->nArg==nArg ){ + match = 4; + }else{ match = 1; - if( p->nArg==nArg || nArg==-1 ){ - match = 4; - } - if( enc==p->iPrefEnc ){ - match += 2; - } - else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || - (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ - match += 1; - } } + + /* Bonus points if the text encoding matches */ + if( enc==(p->funcFlags & SQLITE_FUNC_ENCMASK) ){ + match += 2; /* Exact encoding match */ + }else if( (enc & p->funcFlags & 2)!=0 ){ + match += 1; /* Both are UTF16, but with different byte orders */ + } + return match; } @@ -81402,13 +93185,12 @@ SQLITE_PRIVATE void sqlite3FuncDefInsert( ** ** If the createFlag argument is true, then a new (blank) FuncDef ** structure is created and liked into the "db" structure if a -** no matching function previously existed. When createFlag is true -** and the nArg parameter is -1, then only a function that accepts -** any number of arguments will be returned. +** no matching function previously existed. ** -** If createFlag is false and nArg is -1, then the first valid -** function found is returned. A function is valid if either xFunc -** or xStep is non-zero. +** If nArg is -2, then the first valid function found is returned. A +** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) +** case is used to see if zName is a valid function name for some number +** of arguments. If nArg is -2, then createFlag must be 0. ** ** If createFlag is false, then a function with the required name and ** number of arguments may be returned even if the eTextRep flag does not @@ -81420,15 +93202,15 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( int nName, /* Number of characters in the name */ int nArg, /* Number of arguments. -1 means any number */ u8 enc, /* Preferred text encoding */ - int createFlag /* Create new entry if true and does not otherwise exist */ + u8 createFlag /* Create new entry if true and does not otherwise exist */ ){ FuncDef *p; /* Iterator variable */ FuncDef *pBest = 0; /* Best match found so far */ int bestScore = 0; /* Score of best match */ int h; /* Hash value */ - - assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + assert( nArg>=(-2) ); + assert( nArg>=(-1) || createFlag==0 ); h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); /* First search for a match amongst the application-defined functions. @@ -81473,11 +93255,11 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** exact match for the name, number of arguments and encoding, then add a ** new entry to the hash table and return it. */ - if( createFlag && (bestScore<6 || pBest->nArg!=nArg) && + if( createFlag && bestScorezName = (char *)&pBest[1]; pBest->nArg = (u16)nArg; - pBest->iPrefEnc = enc; + pBest->funcFlags = enc; memcpy(pBest->zName, zName, nName); pBest->zName[nName] = 0; sqlite3FuncDefInsert(&db->aFunc, pBest); @@ -81519,9 +93301,9 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); pSchema->pSeqTab = 0; - if( pSchema->flags & DB_SchemaLoaded ){ + if( pSchema->schemaFlags & DB_SchemaLoaded ){ pSchema->iGeneration++; - pSchema->flags &= ~DB_SchemaLoaded; + pSchema->schemaFlags &= ~DB_SchemaLoaded; } } @@ -81583,7 +93365,7 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ struct SrcList_item *pItem = pSrc->a; Table *pTab; assert( pItem && pSrc->nSrc==1 ); - pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ @@ -81641,32 +93423,26 @@ SQLITE_PRIVATE void sqlite3MaterializeView( Parse *pParse, /* Parsing context */ Table *pView, /* View definition */ Expr *pWhere, /* Optional WHERE clause to be added */ - int iCur /* Cursor number for ephemerial table */ + int iCur /* Cursor number for ephemeral table */ ){ SelectDest dest; - Select *pDup; + Select *pSel; + SrcList *pFrom; sqlite3 *db = pParse->db; - - pDup = sqlite3SelectDup(db, pView->pSelect, 0); - if( pWhere ){ - SrcList *pFrom; - - pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); - if( pFrom ){ - assert( pFrom->nSrc==1 ); - pFrom->a[0].zAlias = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].pSelect = pDup; - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); - }else{ - sqlite3SelectDelete(db, pDup); - } - pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); } + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); - sqlite3Select(pParse, pDup, &dest); - sqlite3SelectDelete(db, pDup); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); } #endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ @@ -81686,7 +93462,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( ExprList *pOrderBy, /* The ORDER BY clause. May be null */ Expr *pLimit, /* The LIMIT clause. May be null */ Expr *pOffset, /* The OFFSET clause. May be null */ - char *zStmtType /* Either DELETE or UPDATE. For error messages. */ + char *zStmtType /* Either DELETE or UPDATE. For err msgs. */ ){ Expr *pWhereRowid = NULL; /* WHERE rowid .. */ Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ @@ -81699,7 +93475,6 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( */ if( pOrderBy && (pLimit == 0) ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - pParse->parseError = 1; goto limit_where_cleanup_2; } @@ -81762,7 +93537,8 @@ limit_where_cleanup_2: sqlite3ExprDelete(pParse->db, pOffset); return 0; } -#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) */ + /* && !defined(SQLITE_OMIT_SUBQUERY) */ /* ** Generate code for a DELETE FROM statement. @@ -81779,18 +93555,34 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ const char *zDb; /* Name of database holding pTab */ - int end, addr = 0; /* A couple addresses of generated code */ int i; /* Loop counter */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Index *pIdx; /* For looping over indices of the table */ - int iCur; /* VDBE Cursor number for pTab */ + int iTabCur; /* Cursor number for the table */ + int iDataCur; /* VDBE cursor for the canonical data source */ + int iIdxCur; /* Cursor number of the first index */ + int nIdx; /* Number of indices */ sqlite3 *db; /* Main database structure */ AuthContext sContext; /* Authorization context */ NameContext sNC; /* Name context to resolve expressions in */ int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - + int okOnePass; /* True for one-pass algorithm without the FIFO */ + int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ + Index *pPk; /* The PRIMARY KEY index on the table */ + int iPk = 0; /* First of nPk registers holding PRIMARY KEY value */ + i16 nPk = 1; /* Number of columns in the PRIMARY KEY */ + int iKey; /* Memory cell holding key of row to be deleted */ + i16 nKey; /* Number of memory cells in the row key */ + int iEphCur = 0; /* Ephemeral table holding all primary key values */ + int iRowSet = 0; /* Register for rowset of rows to delete */ + int addrBypass = 0; /* Address of jump over the delete logic */ + int addrLoop = 0; /* Top of the delete loop */ + int addrDelete = 0; /* Jump directly to the delete logic */ + int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ + #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ @@ -81845,11 +93637,11 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } assert(!isView || pTrigger); - /* Assign cursor number to the table and all its indices. + /* Assign cursor numbers to the table and all its indices. */ assert( pTabList->nSrc==1 ); - iCur = pTabList->a[0].iCursor = pParse->nTab++; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + iTabCur = pTabList->a[0].iCursor = pParse->nTab++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ pParse->nTab++; } @@ -81869,11 +93661,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3BeginWriteOperation(pParse, 1, iDb); /* If we are trying to delete from a view, realize that view into - ** a ephemeral table. + ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iCur); + sqlite3MaterializeView(pParse, pTab, pWhere, iTabCur); + iDataCur = iIdxCur = iTabCur; } #endif @@ -81903,76 +93696,173 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( && 0==sqlite3FkRequired(pParse, pTab, 0, 0) ){ assert( !isView ); - sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt, - pTab->zName, P4_STATIC); + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt, + pTab->zName, P4_STATIC); + } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pIdx->pSchema==pTab->pSchema ); sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); } }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ - /* The usual case: There is a WHERE clause so we have to scan through - ** the table and pick which records to delete. - */ { - int iRowSet = ++pParse->nMem; /* Register for rowset of rows to delete */ - int iRowid = ++pParse->nMem; /* Used for storing rowid values. */ - int regRowid; /* Actual register containing rowids */ - - /* Collect rowids of every row to be deleted. + if( HasRowid(pTab) ){ + /* For a rowid table, initialize the RowSet to an empty set */ + pPk = 0; + nPk = 1; + iRowSet = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); + }else{ + /* For a WITHOUT ROWID table, create an ephemeral table used to + ** hold all primary keys for rows to be deleted. */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + iEphCur = pParse->nTab++; + addrEphOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEphCur, nPk); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + + /* Construct a query to find the rowid or primary key for every row + ** to be deleted, based on the WHERE clause. */ - sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, + WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, + iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); + okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + + /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); } + + /* Extract the rowid or primary key for the current row */ + if( pPk ){ + for(i=0; iaiColumn[i], iPk+i); + } + iKey = iPk; + }else{ + iKey = pParse->nMem + 1; + iKey = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iTabCur, iKey, 0); + if( iKey>pParse->nMem ) pParse->nMem = iKey; + } + + if( okOnePass ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only + ** one, so just keep it in its register(s) and fall through to the + ** delete code. + */ + nKey = nPk; /* OP_Found will use an unpacked key */ + aToOpen = sqlite3DbMallocRaw(db, nIdx+2); + if( aToOpen==0 ){ + sqlite3WhereEnd(pWInfo); + goto delete_from_cleanup; + } + memset(aToOpen, 1, nIdx+1); + aToOpen[nIdx+1] = 0; + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; + if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); + addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ + }else if( pPk ){ + /* Construct a composite key for the row to be deleted and remember it */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Get the rowid of the row to be deleted and remember it in the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } + + /* End of the WHERE loop */ sqlite3WhereEnd(pWInfo); - - /* Delete every item whose key was written to the list during the - ** database scan. We have to delete items after the scan is complete - ** because deleting an item can change the scan order. */ - end = sqlite3VdbeMakeLabel(v); - + if( okOnePass ){ + /* Bypass the delete logic below if the WHERE loop found zero rows */ + addrBypass = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBypass); + sqlite3VdbeJumpHere(v, addrDelete); + } + /* Unless this is a view, open cursors for the table we are ** deleting from and all its indices. If this is a view, then the ** only effect this statement has is to fire the INSTEAD OF - ** triggers. */ + ** triggers. + */ if( !isView ){ - sqlite3OpenTableAndIndices(pParse, pTab, iCur, OP_OpenWrite); + testcase( IsVirtual(pTab) ); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, + &iDataCur, &iIdxCur); + assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); + assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); } - - addr = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, end, iRowid); - + + /* Set up a loop over the rowids/primary-keys that were found in the + ** where-clause loop above. + */ + if( okOnePass ){ + /* Just one row. Hence the top-of-loop is a no-op */ + assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + assert( !IsVirtual(pTab) ); + if( aToOpen[iDataCur-iTabCur] ){ + assert( pPk!=0 || pTab->pSelect!=0 ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); + VdbeCoverage(v); + } + }else if( pPk ){ + addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_RowKey, iEphCur, iKey); + assert( nKey==0 ); /* OP_Found will use a composite key */ + }else{ + addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); + VdbeCoverage(v); + assert( nKey==1 ); + } + /* Delete the row */ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTab) ){ const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); - sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, pVTab, P4_VTAB); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, OE_Abort); sqlite3MayAbort(pParse); }else #endif { int count = (pParse->nested==0); /* True to count changes */ - sqlite3GenerateRowDelete(pParse, pTab, iCur, iRowid, count, pTrigger, OE_Default); + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + iKey, nKey, count, OE_Default, okOnePass); } - - /* End of the delete loop */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); - sqlite3VdbeResolveLabel(v, end); - + + /* End of the loop over all rowids/primary-keys. */ + if( okOnePass ){ + sqlite3VdbeResolveLabel(v, addrBypass); + }else if( pPk ){ + sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrLoop); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); + sqlite3VdbeJumpHere(v, addrLoop); + } + /* Close the cursors open on the table and its indexes. */ if( !isView && !IsVirtual(pTab) ){ - for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - sqlite3VdbeAddOp2(v, OP_Close, iCur + i, pIdx->tnum); + if( !pPk ) sqlite3VdbeAddOp1(v, OP_Close, iDataCur); + for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + sqlite3VdbeAddOp1(v, OP_Close, iIdxCur + i); } - sqlite3VdbeAddOp1(v, OP_Close, iCur); } - } + } /* End non-truncate path */ /* Update the sqlite_sequence table by storing the content of the ** maximum rowid counter values recorded while inserting into @@ -81996,10 +93886,11 @@ delete_from_cleanup: sqlite3AuthContextPop(&sContext); sqlite3SrcListDelete(db, pTabList); sqlite3ExprDelete(db, pWhere); + sqlite3DbFree(db, aToOpen); return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -82010,50 +93901,63 @@ delete_from_cleanup: /* ** This routine generates VDBE code that causes a single row of a -** single table to be deleted. +** single table to be deleted. Both the original table entry and +** all indices are removed. ** -** The VDBE must be in a particular state when this routine is called. -** These are the requirements: +** Preconditions: ** -** 1. A read/write cursor pointing to pTab, the table containing the row -** to be deleted, must be opened as cursor number $iCur. +** 1. iDataCur is an open cursor on the btree that is the canonical data +** store for the table. (This will be either the table itself, +** in the case of a rowid table, or the PRIMARY KEY index in the case +** of a WITHOUT ROWID table.) ** ** 2. Read/write cursors for all indices of pTab must be open as -** cursor number base+i for the i-th index. +** cursor number iIdxCur+i for the i-th index. ** -** 3. The record number of the row to be deleted must be stored in -** memory cell iRowid. -** -** This routine generates code to remove both the table record and all -** index entries that point to that record. +** 3. The primary key for the row to be deleted must be stored in a +** sequence of nPk memory cells starting at iPk. If nPk==0 that means +** that a search record formed from OP_MakeRecord is contained in the +** single memory location iPk. */ SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ Table *pTab, /* Table containing the row to be deleted */ - int iCur, /* Cursor number for the table */ - int iRowid, /* Memory cell that contains the rowid to delete */ - int count, /* If non-zero, increment the row change counter */ Trigger *pTrigger, /* List of triggers to (potentially) fire */ - int onconf /* Default ON CONFLICT policy for triggers */ + int iDataCur, /* Cursor from which column data is extracted */ + int iIdxCur, /* First index cursor */ + int iPk, /* First memory cell containing the PRIMARY KEY */ + i16 nPk, /* Number of PRIMARY KEY memory cells */ + u8 count, /* If non-zero, increment the row change counter */ + u8 onconf, /* Default ON CONFLICT policy for triggers */ + u8 bNoSeek /* iDataCur is already pointing to the row to delete */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ int iLabel; /* Label resolved to end of generated code */ + u8 opSeek; /* Seek opcode */ /* Vdbe is guaranteed to have been allocated by this stage. */ assert( v ); + VdbeModuleComment((v, "BEGIN: GenRowDel(%d,%d,%d,%d)", + iDataCur, iIdxCur, iPk, (int)nPk)); /* Seek cursor iCur to the row to delete. If this row no longer exists ** (this can happen if a trigger program has already deleted it), do ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, iLabel, iRowid); + opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + if( !bNoSeek ){ + sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); + VdbeCoverageIf(v, opSeek==OP_NotExists); + VdbeCoverageIf(v, opSeek==OP_NotFound); + } /* If there are any triggers to fire, allocate a range of registers to ** use for the old.* references in the triggers. */ if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ u32 mask; /* Mask of OLD.* columns in use */ int iCol; /* Iterator used while populating OLD.* */ + int addrStart; /* Start of BEFORE trigger programs */ /* TODO: Could use temporary registers here. Also could attempt to ** avoid copying the contents of the rowid register. */ @@ -82066,36 +93970,44 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( /* Populate the OLD.* pseudo-table register array. These values will be ** used by any BEFORE and AFTER triggers that exist. */ - sqlite3VdbeAddOp2(v, OP_Copy, iRowid, iOld); + sqlite3VdbeAddOp2(v, OP_Copy, iPk, iOld); for(iCol=0; iColnCol; iCol++){ - if( mask==0xffffffff || mask&(1<pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, 0); - sqlite3VdbeAddOp2(v, OP_Delete, iCur, (count?OPFLAG_NCHANGE:0)); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); if( count ){ sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } @@ -82104,7 +94016,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to ** handle rows (possibly in other tables) that refer via a foreign key ** to the row just deleted. */ - sqlite3FkActions(pParse, pTab, 0, iOld); + sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); /* Invoke AFTER DELETE trigger programs. */ sqlite3CodeRowTrigger(pParse, pTrigger, @@ -82115,58 +94027,98 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** trigger programs were invoked. Or if a trigger program throws a ** RAISE(IGNORE) exception. */ sqlite3VdbeResolveLabel(v, iLabel); + VdbeModuleComment((v, "END: GenRowDel()")); } /* ** This routine generates VDBE code that causes the deletion of all -** index entries associated with a single row of a single table. +** index entries associated with a single row of a single table, pTab ** -** The VDBE must be in a particular state when this routine is called. -** These are the requirements: +** Preconditions: ** -** 1. A read/write cursor pointing to pTab, the table containing the row -** to be deleted, must be opened as cursor number "iCur". +** 1. A read/write cursor "iDataCur" must be open on the canonical storage +** btree for the table pTab. (This will be either the table itself +** for rowid tables or to the primary key index for WITHOUT ROWID +** tables.) ** ** 2. Read/write cursors for all indices of pTab must be open as -** cursor number iCur+i for the i-th index. +** cursor number iIdxCur+i for the i-th index. (The pTab->pIndex +** index is the 0-th index.) ** -** 3. The "iCur" cursor must be pointing to the row that is to be -** deleted. +** 3. The "iDataCur" cursor must be already be positioned on the row +** that is to be deleted. */ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( Parse *pParse, /* Parsing and code generating context */ Table *pTab, /* Table containing the row to be deleted */ - int iCur, /* Cursor number for the table */ + int iDataCur, /* Cursor of table holding data. */ + int iIdxCur, /* First index cursor */ int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ ){ - int i; - Index *pIdx; - int r1; + int i; /* Index loop counter */ + int r1 = -1; /* Register holding an index key */ + int iPartIdxLabel; /* Jump destination for skipping partial index entries */ + Index *pIdx; /* Current index */ + Index *pPrior = 0; /* Prior index */ + Vdbe *v; /* The prepared statement under construction */ + Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ - for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - if( aRegIdx!=0 && aRegIdx[i-1]==0 ) continue; - r1 = sqlite3GenerateIndexKey(pParse, pIdx, iCur, 0, 0); - sqlite3VdbeAddOp3(pParse->pVdbe, OP_IdxDelete, iCur+i, r1,pIdx->nColumn+1); + v = pParse->pVdbe; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + assert( iIdxCur+i!=iDataCur || pPk==pIdx ); + if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; + if( pIdx==pPk ) continue; + VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, + &iPartIdxLabel, pPrior, r1); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + pPrior = pIdx; } } /* -** Generate code that will assemble an index key and put it in register +** Generate code that will assemble an index key and stores it in register ** regOut. The key with be for index pIdx which is an index on pTab. ** iCur is the index of a cursor open on the pTab table and pointing to -** the entry that needs indexing. +** the entry that needs indexing. If pTab is a WITHOUT ROWID table, then +** iCur must be the cursor of the PRIMARY KEY index. ** ** Return a register number which is the first in a block of ** registers that holds the elements of the index key. The ** block of registers has already been deallocated by the time ** this routine returns. +** +** If *piPartIdxLabel is not NULL, fill it in with a label and jump +** to that label if pIdx is a partial index that should be skipped. +** The label should be resolved using sqlite3ResolvePartIdxLabel(). +** A partial index should be skipped if its WHERE clause evaluates +** to false or null. If pIdx is not a partial index, *piPartIdxLabel +** will be set to zero which is an empty label that is ignored by +** sqlite3ResolvePartIdxLabel(). +** +** The pPrior and regPrior parameters are used to implement a cache to +** avoid unnecessary register loads. If pPrior is not NULL, then it is +** a pointer to a different index for which an index key has just been +** computed into register regPrior. If the current pIdx index is generating +** its key into the same sequence of registers and if pPrior and pIdx share +** a column in common, then the register corresponding to that column already +** holds the correct value and the loading of that register is skipped. +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** on a table with multiple indices, and especially with the ROWID or +** PRIMARY KEY columns of the index. */ SQLITE_PRIVATE int sqlite3GenerateIndexKey( - Parse *pParse, /* Parsing context */ - Index *pIdx, /* The index for which to generate a key */ - int iCur, /* Cursor number for the pIdx->pTable table */ - int regOut, /* Write the new index key to this register */ - int doMakeRec /* Run the OP_MakeRecord instruction if true */ + Parse *pParse, /* Parsing context */ + Index *pIdx, /* The index for which to generate a key */ + int iDataCur, /* Cursor number from which to take column data */ + int regOut, /* Put the new key into this register if not 0 */ + int prefixOnly, /* Compute only a unique prefix of the key */ + int *piPartIdxLabel, /* OUT: Jump to this label to skip partial index */ + Index *pPrior, /* Previously generated index key */ + int regPrior /* Register holding previous generated key */ ){ Vdbe *v = pParse->pVdbe; int j; @@ -82174,32 +94126,51 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( int regBase; int nCol; - nCol = pIdx->nColumn; - regBase = sqlite3GetTempRange(pParse, nCol+1); - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regBase+nCol); + if( piPartIdxLabel ){ + if( pIdx->pPartIdxWhere ){ + *piPartIdxLabel = sqlite3VdbeMakeLabel(v); + pParse->iPartIdxTab = iDataCur; + sqlite3ExprCachePush(pParse); + sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); + }else{ + *piPartIdxLabel = 0; + } + } + nCol = (prefixOnly && pIdx->uniqNotNull) ? pIdx->nKeyCol : pIdx->nColumn; + regBase = sqlite3GetTempRange(pParse, nCol); + if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]; - if( idx==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regBase+nCol, regBase+j); - }else{ - sqlite3VdbeAddOp3(v, OP_Column, iCur, idx, regBase+j); - sqlite3ColumnDefault(v, pTab, idx, -1); - } + if( pPrior && pPrior->aiColumn[j]==pIdx->aiColumn[j] ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], + regBase+j); + /* If the column affinity is REAL but the number is an integer, then it + ** might be stored in the table as an integer (using a compact + ** representation) then converted to REAL by an OP_RealAffinity opcode. + ** But we are getting ready to store this value back into an index, where + ** it should be converted by to INTEGER again. So omit the OP_RealAffinity + ** opcode if it is present */ + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); } - if( doMakeRec ){ - const char *zAff; - if( pTab->pSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ - zAff = 0; - }else{ - zAff = sqlite3IndexAffinityStr(v, pIdx); - } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut); - sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); + if( regOut ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); } - sqlite3ReleaseTempRange(pParse, regBase, nCol+1); + sqlite3ReleaseTempRange(pParse, regBase, nCol); return regBase; } +/* +** If a prior call to sqlite3GenerateIndexKey() generated a jump-over label +** because it was a partial index, then this routine should be called to +** resolve that label. +*/ +SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ + if( iLabel ){ + sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); + sqlite3ExprCachePop(pParse); + } +} + /************** End of delete.c **********************************************/ /************** Begin file func.c ********************************************/ /* @@ -82213,19 +94184,29 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C functions that implement various SQL -** functions of SQLite. -** -** There is only one exported symbol in this file - the function -** sqliteRegisterBuildinFunctions() found at the bottom of the file. -** All other code has file scope. +** This file contains the C-language implementations for many of the SQL +** functions of SQLite. (Some function, and in particular the date and +** time functions, are implemented separately.) */ +/* #include */ +/* #include */ /* ** Return the collating function associated with a function. */ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ - return context->pColl; + VdbeOp *pOp = &context->pVdbe->aOp[context->iOp-1]; + assert( pOp->opcode==OP_CollSeq ); + assert( pOp->p4type==P4_COLLSEQ ); + return pOp->p4.pColl; +} + +/* +** Indicate that the accumulator load should be skipped on this +** iteration of the aggregate loop. +*/ +static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){ + context->skipFlag = 1; } /* @@ -82329,9 +94310,9 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ case SQLITE_INTEGER: { i64 iVal = sqlite3_value_int64(argv[0]); if( iVal<0 ){ - if( (iVal<<1)==0 ){ - /* IMP: R-35460-15084 If X is the integer -9223372036854775807 then - ** abs(X) throws an integer overflow error since there is no + if( iVal==SMALLEST_INT64 ){ + /* IMP: R-31676-45509 If X is the integer -9223372036854775808 + ** then abs(X) throws an integer overflow error since there is no ** equivalent positive 64-bit two complement value. */ sqlite3_result_error(context, "integer overflow", -1); return; @@ -82360,6 +94341,82 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } +/* +** Implementation of the instr() function. +** +** instr(haystack,needle) finds the first occurrence of needle +** in haystack and returns the number of previous characters plus 1, +** or 0 if needle does not occur within haystack. +** +** If both haystack and needle are BLOBs, then the result is one more than +** the number of bytes in haystack prior to the first occurrence of needle, +** or 0 if needle never occurs in haystack. +*/ +static void instrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zHaystack; + const unsigned char *zNeedle; + int nHaystack; + int nNeedle; + int typeHaystack, typeNeedle; + int N = 1; + int isText; + + UNUSED_PARAMETER(argc); + typeHaystack = sqlite3_value_type(argv[0]); + typeNeedle = sqlite3_value_type(argv[1]); + if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; + nHaystack = sqlite3_value_bytes(argv[0]); + nNeedle = sqlite3_value_bytes(argv[1]); + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else{ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + } + while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; + sqlite3_result_int(context, N); +} + +/* +** Implementation of the printf() function. +*/ +static void printfFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + PrintfArguments x; + StrAccum str; + const char *zFormat; + int n; + + if( argc>=1 && (zFormat = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + x.nArg = argc-1; + x.nUsed = 0; + x.apArg = argv+1; + sqlite3StrAccumInit(&str, 0, 0, SQLITE_MAX_LENGTH); + str.db = sqlite3_context_db_handle(context); + sqlite3XPrintf(&str, SQLITE_PRINTF_SQLFUNC, zFormat, &x); + n = str.nChar; + sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, + SQLITE_DYNAMIC); + } +} + /* ** Implementation of the substr() function. ** @@ -82370,7 +94427,7 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ ** ** If p1 is negative, then we begin abs(p1) from the end of x[]. ** -** If p2 is negative, return the p2 characters preceeding p1. +** If p2 is negative, return the p2 characters preceding p1. */ static void substrFunc( sqlite3_context *context, @@ -82444,13 +94501,14 @@ static void substrFunc( for(z2=z; *z2 && p2; p2--){ SQLITE_SKIP_UTF8(z2); } - sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); + sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, + SQLITE_UTF8); }else{ if( p1+p2>len ){ p2 = len-p1; if( p2<0 ) p2 = 0; } - sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); + sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); } } @@ -82509,7 +94567,7 @@ static void *contextMalloc(sqlite3_context *context, i64 nByte){ sqlite3_result_error_toobig(context); z = 0; }else{ - z = sqlite3Malloc((int)nByte); + z = sqlite3Malloc(nByte); if( !z ){ sqlite3_result_error_nomem(context); } @@ -82532,16 +94590,15 @@ static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ if( z2 ){ z1 = contextMalloc(context, ((i64)n)+1); if( z1 ){ - memcpy(z1, z2, n+1); - for(i=0; z1[i]; i++){ - z1[i] = (char)sqlite3Toupper(z1[i]); + for(i=0; imatchOne; - u8 matchAll = pInfo->matchAll; - u8 matchSet = pInfo->matchSet; - u8 noCase = pInfo->noCase; - int prevEscape = 0; /* True if the previous character was 'escape' */ + u32 c, c2; /* Next pattern and input string chars */ + u32 matchOne = pInfo->matchOne; /* "?" or "_" */ + u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u32 matchOther; /* "[" or the escape character */ + u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ + const u8 *zEscaped = 0; /* One past the last escaped input char */ + + /* The GLOB operator does not have an ESCAPE clause. And LIKE does not + ** have the matchSet operator. So we either have to look for one or + ** the other, never both. Hence the single variable matchOther is used + ** to store the one we have to look for. + */ + matchOther = esc ? esc : pInfo->matchSet; - while( (c = sqlite3Utf8Read(zPattern,&zPattern))!=0 ){ - if( !prevEscape && c==matchAll ){ - while( (c=sqlite3Utf8Read(zPattern,&zPattern)) == matchAll + while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ + if( c==matchAll ){ /* Match "*" */ + /* Skip over multiple "*" characters in the pattern. If there + ** are also "?" characters, skip those as well, but consume a + ** single character of the input string for each "?" skipped */ + while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ - if( c==matchOne && sqlite3Utf8Read(zString, &zString)==0 ){ + if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } } if( c==0 ){ - return 1; - }else if( c==esc ){ - c = sqlite3Utf8Read(zPattern, &zPattern); - if( c==0 ){ - return 0; - } - }else if( c==matchSet ){ - assert( esc==0 ); /* This is GLOB, not LIKE */ - assert( matchSet<0x80 ); /* '[' is a single-byte character */ - while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ - SQLITE_SKIP_UTF8(zString); - } - return *zString!=0; - } - while( (c2 = sqlite3Utf8Read(zString,&zString))!=0 ){ - if( noCase ){ - GlogUpperToLower(c2); - GlogUpperToLower(c); - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); - GlogUpperToLower(c2); - } + return 1; /* "*" at the end of the pattern matches */ + }else if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; }else{ - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); + /* "[...]" immediately follows the "*". We have to do a slow + ** recursive search in this case, but it is an unusual case. */ + assert( matchOther<0x80 ); /* '[' is a single-byte character */ + while( *zString + && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ + SQLITE_SKIP_UTF8(zString); } + return *zString!=0; + } + } + + /* At this point variable c contains the first character of the + ** pattern string past the "*". Search in the input string for the + ** first matching character and recursively contine the match from + ** that point. + ** + ** For a case-insensitive search, set variable cx to be the same as + ** c but in the other case and search the input string for either + ** c or cx. + */ + if( c<=0x80 ){ + u32 cx; + if( noCase ){ + cx = sqlite3Toupper(c); + c = sqlite3Tolower(c); + }else{ + cx = c; + } + while( (c2 = *(zString++))!=0 ){ + if( c2!=c && c2!=cx ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + } + }else{ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ + if( c2!=c ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } - if( c2==0 ) return 0; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( !prevEscape && c==matchOne ){ - if( sqlite3Utf8Read(zString, &zString)==0 ){ - return 0; - } - }else if( c==matchSet ){ - u32 prior_c = 0; - assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ - seen = 0; - invert = 0; - c = sqlite3Utf8Read(zString, &zString); - if( c==0 ) return 0; - c2 = sqlite3Utf8Read(zPattern, &zPattern); - if( c2=='^' ){ - invert = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); - } - if( c2==']' ){ - if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); - } - while( c2 && c2!=']' ){ - if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(zPattern, &zPattern); - if( c>=prior_c && c<=c2 ) seen = 1; - prior_c = 0; - }else{ - if( c==c2 ){ - seen = 1; - } - prior_c = c2; - } - c2 = sqlite3Utf8Read(zPattern, &zPattern); - } - if( c2==0 || (seen ^ invert)==0 ){ - return 0; - } - }else if( esc==c && !prevEscape ){ - prevEscape = 1; - }else{ - c2 = sqlite3Utf8Read(zString, &zString); - if( noCase ){ - GlogUpperToLower(c); - GlogUpperToLower(c2); - } - if( c!=c2 ){ - return 0; - } - prevEscape = 0; } + if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; + zEscaped = zPattern; + }else{ + u32 prior_c = 0; + int seen = 0; + int invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return 0; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return 0; + } + continue; + } + } + c2 = sqlite3Utf8Read(&zString); + if( c==c2 ) continue; + if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ + continue; + } + if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; + return 0; } return *zString==0; } +/* +** The sqlite3_strglob() interface. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + /* ** Count the number of times that the LIKE operator (or GLOB which is ** just a variation of LIKE) gets called. This is used for testing @@ -82914,7 +94987,7 @@ static void likeFunc( "ESCAPE expression must be a single character", -1); return; } - escape = sqlite3Utf8Read(zEsc, &zEsc); + escape = sqlite3Utf8Read(&zEsc); } if( zA && zB ){ struct compareInfo *pInfo = sqlite3_user_data(context); @@ -83043,10 +95116,6 @@ static const char hexdigits[] = { }; /* -** EXPERIMENTAL - This is not an official function. The interface may -** change. This function may disappear. Do not write code that depends -** on this function. -** ** Implementation of the QUOTE() function. This function takes a single ** argument. If the argument is numeric, the return value is the same as ** the argument. If the argument is NULL, the return value is the string @@ -83057,8 +95126,19 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); UNUSED_PARAMETER(argc); switch( sqlite3_value_type(argv[0]) ){ - case SQLITE_INTEGER: case SQLITE_FLOAT: { + double r1, r2; + char zBuf[50]; + r1 = sqlite3_value_double(argv[0]); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); + sqlite3AtoF(zBuf, &r2, 20, SQLITE_UTF8); + if( r1!=r2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.20e", r1); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + break; + } + case SQLITE_INTEGER: { sqlite3_result_value(context, argv[0]); break; } @@ -83114,6 +95194,62 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } +/* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc( argc*4+1 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); +} + /* ** The hex() function. Interpret the argument as a blob. Return ** a hexadecimal rendering as text. @@ -83168,7 +95304,7 @@ static void zeroblobFunc( /* ** The replace() function. Three arguments are all strings: call ** them A, B, and C. The result is also a string which is derived -** from A by replacing every occurance of B with C. The match +** from A by replacing every occurrence of B with C. The match ** must be exact. Collating sequences are not used. */ static void replaceFunc( @@ -83536,11 +95672,12 @@ static void minmaxStep( Mem *pBest; UNUSED_PARAMETER(NotUsed); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); if( !pBest ) return; - if( pBest->flags ){ + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); + }else if( pBest->flags ){ int max; int cmp; CollSeq *pColl = sqlite3GetFuncCollSeq(context); @@ -83556,8 +95693,11 @@ static void minmaxStep( cmp = sqlite3MemCompare(pBest, pArg, pColl); if( (max && cmp<0) || (!max && cmp>0) ){ sqlite3VdbeMemCopy(pBest, pArg); + }else{ + sqlite3SkipAccumulatorLoad(context); } }else{ + pBest->db = sqlite3_context_db_handle(context); sqlite3VdbeMemCopy(pBest, pArg); } } @@ -83565,7 +95705,7 @@ static void minMaxFinalize(sqlite3_context *context){ sqlite3_value *pRes; pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); if( pRes ){ - if( ALWAYS(pRes->flags) ){ + if( pRes->flags ){ sqlite3_result_value(context, pRes); } sqlite3VdbeMemRelease(pRes); @@ -83601,20 +95741,20 @@ static void groupConcatStep( zSep = ","; nSep = 1; } - sqlite3StrAccumAppend(pAccum, zSep, nSep); + if( nSep ) sqlite3StrAccumAppend(pAccum, zSep, nSep); } zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); - sqlite3StrAccumAppend(pAccum, zVal, nVal); + if( zVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); } } static void groupConcatFinalize(sqlite3_context *context){ StrAccum *pAccum; pAccum = sqlite3_aggregate_context(context, 0); if( pAccum ){ - if( pAccum->tooBig ){ + if( pAccum->accError==STRACCUM_TOOBIG ){ sqlite3_result_error_toobig(context); - }else if( pAccum->mallocFailed ){ + }else if( pAccum->accError==STRACCUM_NOMEM ){ sqlite3_result_error_nomem(context); }else{ sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, @@ -83644,7 +95784,7 @@ static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), 2, SQLITE_UTF8, 0); if( ALWAYS(pDef) ){ - pDef->flags = flagVal; + pDef->funcFlags |= flagVal; } } @@ -83688,7 +95828,7 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas pDef = sqlite3FindFunction(db, pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), 2, SQLITE_UTF8, 0); - if( NEVER(pDef==0) || (pDef->flags & SQLITE_FUNC_LIKE)==0 ){ + if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } @@ -83700,12 +95840,12 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); - *pIsNocase = (pDef->flags & SQLITE_FUNC_CASE)==0; + *pIsNocase = (pDef->funcFlags & SQLITE_FUNC_CASE)==0; return 1; } /* -** All all of the FuncDef structures in the aBuiltinFunc[] array above +** All of the FuncDef structures in the aBuiltinFunc[] array above ** to the global function hash table. This occurs at start-time (as ** a consequence of calling sqlite3_initialize()). ** @@ -83729,14 +95869,20 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(trim, 2, 3, 0, trimFunc ), FUNCTION(min, -1, 0, 1, minmaxFunc ), FUNCTION(min, 0, 0, 1, 0 ), - AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + AGGREGATE2(min, 1, 0, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), - AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), - FUNCTION(typeof, 1, 0, 0, typeofFunc ), - FUNCTION(length, 1, 0, 0, lengthFunc ), + AGGREGATE2(max, 1, 1, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION(instr, 2, 0, 0, instrFunc ), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(printf, -1, 0, 0, printfFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), @@ -83746,25 +95892,29 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), -/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, + FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), -/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, - FUNCTION(random, 0, 0, 0, randomFunc ), - FUNCTION(randomblob, 1, 0, 0, randomBlob ), + FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + VFUNCTION(random, 0, 0, 0, randomFunc ), + VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), - FUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), - FUNCTION(changes, 0, 0, 0, changes ), - FUNCTION(total_changes, 0, 0, 0, total_changes ), + VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), + VFUNCTION(changes, 0, 0, 0, changes ), + VFUNCTION(total_changes, 0, 0, 0, total_changes ), FUNCTION(replace, 3, 0, 0, replaceFunc ), FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), #ifdef SQLITE_SOUNDEX @@ -83777,8 +95927,8 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), - /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, + AGGREGATE2(count, 0, 0, 0, countStep, countFinalize, + SQLITE_FUNC_COUNT ), AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), @@ -83804,6 +95954,9 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ #ifndef SQLITE_OMIT_ALTERTABLE sqlite3AlterFunctions(); #endif +#if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) + sqlite3AnalyzeFunctions(); +#endif } /************** End of func.c ************************************************/ @@ -83830,8 +95983,9 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** -------------------------- ** ** Foreign keys in SQLite come in two flavours: deferred and immediate. -** If an immediate foreign key constraint is violated, SQLITE_CONSTRAINT -** is returned and the current statement transaction rolled back. If a +** If an immediate foreign key constraint is violated, +** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current +** statement transaction rolled back. If a ** deferred foreign key constraint is violated, no action is taken ** immediately. However if the application attempts to commit the ** transaction before fixing the constraint violation, the attempt fails. @@ -83895,7 +96049,8 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** Immediate constraints are usually handled similarly. The only difference ** is that the counter used is stored as part of each individual statement ** object (struct Vdbe). If, after the statement has run, its immediate -** constraint counter is greater than zero, it returns SQLITE_CONSTRAINT +** constraint counter is greater than zero, +** it returns SQLITE_CONSTRAINT_FOREIGNKEY ** and the statement transaction is rolled back. An exception is an INSERT ** statement that inserts a single row only (no triggers). In this case, ** instead of using a counter, an exception is thrown immediately if the @@ -83951,7 +96106,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** A foreign key constraint requires that the key columns in the parent ** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. ** Given that pParent is the parent table for foreign key constraint pFKey, -** search the schema a unique index on the parent key columns. +** search the schema for a unique index on the parent key columns. ** ** If successful, zero is returned. If the parent key is an INTEGER PRIMARY ** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx @@ -83980,14 +96135,14 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** ** 4) No parent key columns were provided explicitly as part of the ** foreign key definition, and the PRIMARY KEY of the parent table -** consists of a a different number of columns to the child key in +** consists of a different number of columns to the child key in ** the child table. ** ** then non-zero is returned, and a "foreign key mismatch" error loaded ** into pParse. If an OOM error occurs, non-zero is returned and the ** pParse->db->mallocFailed flag is set. */ -static int locateFkeyIndex( +SQLITE_PRIVATE int sqlite3FkLocateIndex( Parse *pParse, /* Parse context to store any error in */ Table *pParent, /* Parent table of FK constraint pFKey */ FKey *pFKey, /* Foreign key to find index for */ @@ -84032,7 +96187,7 @@ static int locateFkeyIndex( } for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->nColumn==nCol && pIdx->onError!=OE_None ){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) ){ /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number ** of columns. If each indexed column corresponds to a foreign key ** column of pFKey, then this index is a winner. */ @@ -84040,8 +96195,8 @@ static int locateFkeyIndex( if( zKey==0 ){ /* If zKey is NULL, then this foreign key is implicitly mapped to ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be - ** identified by the test (Index.autoIndex==2). */ - if( pIdx->autoIndex==2 ){ + ** identified by the test. */ + if( IsPrimaryKeyIndex(pIdx) ){ if( aiCol ){ int i; for(i=0; iaCol[i].iFrom; @@ -84055,7 +96210,7 @@ static int locateFkeyIndex( ** the default collation sequences for each column. */ int i, j; for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ + i16 iCol = pIdx->aiColumn[i]; /* Index of column in parent tbl */ char *zDfltColl; /* Def. collation for column */ char *zIdxCol; /* Name of indexed column */ @@ -84084,7 +96239,9 @@ static int locateFkeyIndex( if( !pIdx ){ if( !pParse->disableTriggers ){ - sqlite3ErrorMsg(pParse, "foreign key mismatch"); + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); } sqlite3DbFree(pParse->db, aiCol); return 1; @@ -84145,10 +96302,11 @@ static void fkLookupParent( ** search for a matching row in the parent table. */ if( nIncr<0 ){ sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); + VdbeCoverage(v); } for(i=0; inCol; i++){ int iReg = aiCol[i] + regData + 1; - sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); } if( isIgnore==0 ){ @@ -84165,17 +96323,19 @@ static void fkLookupParent( ** will have INTEGER affinity applied to it, which may not be correct. */ sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); + VdbeCoverage(v); /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not ** increment the constraint-counter. */ if( pTab==pFKey->pFrom && nIncr==1 ){ - sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); + sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); } sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); sqlite3VdbeJumpHere(v, iMustBeInt); @@ -84184,10 +96344,9 @@ static void fkLookupParent( int nCol = pFKey->nCol; int regTemp = sqlite3GetTempRange(pParse, nCol); int regRec = sqlite3GetTempReg(pParse); - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); - sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); for(i=0; iisDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){ + if( !pFKey->isDeferred && !(pParse->db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel + && !pParse->isMultiWrite + ){ /* Special case: If this is an INSERT statement that will insert exactly ** one row into the table, raise a constraint immediately instead of ** incrementing a counter. This is necessary as the VM code is being ** generated for will not open a statement transaction. */ assert( nIncr==1 ); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC - ); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ sqlite3ParseToplevel(pParse)->mayAbort = 1; @@ -84247,6 +96408,62 @@ static void fkLookupParent( sqlite3VdbeAddOp1(v, OP_Close, iCur); } + +/* +** Return an Expr object that refers to a memory register corresponding +** to column iCol of table pTab. +** +** regBase is the first of an array of register that contains the data +** for pTab. regBase itself holds the rowid. regBase+1 holds the first +** column. regBase+2 holds the second column, and so forth. +*/ +static Expr *exprTableRegister( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* The table whose content is at r[regBase]... */ + int regBase, /* Contents of table pTab */ + i16 iCol /* Which column of pTab is desired */ +){ + Expr *pExpr; + Column *pCol; + const char *zColl; + sqlite3 *db = pParse->db; + + pExpr = sqlite3Expr(db, TK_REGISTER, 0); + if( pExpr ){ + if( iCol>=0 && iCol!=pTab->iPKey ){ + pCol = &pTab->aCol[iCol]; + pExpr->iTable = regBase + iCol + 1; + pExpr->affinity = pCol->affinity; + zColl = pCol->zColl; + if( zColl==0 ) zColl = db->pDfltColl->zName; + pExpr = sqlite3ExprAddCollateString(pParse, pExpr, zColl); + }else{ + pExpr->iTable = regBase; + pExpr->affinity = SQLITE_AFF_INTEGER; + } + } + return pExpr; +} + +/* +** Return an Expr object that refers to column iCol of table pTab which +** has cursor iCur. +*/ +static Expr *exprTableColumn( + sqlite3 *db, /* The database connection */ + Table *pTab, /* The table whose column is desired */ + int iCursor, /* The open cursor on the table */ + i16 iCol /* The column that is wanted */ +){ + Expr *pExpr = sqlite3Expr(db, TK_COLUMN, 0); + if( pExpr ){ + pExpr->pTab = pTab; + pExpr->iTable = iCursor; + pExpr->iColumn = iCol; + } + return pExpr; +} + /* ** This function is called to generate code executed when a row is deleted ** from the parent table of foreign key constraint pFKey and, if pFKey is @@ -84262,13 +96479,13 @@ static void fkLookupParent( ** -------------------------------------------------------------------------- ** DELETE immediate Increment the "immediate constraint counter". ** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "foreign key constraint failed" exception. +** throw a "FOREIGN KEY constraint failed" exception. ** ** INSERT immediate Decrement the "immediate constraint counter". ** ** DELETE deferred Increment the "deferred constraint counter". ** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "foreign key constraint failed" exception. +** throw a "FOREIGN KEY constraint failed" exception. ** ** INSERT deferred Decrement the "deferred constraint counter". ** @@ -84277,12 +96494,12 @@ static void fkLookupParent( */ static void fkScanChildren( Parse *pParse, /* Parse context */ - SrcList *pSrc, /* SrcList containing the table to scan */ - Table *pTab, - Index *pIdx, /* Foreign key index */ - FKey *pFKey, /* Foreign key relationship */ + SrcList *pSrc, /* The child table to be scanned */ + Table *pTab, /* The parent table */ + Index *pIdx, /* Index on parent covering the foreign key */ + FKey *pFKey, /* The foreign key linking pSrc to pTab */ int *aiCol, /* Map from pIdx cols to child table cols */ - int regData, /* Referenced table data starts here */ + int regData, /* Parent row data starts here */ int nIncr /* Amount to increment deferred counter by */ ){ sqlite3 *db = pParse->db; /* Database handle */ @@ -84293,10 +96510,14 @@ static void fkScanChildren( int iFkIfZero = 0; /* Address of OP_FkIfZero */ Vdbe *v = sqlite3GetVdbe(pParse); - assert( !pIdx || pIdx->pTable==pTab ); + assert( pIdx==0 || pIdx->pTable==pTab ); + assert( pIdx==0 || pIdx->nKeyCol==pFKey->nCol ); + assert( pIdx!=0 || pFKey->nCol==1 ); + assert( pIdx!=0 || HasRowid(pTab) ); if( nIncr<0 ){ iFkIfZero = sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, 0); + VdbeCoverage(v); } /* Create an Expr object representing an SQL expression like: @@ -84311,26 +96532,11 @@ static void fkScanChildren( Expr *pLeft; /* Value from parent table row */ Expr *pRight; /* Column ref to child table */ Expr *pEq; /* Expression (pLeft = pRight) */ - int iCol; /* Index of column in child table */ + i16 iCol; /* Index of column in child table */ const char *zCol; /* Name of column in child table */ - pLeft = sqlite3Expr(db, TK_REGISTER, 0); - if( pLeft ){ - /* Set the collation sequence and affinity of the LHS of each TK_EQ - ** expression to the parent key column defaults. */ - if( pIdx ){ - Column *pCol; - iCol = pIdx->aiColumn[i]; - pCol = &pTab->aCol[iCol]; - if( pTab->iPKey==iCol ) iCol = -1; - pLeft->iTable = regData+iCol+1; - pLeft->affinity = pCol->affinity; - pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); - }else{ - pLeft->iTable = regData; - pLeft->affinity = SQLITE_AFF_INTEGER; - } - } + iCol = pIdx ? pIdx->aiColumn[i] : -1; + pLeft = exprTableRegister(pParse, pTab, regData, iCol); iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iCol>=0 ); zCol = pFKey->pFrom->aCol[iCol].zName; @@ -84339,24 +96545,39 @@ static void fkScanChildren( pWhere = sqlite3ExprAnd(db, pWhere, pEq); } - /* If the child table is the same as the parent table, and this scan - ** is taking place as part of a DELETE operation (operation D.2), omit the - ** row being deleted from the scan by adding ($rowid != rowid) to the WHERE - ** clause, where $rowid is the rowid of the row being deleted. */ + /* If the child table is the same as the parent table, then add terms + ** to the WHERE clause that prevent this entry from being scanned. + ** The added WHERE clause terms are like this: + ** + ** $current_rowid!=rowid + ** NOT( $current_a==a AND $current_b==b AND ... ) + ** + ** The first form is used for rowid tables. The second form is used + ** for WITHOUT ROWID tables. In the second form, the primary key is + ** (a,b,...) + */ if( pTab==pFKey->pFrom && nIncr>0 ){ - Expr *pEq; /* Expression (pLeft = pRight) */ + Expr *pNe; /* Expression (pLeft != pRight) */ Expr *pLeft; /* Value from parent table row */ Expr *pRight; /* Column ref to child table */ - pLeft = sqlite3Expr(db, TK_REGISTER, 0); - pRight = sqlite3Expr(db, TK_COLUMN, 0); - if( pLeft && pRight ){ - pLeft->iTable = regData; - pLeft->affinity = SQLITE_AFF_INTEGER; - pRight->iTable = pSrc->a[0].iCursor; - pRight->iColumn = -1; + if( HasRowid(pTab) ){ + pLeft = exprTableRegister(pParse, pTab, regData, -1); + pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, -1); + pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight, 0); + }else{ + Expr *pEq, *pAll = 0; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pIdx!=0 ); + for(i=0; inKeyCol; i++){ + i16 iCol = pIdx->aiColumn[i]; + pLeft = exprTableRegister(pParse, pTab, regData, iCol); + pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); + pAll = sqlite3ExprAnd(db, pAll, pEq); + } + pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0, 0); } - pEq = sqlite3PExpr(pParse, TK_NE, pLeft, pRight, 0); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + pWhere = sqlite3ExprAnd(db, pWhere, pNe); } /* Resolve the references in the WHERE clause. */ @@ -84369,7 +96590,7 @@ static void fkScanChildren( ** clause. If the constraint is not deferred, throw an exception for ** each row found. Otherwise, for deferred constraints, increment the ** deferred constraint counter by nIncr for each row selected. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); if( nIncr>0 && pFKey->isDeferred==0 ){ sqlite3ParseToplevel(pParse)->mayAbort = 1; } @@ -84386,8 +96607,8 @@ static void fkScanChildren( } /* -** This function returns a pointer to the head of a linked list of FK -** constraints for which table pTab is the parent table. For example, +** This function returns a linked list of FKey objects (connected by +** FKey.pNextTo) holding all children of table pTab. For example, ** given the following schema: ** ** CREATE TABLE t1(a PRIMARY KEY); @@ -84400,8 +96621,7 @@ static void fkScanChildren( ** table). */ SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ - int nName = sqlite3Strlen30(pTab->zName); - return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName, nName); + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName); } /* @@ -84455,11 +96675,11 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa ** when this statement is run. */ FKey *p; for(p=pTab->pFKey; p; p=p->pNextFrom){ - if( p->isDeferred ) break; + if( p->isDeferred || (db->flags & SQLITE_DeferFKs) ) break; } if( !p ) return; iSkip = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); VdbeCoverage(v); } pParse->disableTriggers = 1; @@ -84469,11 +96689,18 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa /* If the DELETE has generated immediate foreign key constraint ** violations, halt the VDBE and return an error at this point, before ** any modifications to the schema are made. This is because statement - ** transactions are not able to rollback schema changes. */ - sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC - ); + ** transactions are not able to rollback schema changes. + ** + ** If the SQLITE_DeferFKs flag is set, then this is not required, as + ** the statement transaction will not be rolled back even if FK + ** constraints are violated. + */ + if( (db->flags & SQLITE_DeferFKs)==0 ){ + sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); + } if( iSkip ){ sqlite3VdbeResolveLabel(v, iSkip); @@ -84481,6 +96708,70 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa } } + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the child table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** child key for FK constraint *p are modified. +*/ +static int fkChildIsModified( + Table *pTab, /* Table being updated */ + FKey *p, /* Foreign key for which pTab is the child */ + int *aChange, /* Array indicating modified columns */ + int bChngRowid /* True if rowid is modified by this update */ +){ + int i; + for(i=0; inCol; i++){ + int iChildKey = p->aCol[i].iFrom; + if( aChange[iChildKey]>=0 ) return 1; + if( iChildKey==pTab->iPKey && bChngRowid ) return 1; + } + return 0; +} + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the parent table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** parent key for FK constraint *p are modified. +*/ +static int fkParentIsModified( + Table *pTab, + FKey *p, + int *aChange, + int bChngRowid +){ + int i; + for(i=0; inCol; i++){ + char *zKey = p->aCol[i].zCol; + int iKey; + for(iKey=0; iKeynCol; iKey++){ + if( aChange[iKey]>=0 || (iKey==pTab->iPKey && bChngRowid) ){ + Column *pCol = &pTab->aCol[iKey]; + if( zKey ){ + if( 0==sqlite3StrICmp(pCol->zName, zKey) ) return 1; + }else if( pCol->colFlags & COLFLAG_PRIMKEY ){ + return 1; + } + } + } + } + return 0; +} + /* ** This function is called when inserting, deleting or updating a row of ** table pTab to generate VDBE code to perform foreign key constraint @@ -84505,7 +96796,9 @@ SQLITE_PRIVATE void sqlite3FkCheck( Parse *pParse, /* Parse context */ Table *pTab, /* Row is being deleted from this table */ int regOld, /* Previous row data is stored here */ - int regNew /* New row data is stored here */ + int regNew, /* New row data is stored here */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ ){ sqlite3 *db = pParse->db; /* Database handle */ FKey *pFKey; /* Used to iterate through FKs */ @@ -84533,6 +96826,13 @@ SQLITE_PRIVATE void sqlite3FkCheck( int i; int isIgnore = 0; + if( aChange + && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 + && fkChildIsModified(pTab, pFKey, aChange, bChngRowid)==0 + ){ + continue; + } + /* Find the parent table of this foreign key. Also find a unique index ** on the parent key columns in the parent table. If either of these ** schema items cannot be located, set an error in pParse and return @@ -84542,8 +96842,25 @@ SQLITE_PRIVATE void sqlite3FkCheck( }else{ pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); } - if( !pTo || locateFkeyIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); if( !isIgnoreErrors || db->mallocFailed ) return; + if( pTo==0 ){ + /* If isIgnoreErrors is true, then a table is being dropped. In this + ** case SQLite runs a "DELETE FROM xxx" on the table being dropped + ** before actually dropping it in order to check FK constraints. + ** If the parent table of an FK constraint on the current table is + ** missing, behave as if it is empty. i.e. decrement the relevant + ** FK counter for each row of the current table with non-NULL keys. + */ + Vdbe *v = sqlite3GetVdbe(pParse); + int iJump = sqlite3VdbeCurrentAddr(v) + pFKey->nCol + 1; + for(i=0; inCol; i++){ + int iReg = pFKey->aCol[i].iFrom + regOld + 1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, -1); + } continue; } assert( pFKey->nCol==1 || (aiFree && pIdx) ); @@ -84592,28 +96909,34 @@ SQLITE_PRIVATE void sqlite3FkCheck( sqlite3DbFree(db, aiFree); } - /* Loop through all the foreign key constraints that refer to this table */ + /* Loop through all the foreign key constraints that refer to this table. + ** (the "child" constraints) */ for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ Index *pIdx = 0; /* Foreign key index for pFKey */ SrcList *pSrc; int *aiCol = 0; - if( !pFKey->isDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){ + if( aChange && fkParentIsModified(pTab, pFKey, aChange, bChngRowid)==0 ){ + continue; + } + + if( !pFKey->isDeferred && !(db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel && !pParse->isMultiWrite + ){ assert( regOld==0 && regNew!=0 ); /* Inserting a single row into a parent table cannot cause an immediate ** foreign key violation. So do nothing in this case. */ continue; } - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ if( !isIgnoreErrors || db->mallocFailed ) return; continue; } assert( aiCol || pFKey->nCol==1 ); - /* Create a SrcList structure containing a single table (the table - ** the foreign key that refers to this table is attached to). This - ** is required for the sqlite3WhereXXX() interface. */ + /* Create a SrcList structure containing the child table. We need the + ** child table as a SrcList for sqlite3WhereBegin() */ pSrc = sqlite3SrcListAppend(db, 0, 0, 0); if( pSrc ){ struct SrcList_item *pItem = pSrc->a; @@ -84660,15 +96983,16 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( } for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ Index *pIdx = 0; - locateFkeyIndex(pParse, pTab, p, &pIdx, 0); + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ - for(i=0; inColumn; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + for(i=0; inKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); } } } return mask; } + /* ** This function is called before generating code to update or delete a ** row contained in table pTab. If the operation is a DELETE, then @@ -84698,31 +97022,16 @@ SQLITE_PRIVATE int sqlite3FkRequired( }else{ /* This is an UPDATE. Foreign key processing is only required if the ** operation modifies one or more child or parent key columns. */ - int i; FKey *p; /* Check if any child key columns are being modified. */ for(p=pTab->pFKey; p; p=p->pNextFrom){ - for(i=0; inCol; i++){ - int iChildKey = p->aCol[i].iFrom; - if( aChange[iChildKey]>=0 ) return 1; - if( iChildKey==pTab->iPKey && chngRowid ) return 1; - } + if( fkChildIsModified(pTab, p, aChange, chngRowid) ) return 1; } /* Check if any parent key columns are being modified. */ for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - for(i=0; inCol; i++){ - char *zKey = p->aCol[i].zCol; - int iKey; - for(iKey=0; iKeynCol; iKey++){ - Column *pCol = &pTab->aCol[iKey]; - if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ - if( aChange[iKey]>=0 ) return 1; - if( iKey==pTab->iPKey && chngRowid ) return 1; - } - } - } + if( fkParentIsModified(pTab, p, aChange, chngRowid) ) return 1; } } } @@ -84785,7 +97094,7 @@ static Trigger *fkActionTrigger( int i; /* Iterator variable */ Expr *pWhen = 0; /* WHEN clause for the trigger */ - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; assert( aiCol || pFKey->nCol==1 ); for(i=0; inCol; i++){ @@ -84868,7 +97177,7 @@ static Trigger *fkActionTrigger( tFrom.z = zFrom; tFrom.n = nFrom; - pRaise = sqlite3Expr(db, TK_RAISE, "foreign key constraint failed"); + pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); if( pRaise ){ pRaise->affinity = OE_Abort; } @@ -84916,6 +97225,7 @@ static Trigger *fkActionTrigger( fkTriggerDelete(db, pTrigger); return 0; } + assert( pStep!=0 ); switch( action ){ case OE_Restrict: @@ -84947,7 +97257,9 @@ SQLITE_PRIVATE void sqlite3FkActions( Parse *pParse, /* Parse context */ Table *pTab, /* Table being updated or deleted from */ ExprList *pChanges, /* Change-list for UPDATE, NULL for DELETE */ - int regOld /* Address of array containing old row */ + int regOld, /* Address of array containing old row */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ ){ /* If foreign-key support is enabled, iterate through all FKs that ** refer to table pTab. If there is an action associated with the FK @@ -84956,9 +97268,11 @@ SQLITE_PRIVATE void sqlite3FkActions( if( pParse->db->flags&SQLITE_ForeignKeys ){ FKey *pFKey; /* Iterator variable */ for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ - Trigger *pAction = fkActionTrigger(pParse, pTab, pFKey, pChanges); - if( pAction ){ - sqlite3CodeRowTriggerDirect(pParse, pAction, pTab, regOld, OE_Abort, 0); + if( aChange==0 || fkParentIsModified(pTab, pFKey, aChange, bChngRowid) ){ + Trigger *pAct = fkActionTrigger(pParse, pTab, pFKey, pChanges); + if( pAct ){ + sqlite3CodeRowTriggerDirect(pParse, pAct, pTab, regOld, OE_Abort, 0); + } } } } @@ -84985,7 +97299,7 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ }else{ void *p = (void *)pFKey->pNextTo; const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; @@ -85027,23 +97341,38 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ */ /* -** Generate code that will open a table for reading. +** Generate code that will +** +** (1) acquire a lock for table pTab then +** (2) open pTab as cursor iCur. +** +** If pTab is a WITHOUT ROWID table, then it is the PRIMARY KEY index +** for that table that is actually opened. */ SQLITE_PRIVATE void sqlite3OpenTable( - Parse *p, /* Generate code into this VDBE */ + Parse *pParse, /* Generate code into this VDBE */ int iCur, /* The cursor number of the table */ int iDb, /* The database index in sqlite3.aDb[] */ Table *pTab, /* The table to be opened */ int opcode /* OP_OpenRead or OP_OpenWrite */ ){ Vdbe *v; - if( IsVirtual(pTab) ) return; - v = sqlite3GetVdbe(p); + assert( !IsVirtual(pTab) ); + v = sqlite3GetVdbe(pParse); assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); - sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); - sqlite3VdbeAddOp3(v, opcode, iCur, pTab->tnum, iDb); - sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(pTab->nCol), P4_INT32); - VdbeComment((v, "%s", pTab->zName)); + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nCol); + VdbeComment((v, "%s", pTab->zName)); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->tnum=pTab->tnum ); + sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + VdbeComment((v, "%s", pTab->zName)); + } } /* @@ -85053,13 +97382,13 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' NONE +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'F' REAL ** -** An extra 'b' is appended to the end of the string to cover the +** An extra 'D' is appended to the end of the string to cover the ** rowid that appears as the last column in every index. ** ** Memory for the buffer containing the column index affinity string @@ -85079,15 +97408,15 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ int n; Table *pTab = pIdx->pTable; sqlite3 *db = sqlite3VdbeDb(v); - pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+2); + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ db->mallocFailed = 1; return 0; } for(n=0; nnColumn; n++){ - pIdx->zColAff[n] = pTab->aCol[pIdx->aiColumn[n]].affinity; + i16 x = pIdx->aiColumn[n]; + pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; } - pIdx->zColAff[n++] = SQLITE_AFF_NONE; pIdx->zColAff[n] = 0; } @@ -85095,32 +97424,30 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ } /* -** Set P4 of the most recently inserted opcode to a column affinity -** string for table pTab. A column affinity string has one character -** for each column indexed by the index, according to the affinity of the -** column: +** Compute the affinity string for table pTab, if it has not already been +** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. +** +** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and +** if iReg>0 then code an OP_Affinity opcode that will set the affinities +** for register iReg and following. Or if affinities exists and iReg==0, +** then just set the P4 operand of the previous opcode (which should be +** an OP_MakeRecord) to the affinity string. +** +** A column affinity string has one character per column: ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' NONE +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'E' REAL */ -SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ - /* The first time a column affinity string for a particular table - ** is required, it is allocated and populated here. It is then - ** stored as a member of the Table structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqlite3DeleteTable() when the Table structure itself is cleaned up. - */ - if( !pTab->zColAff ){ - char *zColAff; - int i; +SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ + int i; + char *zColAff = pTab->zColAff; + if( zColAff==0 ){ sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ db->mallocFailed = 1; @@ -85130,22 +97457,28 @@ SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ for(i=0; inCol; i++){ zColAff[i] = pTab->aCol[i].affinity; } - zColAff[pTab->nCol] = '\0'; - + do{ + zColAff[i--] = 0; + }while( i>=0 && zColAff[i]==SQLITE_AFF_NONE ); pTab->zColAff = zColAff; } - - sqlite3VdbeChangeP4(v, -1, pTab->zColAff, P4_TRANSIENT); + i = sqlite3Strlen30(zColAff); + if( i ){ + if( iReg ){ + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); + }else{ + sqlite3VdbeChangeP4(v, -1, zColAff, i); + } + } } /* ** Return non-zero if the table pTab in database iDb or any of its indices -** have been opened at any point in the VDBE program beginning at location -** iStartAddr throught the end of the program. This is used to see if +** have been opened at any point in the VDBE program. This is used to see if ** a statement of the form "INSERT INTO SELECT ..." can -** run without using temporary table for the results of the SELECT. +** run without using a temporary table for the results of the SELECT. */ -static int readsTable(Parse *p, int iStartAddr, int iDb, Table *pTab){ +static int readsTable(Parse *p, int iDb, Table *pTab){ Vdbe *v = sqlite3GetVdbe(p); int i; int iEnd = sqlite3VdbeCurrentAddr(v); @@ -85153,7 +97486,7 @@ static int readsTable(Parse *p, int iStartAddr, int iDb, Table *pTab){ VTable *pVTab = IsVirtual(pTab) ? sqlite3GetVTable(p->db, pTab) : 0; #endif - for(i=iStartAddr; iopcode==OP_OpenRead && pOp->p3==iDb ){ @@ -85251,16 +97584,17 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ memId = p->regCtr; assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); addr = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); + sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); - sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); + sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); - sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); + sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); sqlite3VdbeAddOp0(v, OP_Close); } @@ -85295,25 +97629,16 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ Db *pDb = &db->aDb[p->iDb]; - int j1, j2, j3, j4, j5; + int j1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); - j2 = sqlite3VdbeAddOp0(v, OP_Rewind); - j3 = sqlite3VdbeAddOp3(v, OP_Column, 0, 0, iRec); - j4 = sqlite3VdbeAddOp3(v, OP_Eq, memId-1, 0, iRec); - sqlite3VdbeAddOp2(v, OP_Next, 0, j3); - sqlite3VdbeJumpHere(v, j2); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - j5 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, j4); - sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeJumpHere(v, j5); sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -85341,7 +97666,7 @@ static int xferOptimization( ); /* -** This routine is call to handle SQL of the following forms: +** This routine is called to handle SQL of the following forms: ** ** insert into TABLE (IDLIST) values(EXPRLIST) ** insert into TABLE (IDLIST) select @@ -85356,12 +97681,12 @@ static int xferOptimization( ** data for the insert. ** ** The code generated follows one of four templates. For a simple -** select with data coming from a VALUES clause, the code executes +** insert with data coming from a VALUES clause, the code executes ** once straight down through. Pseudo-code follows (we call this ** the "1st template"): ** ** open write cursor to
and its indices -** puts VALUES clause expressions onto the stack +** put VALUES clause expressions into registers ** write the resulting record into
** cleanup ** @@ -85393,7 +97718,6 @@ static int xferOptimization( ** and the SELECT clause does not read from
at any time. ** The generated code follows this template: ** -** EOF <- 0 ** X <- A ** goto B ** A: setup for the SELECT @@ -85402,12 +97726,9 @@ static int xferOptimization( ** yield X ** end loop ** cleanup after the SELECT -** EOF <- 1 -** yield X -** goto A +** end-coroutine X ** B: open write cursor to
and its indices -** C: yield X -** if EOF goto D +** C: yield X, at EOF goto D ** insert the select result into
from R..R+n ** goto C ** D: cleanup @@ -85415,10 +97736,9 @@ static int xferOptimization( ** The 4th template is used if the insert statement takes its ** values from a SELECT but the data is being inserted into a table ** that is also read as part of the SELECT. In the third form, -** we have to use a intermediate table to store the results of +** we have to use an intermediate table to store the results of ** the select. The template is like this: ** -** EOF <- 0 ** X <- A ** goto B ** A: setup for the SELECT @@ -85427,12 +97747,9 @@ static int xferOptimization( ** yield X ** end loop ** cleanup after the SELECT -** EOF <- 1 -** yield X -** halt-error +** end co-routine R ** B: open temp table -** L: yield X -** if EOF goto M +** L: yield X, at EOF goto M ** insert row from R..R+n into temp table ** goto L ** M: open write cursor to
and its indices @@ -85445,7 +97762,6 @@ static int xferOptimization( SQLITE_PRIVATE void sqlite3Insert( Parse *pParse, /* Parser context */ SrcList *pTabList, /* Name of table into which we are inserting */ - ExprList *pList, /* List of values to be inserted */ Select *pSelect, /* A SELECT statement to use as the data source */ IdList *pColumn, /* Column names corresponding to IDLIST. */ int onError /* How to handle constraint errors */ @@ -85459,18 +97775,21 @@ SQLITE_PRIVATE void sqlite3Insert( Index *pIdx; /* For looping over indices of the table */ int nColumn; /* Number of columns in the data */ int nHidden = 0; /* Number of hidden columns if TABLE is virtual */ - int baseCur = 0; /* VDBE Cursor number for pTab */ - int keyColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ + int iDataCur = 0; /* VDBE cursor that is the main data repository */ + int iIdxCur = 0; /* First index cursor */ + int ipkColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ int endOfLoop; /* Label for the end of the insertion loop */ - int useTempTable = 0; /* Store SELECT results in intermediate table */ int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ int addrInsTop = 0; /* Jump to label "D" */ int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ - int addrSelect = 0; /* Address of coroutine that implements the SELECT */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ Db *pDb; /* The database containing table being inserted into */ - int appendFlag = 0; /* True if the insert is likely to be an append */ + u8 useTempTable = 0; /* Store SELECT results in intermediate table */ + u8 appendFlag = 0; /* True if the insert is likely to be an append */ + u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + u8 bIdListInOrder = 1; /* True if IDLIST is in table order */ + ExprList *pList = 0; /* List of VALUES() to be inserted */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ @@ -85479,7 +97798,6 @@ SQLITE_PRIVATE void sqlite3Insert( int regIns; /* Block of regs holding rowid+data being inserted */ int regRowid; /* registers holding insert rowid */ int regData; /* register holding first column to insert */ - int regEof = 0; /* Register recording end of SELECT data */ int *aRegIdx = 0; /* One register allocated to each index */ #ifndef SQLITE_OMIT_TRIGGER @@ -85494,6 +97812,17 @@ SQLITE_PRIVATE void sqlite3Insert( goto insert_cleanup; } + /* If the Select object is really just a simple VALUES() list with a + ** single row values (the common case) then keep that one row of values + ** and go ahead and discard the Select object + */ + if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ + pList = pSelect->pEList; + pSelect->pEList = 0; + sqlite3SelectDelete(db, pSelect); + pSelect = 0; + } + /* Locate the table into which we will be inserting new information. */ assert( pTabList->nSrc==1 ); @@ -85510,6 +97839,7 @@ SQLITE_PRIVATE void sqlite3Insert( if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ goto insert_cleanup; } + withoutRowid = !HasRowid(pTab); /* Figure out if we have any triggers and if the table being ** inserted into is a view @@ -85529,16 +97859,13 @@ SQLITE_PRIVATE void sqlite3Insert( assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); /* If pTab is really a view, make sure it has been initialized. - ** ViewGetColumnNames() is a no-op if pTab is not a view (or virtual - ** module table). + ** ViewGetColumnNames() is a no-op if pTab is not a view. */ if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto insert_cleanup; } - /* Ensure that: - * (a) the table is not read-only, - * (b) that if it is a view then ON INSERT triggers exist + /* Cannot insert into a read-only table. */ if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ goto insert_cleanup; @@ -85573,72 +97900,93 @@ SQLITE_PRIVATE void sqlite3Insert( */ regAutoinc = autoIncBegin(pParse, iDb, pTab); + /* Allocate registers for holding the rowid of the new row, + ** the content of the new row, and the assembled row record. + */ + regRowid = regIns = pParse->nMem+1; + pParse->nMem += pTab->nCol + 1; + if( IsVirtual(pTab) ){ + regRowid++; + pParse->nMem++; + } + regData = regRowid+1; + + /* If the INSERT statement included an IDLIST term, then make sure + ** all elements of the IDLIST really are columns of the table and + ** remember the column indices. + ** + ** If the table has an INTEGER PRIMARY KEY column and that column + ** is named in the IDLIST, then record in the ipkColumn variable + ** the index into IDLIST of the primary key column. ipkColumn is + ** the index of the primary key as it appears in IDLIST, not as + ** is appears in the original table. (The index of the INTEGER + ** PRIMARY KEY in the original table is pTab->iPKey.) + */ + if( pColumn ){ + for(i=0; inId; i++){ + pColumn->a[i].idx = -1; + } + for(i=0; inId; i++){ + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ + pColumn->a[i].idx = j; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } + break; + } + } + if( j>=pTab->nCol ){ + if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ + ipkColumn = i; + bIdListInOrder = 0; + }else{ + sqlite3ErrorMsg(pParse, "table %S has no column named %s", + pTabList, 0, pColumn->a[i].zName); + pParse->checkSchema = 1; + goto insert_cleanup; + } + } + } + } + /* Figure out how many columns of data are supplied. If the data ** is coming from a SELECT statement, then generate a co-routine that ** produces a single row of the SELECT on each invocation. The ** co-routine is the common header to the 3rd and 4th templates. */ if( pSelect ){ - /* Data is coming from a SELECT. Generate code to implement that SELECT - ** as a co-routine. The code is common to both the 3rd and 4th - ** templates: - ** - ** EOF <- 0 - ** X <- A - ** goto B - ** A: setup for the SELECT - ** loop over the tables in the SELECT - ** load value into register R..R+n - ** yield X - ** end loop - ** cleanup after the SELECT - ** EOF <- 1 - ** yield X - ** halt-error - ** - ** On each invocation of the co-routine, it puts a single row of the - ** SELECT result into registers dest.iMem...dest.iMem+dest.nMem-1. - ** (These output registers are allocated by sqlite3Select().) When - ** the SELECT completes, it sets the EOF flag stored in regEof. - */ - int rc, j1; + /* Data is coming from a SELECT. Generate a co-routine to run the SELECT */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int rc; /* Result code */ - regEof = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ - VdbeComment((v, "SELECT eof flag")); - sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); - addrSelect = sqlite3VdbeCurrentAddr(v)+2; - sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iParm); - j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - VdbeComment((v, "Jump over SELECT coroutine")); - - /* Resolve the expressions in the SELECT statement and execute it. */ + regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; assert( pParse->nErr==0 || rc ); - if( rc || NEVER(pParse->nErr) || db->mallocFailed ){ - goto insert_cleanup; - } - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ - sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); /* yield X */ - sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); - VdbeComment((v, "End of SELECT coroutine")); - sqlite3VdbeJumpHere(v, j1); /* label B: */ - - regFromSelect = dest.iMem; + if( rc || db->mallocFailed ) goto insert_cleanup; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; - assert( dest.nMem==nColumn ); /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to - ** FALSE if each* row of the SELECT can be written directly into + ** FALSE if each output row of the SELECT can be written directly into ** the destination table (template 3). ** ** A temp table must be used if the table being updated is also one ** of the tables being read by the SELECT statement. Also use a ** temp table in the case of row triggers. */ - if( pTrigger || readsTable(pParse, addrSelect, iDb, pTab) ){ + if( pTrigger || readsTable(pParse, iDb, pTab) ){ useTempTable = 1; } @@ -85648,28 +97996,25 @@ SQLITE_PRIVATE void sqlite3Insert( ** here is from the 4th template: ** ** B: open temp table - ** L: yield X - ** if EOF goto M + ** L: yield X, goto M at EOF ** insert row from R..R+n into temp table ** goto L ** M: ... */ int regRec; /* Register to hold packed record */ int regTempRowid; /* Register to hold temp table ROWID */ - int addrTop; /* Label "L" */ - int addrIf; /* Address of jump to M */ + int addrL; /* Label "L" */ srcTab = pParse->nTab++; regRec = sqlite3GetTempReg(pParse); regTempRowid = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); - addrTop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); - addrIf = sqlite3VdbeAddOp1(v, OP_If, regEof); + addrL = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - sqlite3VdbeJumpHere(v, addrIf); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); + sqlite3VdbeJumpHere(v, addrL); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempReg(pParse, regTempRowid); } @@ -85690,6 +98035,14 @@ SQLITE_PRIVATE void sqlite3Insert( } } + /* If there is no IDLIST term but the table has an integer primary + ** key, the set the ipkColumn variable to the integer primary key + ** column index in the original table definition. + */ + if( pColumn==0 && nColumn>0 ){ + ipkColumn = pTab->iPKey; + } + /* Make sure the number of columns in the source data matches the number ** of columns to be inserted into the table. */ @@ -85708,52 +98061,6 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); goto insert_cleanup; } - - /* If the INSERT statement included an IDLIST term, then make sure - ** all elements of the IDLIST really are columns of the table and - ** remember the column indices. - ** - ** If the table has an INTEGER PRIMARY KEY column and that column - ** is named in the IDLIST, then record in the keyColumn variable - ** the index into IDLIST of the primary key column. keyColumn is - ** the index of the primary key as it appears in IDLIST, not as - ** is appears in the original table. (The index of the primary - ** key in the original table is pTab->iPKey.) - */ - if( pColumn ){ - for(i=0; inId; i++){ - pColumn->a[i].idx = -1; - } - for(i=0; inId; i++){ - for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ - pColumn->a[i].idx = j; - if( j==pTab->iPKey ){ - keyColumn = i; - } - break; - } - } - if( j>=pTab->nCol ){ - if( sqlite3IsRowid(pColumn->a[i].zName) ){ - keyColumn = i; - }else{ - sqlite3ErrorMsg(pParse, "table %S has no column named %s", - pTabList, 0, pColumn->a[i].zName); - pParse->checkSchema = 1; - goto insert_cleanup; - } - } - } - } - - /* If there is no IDLIST term but the table has an integer primary - ** key, the set the keyColumn variable to the primary key column index - ** in the original table definition. - */ - if( pColumn==0 && nColumn>0 ){ - keyColumn = pTab->iPKey; - } /* Initialize the count of rows to be inserted */ @@ -85765,9 +98072,8 @@ SQLITE_PRIVATE void sqlite3Insert( /* If this is not a view, open the table and and all indices */ if( !isView ){ int nIdx; - - baseCur = pParse->nTab; - nIdx = sqlite3OpenTableAndIndices(pParse, pTab, baseCur, OP_OpenWrite); + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, -1, 0, + &iDataCur, &iIdxCur); aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); if( aRegIdx==0 ){ goto insert_cleanup; @@ -85782,39 +98088,27 @@ SQLITE_PRIVATE void sqlite3Insert( /* This block codes the top of loop only. The complete loop is the ** following pseudocode (template 4): ** - ** rewind temp table + ** rewind temp table, if empty goto D ** C: loop over rows of intermediate table ** transfer values form intermediate table into
** end loop ** D: ... */ - addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); + addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); VdbeCoverage(v); addrCont = sqlite3VdbeCurrentAddr(v); }else if( pSelect ){ /* This block codes the top of loop only. The complete loop is the ** following pseudocode (template 3): ** - ** C: yield X - ** if EOF goto D + ** C: yield X, at EOF goto D ** insert the select result into
from R..R+n ** goto C ** D: ... */ - addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); - addrInsTop = sqlite3VdbeAddOp1(v, OP_If, regEof); + addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); } - /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assemblied row record. - */ - regRowid = regIns = pParse->nMem+1; - pParse->nMem += pTab->nCol + 1; - if( IsVirtual(pTab) ){ - regRowid++; - pParse->nMem++; - } - regData = regRowid+1; - /* Run the BEFORE and INSTEAD OF triggers, if there are any */ endOfLoop = sqlite3VdbeMakeLabel(v); @@ -85827,20 +98121,21 @@ SQLITE_PRIVATE void sqlite3Insert( ** we do not know what the unique ID will be (because the insert has ** not happened yet) so we substitute a rowid of -1 */ - if( keyColumn<0 ){ + if( ipkColumn<0 ){ sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); }else{ int j1; + assert( !withoutRowid ); if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regCols); + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); }else{ assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regCols); + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } /* Cannot have triggers on a virtual table. If it were possible, @@ -85874,8 +98169,7 @@ SQLITE_PRIVATE void sqlite3Insert( ** table column affinities. */ if( !isView ){ - sqlite3VdbeAddOp2(v, OP_Affinity, regCols+1, pTab->nCol); - sqlite3TableAffinityStr(v, pTab); + sqlite3TableAffinity(v, pTab, regCols+1); } /* Fire BEFORE or INSTEAD OF triggers */ @@ -85885,29 +98179,27 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); } - /* Push the record number for the new entry onto the stack. The - ** record number is a randomly generate integer created by NewRowid - ** except when the table has an INTEGER PRIMARY KEY column, in which - ** case the record number is the same as that column. + /* Compute the content of the next row to insert into a range of + ** registers beginning at regIns. */ if( !isView ){ if( IsVirtual(pTab) ){ /* The row that the VUpdate opcode will delete: none */ sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); } - if( keyColumn>=0 ){ + if( ipkColumn>=0 ){ if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regRowid); + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+keyColumn, regRowid); + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); }else{ VdbeOp *pOp; - sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regRowid); + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); pOp = sqlite3VdbeGetOp(v, -1); if( ALWAYS(pOp) && pOp->opcode==OP_Null && !IsVirtual(pTab) ){ appendFlag = 1; pOp->opcode = OP_NewRowid; - pOp->p1 = baseCur; + pOp->p1 = iDataCur; pOp->p2 = regRowid; pOp->p3 = regAutoinc; } @@ -85918,24 +98210,24 @@ SQLITE_PRIVATE void sqlite3Insert( if( !appendFlag ){ int j1; if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); - sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); sqlite3VdbeJumpHere(v, j1); }else{ j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); } - sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } - }else if( IsVirtual(pTab) ){ + }else if( IsVirtual(pTab) || withoutRowid ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); }else{ - sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); appendFlag = 1; } autoIncStep(pParse, regAutoinc, regRowid); - /* Push onto the stack, data for all columns of the new entry, beginning + /* Compute data for all columns of the new entry, beginning ** with the first column. */ nHidden = 0; @@ -85943,10 +98235,11 @@ SQLITE_PRIVATE void sqlite3Insert( int iRegStore = regRowid+1+i; if( i==pTab->iPKey ){ /* The value of the INTEGER PRIMARY KEY column is always a NULL. - ** Whenever this column is read, the record number will be substituted - ** in its place. So will fill this column with a NULL to avoid - ** taking up data space with information that will never be used. */ - sqlite3VdbeAddOp2(v, OP_Null, 0, iRegStore); + ** Whenever this column is read, the rowid will be substituted + ** in its place. Hence, fill this column with a NULL to avoid + ** taking up data space with information that will never be used. + ** As there may be shallow copies of this value, make it a soft-NULL */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); continue; } if( pColumn==0 ){ @@ -85963,11 +98256,13 @@ SQLITE_PRIVATE void sqlite3Insert( } } if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, iRegStore); + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); }else if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); + } }else{ sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); } @@ -85987,13 +98282,12 @@ SQLITE_PRIVATE void sqlite3Insert( #endif { int isReplace; /* Set to true if constraints may cause a replace */ - sqlite3GenerateConstraintChecks(pParse, pTab, baseCur, regIns, aRegIdx, - keyColumn>=0, 0, onError, endOfLoop, &isReplace - ); - sqlite3FkCheck(pParse, pTab, 0, regIns); - sqlite3CompleteInsertion( - pParse, pTab, baseCur, regIns, aRegIdx, 0, appendFlag, isReplace==0 + sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace ); + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, + regIns, aRegIdx, 0, appendFlag, isReplace==0); } } @@ -86014,7 +98308,7 @@ SQLITE_PRIVATE void sqlite3Insert( */ sqlite3VdbeResolveLabel(v, endOfLoop); if( useTempTable ){ - sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); + sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrInsTop); sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ @@ -86024,9 +98318,9 @@ SQLITE_PRIVATE void sqlite3Insert( if( !IsVirtual(pTab) && !isView ){ /* Close all tables opened */ - sqlite3VdbeAddOp1(v, OP_Close, baseCur); - for(idx=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, idx++){ - sqlite3VdbeAddOp1(v, OP_Close, idx+baseCur); + if( iDataCurpIndex; pIdx; pIdx=pIdx->pNext, idx++){ + sqlite3VdbeAddOp1(v, OP_Close, idx+iIdxCur); } } @@ -86059,7 +98353,7 @@ insert_cleanup: } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -86071,36 +98365,48 @@ insert_cleanup: #undef tmask #endif - /* -** Generate code to do constraint checks prior to an INSERT or an UPDATE. +** Generate code to do constraint checks prior to an INSERT or an UPDATE +** on table pTab. ** -** The input is a range of consecutive registers as follows: +** The regNewData parameter is the first register in a range that contains +** the data to be inserted or the data after the update. There will be +** pTab->nCol+1 registers in this range. The first register (the one +** that regNewData points to) will contain the new rowid, or NULL in the +** case of a WITHOUT ROWID table. The second register in the range will +** contain the content of the first table column. The third register will +** contain the content of the second table column. And so forth. ** -** 1. The rowid of the row after the update. +** The regOldData parameter is similar to regNewData except that it contains +** the data prior to an UPDATE rather than afterwards. regOldData is zero +** for an INSERT. This routine can distinguish between UPDATE and INSERT by +** checking regOldData for zero. ** -** 2. The data in the first column of the entry after the update. +** For an UPDATE, the pkChng boolean is true if the true primary key (the +** rowid for a normal table or the PRIMARY KEY for a WITHOUT ROWID table) +** might be modified by the UPDATE. If pkChng is false, then the key of +** the iDataCur content table is guaranteed to be unchanged by the UPDATE. ** -** i. Data from middle columns... +** For an INSERT, the pkChng boolean indicates whether or not the rowid +** was explicitly specified as part of the INSERT statement. If pkChng +** is zero, it means that the either rowid is computed automatically or +** that the table is a WITHOUT ROWID table and has no rowid. On an INSERT, +** pkChng will only be true if the INSERT statement provides an integer +** value for either the rowid column or its INTEGER PRIMARY KEY alias. ** -** N. The data in the last column of the entry after the update. -** -** The regRowid parameter is the index of the register containing (1). -** -** If isUpdate is true and rowidChng is non-zero, then rowidChng contains -** the address of a register containing the rowid before the update takes -** place. isUpdate is true for UPDATEs and false for INSERTs. If isUpdate -** is false, indicating an INSERT statement, then a non-zero rowidChng -** indicates that the rowid was explicitly specified as part of the -** INSERT statement. If rowidChng is false, it means that the rowid is -** computed automatically in an insert or that the rowid value is not -** modified by an update. -** -** The code generated by this routine store new index entries into +** The code generated by this routine will store new index entries into ** registers identified by aRegIdx[]. No index entry is created for ** indices where aRegIdx[i]==0. The order of indices in aRegIdx[] is ** the same as the order of indices on the linked list of indices -** attached to the table. +** at pTab->pIndex. +** +** The caller must have already opened writeable cursors on the main +** table and all applicable indices (that is to say, all indices for which +** aRegIdx[] is not zero). iDataCur is the cursor for the main table when +** inserting or updating a rowid table, or the cursor for the PRIMARY KEY +** index when operating on a WITHOUT ROWID table. iIdxCur is the cursor +** for the first index in the pTab->pIndex list. Cursors for other indices +** are at iIdxCur+N for the N-th element of the pTab->pIndex list. ** ** This routine also generates code to check constraints. NOT NULL, ** CHECK, and UNIQUE constraints are all checked. If a constraint fails, @@ -86110,22 +98416,23 @@ insert_cleanup: ** Constraint type Action What Happens ** --------------- ---------- ---------------------------------------- ** any ROLLBACK The current transaction is rolled back and -** sqlite3_exec() returns immediately with a +** sqlite3_step() returns immediately with a ** return code of SQLITE_CONSTRAINT. ** ** any ABORT Back out changes from the current command ** only (do not do a complete rollback) then -** cause sqlite3_exec() to return immediately +** cause sqlite3_step() to return immediately ** with SQLITE_CONSTRAINT. ** -** any FAIL Sqlite_exec() returns immediately with a +** any FAIL Sqlite3_step() returns immediately with a ** return code of SQLITE_CONSTRAINT. The ** transaction is not rolled back and any -** prior changes are retained. +** changes to prior rows are retained. ** -** any IGNORE The record number and data is popped from -** the stack and there is an immediate jump -** to label ignoreDest. +** any IGNORE The attempt in insert or update the current +** row is skipped, without throwing an error. +** Processing continues with the next row. +** (There is an immediate jump to ignoreDest.) ** ** NOT NULL REPLACE The NULL value is replace by the default ** value for that column. If the default value @@ -86140,42 +98447,59 @@ insert_cleanup: ** Or if overrideError==OE_Default, then the pParse->onError parameter ** is used. Or if pParse->onError==OE_Default then the onError value ** for the constraint is used. -** -** The calling routine must open a read/write cursor for pTab with -** cursor number "baseCur". All indices of pTab must also have open -** read/write cursors with cursor number baseCur+i for the i-th cursor. -** Except, if there is no possibility of a REPLACE action then -** cursors do not need to be open for indices where aRegIdx[i]==0. */ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( - Parse *pParse, /* The parser context */ - Table *pTab, /* the table into which we are inserting */ - int baseCur, /* Index of a read/write cursor pointing at pTab */ - int regRowid, /* Index of the range of input registers */ - int *aRegIdx, /* Register used by each index. 0 for unused indices */ - int rowidChng, /* True if the rowid might collide with existing entry */ - int isUpdate, /* True for UPDATE, False for INSERT */ - int overrideError, /* Override onError to this if not OE_Default */ - int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ - int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ + Parse *pParse, /* The parser context */ + Table *pTab, /* The table being inserted or updated */ + int *aRegIdx, /* Use register aRegIdx[i] for index i. 0 for unused */ + int iDataCur, /* Canonical data cursor (main table or PK index) */ + int iIdxCur, /* First index cursor */ + int regNewData, /* First register in a range holding values to insert */ + int regOldData, /* Previous content. 0 for INSERTs */ + u8 pkChng, /* Non-zero if the rowid or PRIMARY KEY changed */ + u8 overrideError, /* Override onError to this if not OE_Default */ + int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ + int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ ){ - int i; /* loop counter */ - Vdbe *v; /* VDBE under constrution */ - int nCol; /* Number of columns */ - int onError; /* Conflict resolution strategy */ - int j1; /* Addresss of jump instruction */ - int j2 = 0, j3; /* Addresses of jump instructions */ - int regData; /* Register containing first data column */ - int iCur; /* Table cursor number */ + Vdbe *v; /* VDBE under constrution */ Index *pIdx; /* Pointer to one of the indices */ + Index *pPk = 0; /* The PRIMARY KEY index */ + sqlite3 *db; /* Database connection */ + int i; /* loop counter */ + int ix; /* Index loop counter */ + int nCol; /* Number of columns */ + int onError; /* Conflict resolution strategy */ + int j1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ - int regOldRowid = (rowidChng && isUpdate) ? rowidChng : regRowid; + int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ + int ipkTop = 0; /* Top of the rowid change constraint check */ + int ipkBottom = 0; /* Bottom of the rowid change constraint check */ + u8 isUpdate; /* True if this is an UPDATE operation */ + u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ + int regRowid = -1; /* Register holding ROWID value */ + isUpdate = regOldData!=0; + db = pParse->db; v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ nCol = pTab->nCol; - regData = regRowid + 1; + + /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for + ** normal rowid tables. nPkField is the number of key fields in the + ** pPk index or 1 for a rowid table. In other words, nPkField is the + ** number of fields in the true primary key of the table. */ + if( HasRowid(pTab) ){ + pPk = 0; + nPkField = 1; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + nPkField = pPk->nKeyCol; + } + + /* Record that this module has started */ + VdbeModuleComment((v, "BEGIN: GenCnstCks(%d,%d,%d,%d,%d)", + iDataCur, iIdxCur, regNewData, regOldData, pkChng)); /* Test all NOT NULL constraints. */ @@ -86198,24 +98522,26 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( switch( onError ){ case OE_Abort: sqlite3MayAbort(pParse); + /* Fall through */ case OE_Rollback: case OE_Fail: { - char *zMsg; - sqlite3VdbeAddOp3(v, OP_HaltIfNull, - SQLITE_CONSTRAINT, onError, regData+i); - zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", - pTab->zName, pTab->aCol[i].zName); - sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); + char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, + pTab->aCol[i].zName); + sqlite3VdbeAddOp4(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, + regNewData+1+i, zMsg, P4_DYNAMIC); + sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); + VdbeCoverage(v); break; } case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_IsNull, regData+i, ignoreDest); + sqlite3VdbeAddOp2(v, OP_IsNull, regNewData+1+i, ignoreDest); + VdbeCoverage(v); break; } default: { assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regData+i); - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regData+i); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); sqlite3VdbeJumpHere(v, j1); break; } @@ -86225,37 +98551,71 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* Test all CHECK constraints */ #ifndef SQLITE_OMIT_CHECK - if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ - int allOk = sqlite3VdbeMakeLabel(v); - pParse->ckBase = regData; - sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; + pParse->ckBase = regNewData+1; onError = overrideError!=OE_Default ? overrideError : OE_Abort; - if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - }else{ - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ - sqlite3HaltConstraint(pParse, onError, 0, 0); + for(i=0; inExpr; i++){ + int allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + }else{ + char *zName = pCheck->a[i].zName; + if( zName==0 ) zName = pTab->zName; + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zName, P4_TRANSIENT, + P5_ConstraintCheck); + } + sqlite3VdbeResolveLabel(v, allOk); } - sqlite3VdbeResolveLabel(v, allOk); } #endif /* !defined(SQLITE_OMIT_CHECK) */ - /* If we have an INTEGER PRIMARY KEY, make sure the primary key - ** of the new record does not previously exist. Except, if this - ** is an UPDATE and the primary key is not changing, that is OK. + /* If rowid is changing, make sure the new rowid does not previously + ** exist in the table. */ - if( rowidChng ){ + if( pkChng && pPk==0 ){ + int addrRowidOk = sqlite3VdbeMakeLabel(v); + + /* Figure out what action to take in case of a rowid collision */ onError = pTab->keyConf; if( overrideError!=OE_Default ){ onError = overrideError; }else if( onError==OE_Default ){ onError = OE_Abort; } - + if( isUpdate ){ - j2 = sqlite3VdbeAddOp3(v, OP_Eq, regRowid, 0, rowidChng); + /* pkChng!=0 does not mean that the rowid has change, only that + ** it might have changed. Skip the conflict logic below if the rowid + ** is unchanged. */ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); } - j3 = sqlite3VdbeAddOp3(v, OP_NotExists, baseCur, 0, regRowid); + + /* If the response to a rowid conflict is REPLACE but the response + ** to some other UNIQUE constraint is FAIL or IGNORE, then we need + ** to defer the running of the rowid conflict checking until after + ** the UNIQUE constraints have run. + */ + if( onError==OE_Replace && overrideError!=OE_Replace ){ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_Ignore || pIdx->onError==OE_Fail ){ + ipkTop = sqlite3VdbeAddOp0(v, OP_Goto); + break; + } + } + } + + /* Check to see if the new rowid already exists in the table. Skip + ** the following conflict logic if it does not. */ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData); + VdbeCoverage(v); + + /* Generate code that deals with a rowid collision */ switch( onError ){ default: { onError = OE_Abort; @@ -86264,14 +98624,13 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( case OE_Rollback: case OE_Abort: case OE_Fail: { - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3RowidConstraint(pParse, onError, pTab); break; } case OE_Replace: { /* If there are DELETE triggers on this table and the ** recursive-triggers flag is set, call GenerateRowDelete() to - ** remove the conflicting row from the the table. This will fire + ** remove the conflicting row from the table. This will fire ** the triggers and remove both the table and index b-tree entries. ** ** Otherwise, if there are no triggers or the recursive-triggers @@ -86292,62 +98651,98 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** table. */ Trigger *pTrigger = 0; - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); - sqlite3GenerateRowDelete( - pParse, pTab, baseCur, regRowid, 0, pTrigger, OE_Replace - ); + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regNewData, 1, 0, OE_Replace, 1); }else if( pTab->pIndex ){ sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, baseCur, 0); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); } seenReplace = 1; break; } case OE_Ignore: { - assert( seenReplace==0 ); + /*assert( seenReplace==0 );*/ sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); break; } } - sqlite3VdbeJumpHere(v, j3); - if( isUpdate ){ - sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeResolveLabel(v, addrRowidOk); + if( ipkTop ){ + ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, ipkTop); } } /* Test all UNIQUE constraints by creating entries for each UNIQUE ** index and making sure that duplicate entries do not already exist. - ** Add the new records to the indices as we go. + ** Compute the revised record entries for indices as we go. + ** + ** This loop also handles the case of the PRIMARY KEY index for a + ** WITHOUT ROWID table. */ - for(iCur=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, iCur++){ - int regIdx; - int regR; + for(ix=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, ix++){ + int regIdx; /* Range of registers hold conent for pIdx */ + int regR; /* Range of registers holding conflicting PK */ + int iThisCur; /* Cursor for this UNIQUE index */ + int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ - if( aRegIdx[iCur]==0 ) continue; /* Skip unused indices */ - - /* Create a key for accessing the index entry */ - regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn+1); - for(i=0; inColumn; i++){ - int idx = pIdx->aiColumn[i]; - if( idx==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, regData+idx, regIdx+i); - } + if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; } - sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn+1, aRegIdx[iCur]); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), P4_TRANSIENT); - sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn+1); + iThisCur = iIdxCur+ix; + addrUniqueOk = sqlite3VdbeMakeLabel(v); - /* Find out what action to take in case there is an indexing conflict */ + /* Skip partial indices for which the WHERE clause is not true */ + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); + pParse->ckBase = regNewData+1; + sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); + pParse->ckBase = 0; + } + + /* Create a record for this index entry as it should appear after + ** the insert or update. Store that record in the aRegIdx[ix] register + */ + regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn); + for(i=0; inColumn; i++){ + int iField = pIdx->aiColumn[i]; + int x; + if( iField<0 || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); + VdbeComment((v, "for %s", pIdx->zName)); + sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn); + + /* In an UPDATE operation, if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table and there has been no change the + ** primary key, then no collision is possible. The collision detection + ** logic below can all be skipped. */ + if( isUpdate && pPk==pIdx && pkChng==0 ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } + + /* Find out what action to take in case there is a uniqueness conflict */ onError = pIdx->onError; if( onError==OE_None ){ - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn+1); + sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); + sqlite3VdbeResolveLabel(v, addrUniqueOk); continue; /* pIdx is not a UNIQUE index */ } if( overrideError!=OE_Default ){ @@ -86355,18 +98750,64 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( }else if( onError==OE_Default ){ onError = OE_Abort; } - if( seenReplace ){ - if( onError==OE_Ignore ) onError = OE_Replace; - else if( onError==OE_Fail ) onError = OE_Abort; - } /* Check to see if the new index entry will be unique */ - regR = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_SCopy, regOldRowid, regR); - j3 = sqlite3VdbeAddOp4(v, OP_IsUnique, baseCur+iCur+1, 0, - regR, SQLITE_INT_TO_PTR(regIdx), - P4_INT32); - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn+1); + sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, + regIdx, pIdx->nKeyCol); VdbeCoverage(v); + + /* Generate code to handle collisions */ + regR = (pIdx==pPk) ? regIdx : sqlite3GetTempRange(pParse, nPkField); + if( isUpdate || onError==OE_Replace ){ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_IdxRowid, iThisCur, regR); + /* Conflict only if the rowid of the existing index entry + ** is different from old-rowid */ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regR, addrUniqueOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + }else{ + int x; + /* Extract the PRIMARY KEY from the end of the index entry and + ** store it in registers regR..regR+nPk-1 */ + if( pIdx!=pPk ){ + for(i=0; inKeyCol; i++){ + x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); + sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); + VdbeComment((v, "%s.%s", pTab->zName, + pTab->aCol[pPk->aiColumn[i]].zName)); + } + } + if( isUpdate ){ + /* If currently processing the PRIMARY KEY of a WITHOUT ROWID + ** table, only conflict if the new PRIMARY KEY values are actually + ** different from the old. + ** + ** For a UNIQUE index, only conflict if the PRIMARY KEY values + ** of the matched index row are different from the original PRIMARY + ** KEY values of this row before the update. */ + int addrJump = sqlite3VdbeCurrentAddr(v)+pPk->nKeyCol; + int op = OP_Ne; + int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); + + for(i=0; inKeyCol; i++){ + char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); + x = pPk->aiColumn[i]; + if( i==(pPk->nKeyCol-1) ){ + addrJump = addrUniqueOk; + op = OP_Eq; + } + sqlite3VdbeAddOp4(v, op, + regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ + ); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverageIf(v, op==OP_Eq); + VdbeCoverageIf(v, op==OP_Ne); + } + } + } + } /* Generate code that executes if the new index entry is not unique */ assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail @@ -86375,29 +98816,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( case OE_Rollback: case OE_Abort: case OE_Fail: { - int j; - StrAccum errMsg; - const char *zSep; - char *zErr; - - sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; - zSep = pIdx->nColumn>1 ? "columns " : "column "; - for(j=0; jnColumn; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - sqlite3StrAccumAppend(&errMsg, zSep, -1); - zSep = ", "; - sqlite3StrAccumAppend(&errMsg, zCol, -1); - } - sqlite3StrAccumAppend(&errMsg, - pIdx->nColumn>1 ? " are not unique" : " is not unique", -1); - zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, onError, zErr, 0); - sqlite3DbFree(errMsg.db, zErr); + sqlite3UniqueConstraint(pParse, onError, pIdx); break; } case OE_Ignore: { - assert( seenReplace==0 ); sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); break; } @@ -86405,29 +98827,32 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( Trigger *pTrigger = 0; assert( onError==OE_Replace ); sqlite3MultiWrite(pParse); - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } - sqlite3GenerateRowDelete( - pParse, pTab, baseCur, regR, 0, pTrigger, OE_Replace - ); + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regR, nPkField, 0, OE_Replace, pIdx==pPk); seenReplace = 1; break; } } - sqlite3VdbeJumpHere(v, j3); - sqlite3ReleaseTempReg(pParse, regR); + sqlite3VdbeResolveLabel(v, addrUniqueOk); + sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); + if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); + } + if( ipkTop ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ipkTop+1); + sqlite3VdbeJumpHere(v, ipkBottom); } - if( pbMayReplace ){ - *pbMayReplace = seenReplace; - } + *pbMayReplace = seenReplace; + VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } /* ** This routine generates code to finish the INSERT or UPDATE operation ** that was started by a prior call to sqlite3GenerateConstraintChecks. -** A consecutive range of registers starting at regRowid contains the +** A consecutive range of registers starting at regNewData contains the ** rowid and the content to be inserted. ** ** The arguments to this routine should be the same as the first six @@ -86436,36 +98861,46 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( SQLITE_PRIVATE void sqlite3CompleteInsertion( Parse *pParse, /* The parser context */ Table *pTab, /* the table into which we are inserting */ - int baseCur, /* Index of a read/write cursor pointing at pTab */ - int regRowid, /* Range of content */ + int iDataCur, /* Cursor of the canonical data source */ + int iIdxCur, /* First index cursor */ + int regNewData, /* Range of content */ int *aRegIdx, /* Register used by each index. 0 for unused indices */ int isUpdate, /* True for UPDATE, False for INSERT */ int appendBias, /* True if this is likely to be an append */ int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ ){ - int i; - Vdbe *v; - int nIdx; - Index *pIdx; - u8 pik_flags; - int regData; - int regRec; + Vdbe *v; /* Prepared statements under construction */ + Index *pIdx; /* An index being inserted or updated */ + u8 pik_flags; /* flag values passed to the btree insert */ + int regData; /* Content registers (after the rowid) */ + int regRec; /* Register holding assembled record for the table */ + int i; /* Loop counter */ + u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ - for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){} - for(i=nIdx-1; i>=0; i--){ + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ if( aRegIdx[i]==0 ) continue; - sqlite3VdbeAddOp2(v, OP_IdxInsert, baseCur+i+1, aRegIdx[i]); - if( useSeekResult ){ - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + bAffinityDone = 1; + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i]); + pik_flags = 0; + if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + assert( pParse->nested==0 ); + pik_flags |= OPFLAG_NCHANGE; + } + if( pik_flags ) sqlite3VdbeChangeP5(v, pik_flags); } - regData = regRowid + 1; + if( !HasRowid(pTab) ) return; + regData = regNewData + 1; regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); - sqlite3TableAffinityStr(v, pTab); + if( !bAffinityDone ) sqlite3TableAffinity(v, pTab, 0); sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); if( pParse->nested ){ pik_flags = 0; @@ -86479,7 +98914,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( if( useSeekResult ){ pik_flags |= OPFLAG_USESEEKRESULT; } - sqlite3VdbeAddOp3(v, OP_Insert, baseCur, regRec, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); if( !pParse->nested ){ sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } @@ -86487,39 +98922,74 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( } /* -** Generate code that will open cursors for a table and for all -** indices of that table. The "baseCur" parameter is the cursor number used -** for the table. Indices are opened on subsequent cursors. +** Allocate cursors for the pTab table and all its indices and generate +** code to open and initialized those cursors. ** -** Return the number of indices on the table. +** The cursor for the object that contains the complete data (normally +** the table itself, but the PRIMARY KEY index in the case of a WITHOUT +** ROWID table) is returned in *piDataCur. The first index cursor is +** returned in *piIdxCur. The number of indices is returned. +** +** Use iBase as the first cursor (either the *piDataCur for rowid tables +** or the first index for WITHOUT ROWID tables) if it is non-negative. +** If iBase is negative, then allocate the next available cursor. +** +** For a rowid table, *piDataCur will be exactly one less than *piIdxCur. +** For a WITHOUT ROWID table, *piDataCur will be somewhere in the range +** of *piIdxCurs, depending on where the PRIMARY KEY index appears on the +** pTab->pIndex list. +** +** If pTab is a virtual table, then this routine is a no-op and the +** *piDataCur and *piIdxCur values are left uninitialized. */ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ Table *pTab, /* Table to be opened */ - int baseCur, /* Cursor number assigned to the table */ - int op /* OP_OpenRead or OP_OpenWrite */ + int op, /* OP_OpenRead or OP_OpenWrite */ + int iBase, /* Use this for the table cursor, if there is one */ + u8 *aToOpen, /* If not NULL: boolean for each table and index */ + int *piDataCur, /* Write the database source cursor number here */ + int *piIdxCur /* Write the first index cursor number here */ ){ int i; int iDb; + int iDataCur; Index *pIdx; Vdbe *v; - if( IsVirtual(pTab) ) return 0; + assert( op==OP_OpenRead || op==OP_OpenWrite ); + if( IsVirtual(pTab) ){ + /* This routine is a no-op for virtual tables. Leave the output + ** variables *piDataCur and *piIdxCur uninitialized so that valgrind + ** can detect if they are used by mistake in the caller. */ + return 0; + } iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); v = sqlite3GetVdbe(pParse); assert( v!=0 ); - sqlite3OpenTable(pParse, baseCur, iDb, pTab, op); - for(i=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + if( iBase<0 ) iBase = pParse->nTab; + iDataCur = iBase++; + if( piDataCur ) *piDataCur = iDataCur; + if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ + sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); + } + if( piIdxCur ) *piIdxCur = iBase; + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp4(v, op, i+baseCur, pIdx->tnum, iDb, - (char*)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIdx->zName)); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) && piDataCur ){ + *piDataCur = iIdxCur; + } + if( aToOpen==0 || aToOpen[i+1] ){ + sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + } } - if( pParse->nTabnTab = baseCur+i; - } - return i-1; + if( iBase>pParse->nTab ) pParse->nTab = iBase; + return i; } @@ -86528,7 +98998,7 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( ** The following global variable is incremented whenever the ** transfer optimization is used. This is used for testing ** purposes only - to make sure the transfer optimization really -** is happening when it is suppose to. +** is happening when it is supposed to. */ SQLITE_API int sqlite3_xferopt_count; #endif /* SQLITE_TEST */ @@ -86558,18 +99028,19 @@ static int xferCompatibleCollation(const char *z1, const char *z2){ ** * The same DESC and ASC markings occurs on all columns ** * The same onError processing (OE_Abort, OE_Ignore, etc) ** * The same collating sequence on each column +** * The index has the exact same WHERE clause */ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ int i; assert( pDest && pSrc ); assert( pDest->pTable!=pSrc->pTable ); - if( pDest->nColumn!=pSrc->nColumn ){ + if( pDest->nKeyCol!=pSrc->nKeyCol ){ return 0; /* Different number of columns */ } if( pDest->onError!=pSrc->onError ){ return 0; /* Different conflict resolution strategies */ } - for(i=0; inColumn; i++){ + for(i=0; inKeyCol; i++){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } @@ -86580,6 +99051,9 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ return 0; /* Different collating sequences */ } } + if( sqlite3ExprCompare(pSrc->pPartIdxWhere, pDest->pPartIdxWhere, -1) ){ + return 0; /* Different WHERE clauses */ + } /* If no test above fails then the indices must be compatible */ return 1; @@ -86590,31 +99064,25 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ ** ** INSERT INTO tab1 SELECT * FROM tab2; ** -** This optimization is only attempted if +** The xfer optimization transfers raw records from tab2 over to tab1. +** Columns are not decoded and reassembled, which greatly improves +** performance. Raw index records are transferred in the same way. ** -** (1) tab1 and tab2 have identical schemas including all the -** same indices and constraints +** The xfer optimization is only attempted if tab1 and tab2 are compatible. +** There are lots of rules for determining compatibility - see comments +** embedded in the code for details. ** -** (2) tab1 and tab2 are different tables +** This routine returns TRUE if the optimization is guaranteed to be used. +** Sometimes the xfer optimization will only work if the destination table +** is empty - a factor that can only be determined at run-time. In that +** case, this routine generates code for the xfer optimization but also +** does a test to see if the destination table is empty and jumps over the +** xfer optimization code if the test fails. In that case, this routine +** returns FALSE so that the caller will know to go ahead and generate +** an unoptimized transfer. This routine also returns FALSE if there +** is no chance that the xfer optimization can be applied. ** -** (3) There must be no triggers on tab1 -** -** (4) The result set of the SELECT statement is "*" -** -** (5) The SELECT statement has no WHERE, HAVING, ORDER BY, GROUP BY, -** or LIMIT clause. -** -** (6) The SELECT statement is a simple (not a compound) select that -** contains only tab2 in its FROM clause -** -** This method for implementing the INSERT transfers raw records from -** tab2 over to tab1. The columns are not decoded. Raw records from -** the indices of tab2 are transfered to tab1 as well. In so doing, -** the resulting tab1 has much less fragmentation. -** -** This routine returns TRUE if the optimization is attempted. If any -** of the conditions above fail so that the optimization should not -** be attempted, then this routine returns FALSE. +** This optimization is particularly useful at making VACUUM run faster. */ static int xferOptimization( Parse *pParse, /* Parser context */ @@ -86631,10 +99099,9 @@ static int xferOptimization( int iDbSrc; /* The database of pSrc */ int iSrc, iDest; /* Cursors from source and destination */ int addr1, addr2; /* Loop addresses */ - int emptyDestTest; /* Address of test for empty pDest */ - int emptySrcTest; /* Address of test for empty pSrc */ + int emptyDestTest = 0; /* Address of test for empty pDest */ + int emptySrcTest = 0; /* Address of test for empty pSrc */ Vdbe *v; /* The VDBE we are building */ - KeyInfo *pKey; /* Key information for an index */ int regAutoinc; /* Memory register used by AUTOINC */ int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ int regData, regRowid; /* Registers holding data and rowid */ @@ -86642,6 +99109,12 @@ static int xferOptimization( if( pSelect==0 ){ return 0; /* Must be of the form INSERT INTO ... SELECT ... */ } + if( pParse->pWith || pSelect->pWith ){ + /* Do not attempt to process this query if there are an WITH clauses + ** attached to it. Proceeding may generate a false "no such table: xxx" + ** error if pSelect reads from a CTE named "xxx". */ + return 0; + } if( sqlite3TriggerList(pParse, pDest) ){ return 0; /* tab1 must not have triggers */ } @@ -86651,10 +99124,8 @@ static int xferOptimization( } #endif if( onError==OE_Default ){ - onError = OE_Abort; - } - if( onError!=OE_Abort && onError!=OE_Rollback ){ - return 0; /* Cannot do OR REPLACE or OR IGNORE or OR FAIL */ + if( pDest->iPKey>=0 ) onError = pDest->keyConf; + if( onError==OE_Default ) onError = OE_Abort; } assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ if( pSelect->pSrc->nSrc!=1 ){ @@ -86699,13 +99170,16 @@ static int xferOptimization( ** we have to check the semantics. */ pItem = pSelect->pSrc->a; - pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } if( pSrc==pDest ){ return 0; /* tab1 and tab2 may not be the same table */ } + if( HasRowid(pDest)!=HasRowid(pSrc) ){ + return 0; /* source and destination must both be WITHOUT ROWID or not */ + } #ifndef SQLITE_OMIT_VIRTUALTABLE if( pSrc->tabFlags & TF_Virtual ){ return 0; /* tab2 must not be a virtual table */ @@ -86721,18 +99195,27 @@ static int xferOptimization( return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ } for(i=0; inCol; i++){ - if( pDest->aCol[i].affinity!=pSrc->aCol[i].affinity ){ + Column *pDestCol = &pDest->aCol[i]; + Column *pSrcCol = &pSrc->aCol[i]; + if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( !xferCompatibleCollation(pDest->aCol[i].zColl, pSrc->aCol[i].zColl) ){ + if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ return 0; /* Collating sequence must be the same on all columns */ } - if( pDest->aCol[i].notNull && !pSrc->aCol[i].notNull ){ + if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } + /* Default values for second and subsequent columns need to match. */ + if( i>0 + && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) + ){ + return 0; /* Default values must be the same for all columns */ + } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - if( pDestIdx->onError!=OE_None ){ + if( IsUniqueIndex(pDestIdx) ){ destHasUniqueIdx = 1; } for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ @@ -86743,7 +99226,7 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif @@ -86759,14 +99242,13 @@ static int xferOptimization( return 0; } #endif + if( (pParse->db->flags & SQLITE_CountRows)!=0 ){ + return 0; /* xfer opt does not play well with PRAGMA count_changes */ + } - /* If we get this far, it means either: - ** - ** * We can always do the transfer if the table contains an - ** an integer primary key - ** - ** * We can conditionally do the transfer if the destination - ** table is empty. + /* If we get this far, it means that the xfer optimization is at + ** least a possibility, though it might only work if the destination + ** table (tab1) is initially empty. */ #ifdef SQLITE_TEST sqlite3_xferopt_count++; @@ -86777,72 +99259,82 @@ static int xferOptimization( iSrc = pParse->nTab++; iDest = pParse->nTab++; regAutoinc = autoIncBegin(pParse, iDbDest, pDest); - sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); - if( (pDest->iPKey<0 && pDest->pIndex!=0) || destHasUniqueIdx ){ - /* If tables do not have an INTEGER PRIMARY KEY and there - ** are indices to be copied and the destination is not empty, - ** we have to disallow the transfer optimization because the - ** the rowids might change which will mess up indexing. - ** - ** Or if the destination has a UNIQUE index and is not empty, - ** we also disallow the transfer optimization because we cannot - ** insure that all entries in the union of DEST and SRC will be - ** unique. - */ - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); - emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - sqlite3VdbeJumpHere(v, addr1); - }else{ - emptyDestTest = 0; - } - sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); - emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); regData = sqlite3GetTempReg(pParse); regRowid = sqlite3GetTempReg(pParse); - if( pDest->iPKey>=0 ){ - addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); - addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); - sqlite3VdbeJumpHere(v, addr2); - autoIncStep(pParse, regAutoinc, regRowid); - }else if( pDest->pIndex==0 ){ - addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); - }else{ - addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); - assert( (pDest->tabFlags & TF_Autoincrement)==0 ); + sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); + assert( HasRowid(pDest) || destHasUniqueIdx ); + if( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + || destHasUniqueIdx /* (2) */ + || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ + ){ + /* In some circumstances, we are able to run the xfer optimization + ** only if the destination table is initially empty. This code makes + ** that determination. Conditions under which the destination must + ** be empty: + ** + ** (1) There is no INTEGER PRIMARY KEY but there are indices. + ** (If the destination is not initially empty, the rowid fields + ** of index entries might need to change.) + ** + ** (2) The destination has a unique index. (The xfer optimization + ** is unable to test uniqueness.) + ** + ** (3) onError is something other than OE_Abort and OE_Rollback. + */ + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); + emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + sqlite3VdbeJumpHere(v, addr1); + } + if( HasRowid(pSrc) ){ + sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); + emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); + if( pDest->iPKey>=0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, onError, pDest); + sqlite3VdbeJumpHere(v, addr2); + autoIncStep(pParse, regAutoinc, regRowid); + }else if( pDest->pIndex==0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); + }else{ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + assert( (pDest->tabFlags & TF_Autoincrement)==0 ); + } + sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + }else{ + sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName); + sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); } - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); - sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); - sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); - sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); - sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } assert( pSrcIdx ); - sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); - sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); - pKey = sqlite3IndexKeyinfo(pParse, pSrcIdx); - sqlite3VdbeAddOp4(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc, - (char*)pKey, P4_KEYINFO_HANDOFF); + sqlite3VdbeAddOp3(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc); + sqlite3VdbeSetP4KeyInfo(pParse, pSrcIdx); VdbeComment((v, "%s", pSrcIdx->zName)); - pKey = sqlite3IndexKeyinfo(pParse, pDestIdx); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest, - (char*)pKey, P4_KEYINFO_HANDOFF); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest); + sqlite3VdbeSetP4KeyInfo(pParse, pDestIdx); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); } - sqlite3VdbeJumpHere(v, emptySrcTest); + if( emptySrcTest ) sqlite3VdbeJumpHere(v, emptySrcTest); sqlite3ReleaseTempReg(pParse, regRowid); sqlite3ReleaseTempReg(pParse, regData); - sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); - sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); if( emptyDestTest ){ sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); sqlite3VdbeJumpHere(v, emptyDestTest); @@ -86895,20 +99387,19 @@ SQLITE_API int sqlite3_exec( const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3_stmt *pStmt = 0; /* The current SQL statement */ char **azCols = 0; /* Names of result columns */ - int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; if( zSql==0 ) zSql = ""; sqlite3_mutex_enter(db->mutex); - sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + sqlite3Error(db, SQLITE_OK); + while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); assert( rc==SQLITE_OK || pStmt==0 ); if( rc!=SQLITE_OK ){ continue; @@ -86954,10 +99445,13 @@ SQLITE_API int sqlite3_exec( } } if( xCallback(pArg, nCol, azVals, azCols) ){ + /* EVIDENCE-OF: R-38229-40159 If the callback function to + ** sqlite3_exec() returns non-zero, then sqlite3_exec() will + ** return SQLITE_ABORT. */ rc = SQLITE_ABORT; sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - sqlite3Error(db, SQLITE_ABORT, 0); + sqlite3Error(db, SQLITE_ABORT); goto exec_out; } } @@ -86965,11 +99459,8 @@ SQLITE_API int sqlite3_exec( if( rc!=SQLITE_ROW ){ rc = sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; break; } } @@ -86983,14 +99474,14 @@ exec_out: sqlite3DbFree(db, azCols); rc = sqlite3ApiExit(db, rc); - if( rc!=SQLITE_OK && ALWAYS(rc==sqlite3_errcode(db)) && pzErrMsg ){ + if( rc!=SQLITE_OK && pzErrMsg ){ int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); *pzErrMsg = sqlite3Malloc(nErrMsg); if( *pzErrMsg ){ memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); }else{ rc = SQLITE_NOMEM; - sqlite3Error(db, SQLITE_NOMEM, 0); + sqlite3Error(db, SQLITE_NOMEM); } }else if( pzErrMsg ){ *pzErrMsg = 0; @@ -87052,7 +99543,7 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** WARNING: In order to maintain backwards compatibility, add new ** interfaces to the end of this structure only. If you insert new ** interfaces in the middle of this structure, then older different -** versions of SQLite will not be able to load each others' shared +** versions of SQLite will not be able to load each other's shared ** libraries! */ struct sqlite3_api_routines { @@ -87073,8 +99564,10 @@ struct sqlite3_api_routines { int (*busy_timeout)(sqlite3*,int ms); int (*changes)(sqlite3*); int (*close)(sqlite3*); - int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const char*)); - int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const void*)); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const void*)); const void * (*column_blob)(sqlite3_stmt*,int iCol); int (*column_bytes)(sqlite3_stmt*,int iCol); int (*column_bytes16)(sqlite3_stmt*,int iCol); @@ -87099,10 +99592,18 @@ struct sqlite3_api_routines { void * (*commit_hook)(sqlite3*,int(*)(void*),void*); int (*complete)(const char*sql); int (*complete16)(const void*sql); - int (*create_collation)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_collation16)(sqlite3*,const void*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_function)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); - int (*create_function16)(sqlite3*,const void*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); + int (*create_collation)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); int (*data_count)(sqlite3_stmt*pStmt); sqlite3 * (*db_handle)(sqlite3_stmt*); @@ -87147,16 +99648,19 @@ struct sqlite3_api_routines { void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); void (*result_value)(sqlite3_context*,sqlite3_value*); void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); - int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,const char*,const char*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, + const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); char * (*snprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); - int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,char const**,char const**,int*,int*,int*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, + char const**,char const**,int*,int*,int*); void (*thread_cleanup)(void); int (*total_changes)(sqlite3*); void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); - void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,sqlite_int64),void*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, + sqlite_int64),void*); void * (*user_data)(sqlite3_context*); const void * (*value_blob)(sqlite3_value*); int (*value_bytes)(sqlite3_value*); @@ -87178,15 +99682,19 @@ struct sqlite3_api_routines { int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); int (*clear_bindings)(sqlite3_stmt*); /* Added by 3.4.1 */ - int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,void (*xDestroy)(void *)); + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, + void (*xDestroy)(void *)); /* Added by 3.5.0 */ int (*bind_zeroblob)(sqlite3_stmt*,int,int); int (*blob_bytes)(sqlite3_blob*); int (*blob_close)(sqlite3_blob*); - int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,int,sqlite3_blob**); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, + int,sqlite3_blob**); int (*blob_read)(sqlite3_blob*,void*,int,int); int (*blob_write)(sqlite3_blob*,const void*,int,int); - int (*create_collation_v2)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*),void(*)(void*)); + int (*create_collation_v2)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*), + void(*)(void*)); int (*file_control)(sqlite3*,const char*,int,void*); sqlite3_int64 (*memory_highwater)(int); sqlite3_int64 (*memory_used)(void); @@ -87222,7 +99730,11 @@ struct sqlite3_api_routines { int (*backup_step)(sqlite3_backup*,int); const char *(*compileoption_get)(int); int (*compileoption_used)(const char*); - int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*)); int (*db_config)(sqlite3*,int,...); sqlite3_mutex *(*db_mutex)(sqlite3*); int (*db_status)(sqlite3*,int,int*,int*,int); @@ -87236,11 +99748,45 @@ struct sqlite3_api_routines { int (*wal_autocheckpoint)(sqlite3*,int); int (*wal_checkpoint)(sqlite3*,const char*); void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*vsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); + /* Version 3.8.7 and later */ + int (*auto_extension)(void(*)(void)); + int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, + void(*)(void*)); + int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, + void(*)(void*),unsigned char); + int (*cancel_auto_extension)(void(*)(void)); + int (*load_extension)(sqlite3*,const char*,const char*,char**); + void *(*malloc64)(sqlite3_uint64); + sqlite3_uint64 (*msize)(void*); + void *(*realloc64)(void*,sqlite3_uint64); + void (*reset_auto_extension)(void); + void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, + void(*)(void*)); + void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, + void(*)(void*), unsigned char); + int (*strglob)(const char*,const char*); }; /* ** The following macros redefine the API routines so that they are -** redirected throught the global sqlite3_api structure. +** redirected through the global sqlite3_api structure. ** ** This header file is also used by the loadext.c source file ** (part of the main SQLite library - not an extension) so that @@ -87436,15 +99982,58 @@ struct sqlite3_api_routines { #define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint #define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint #define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 +/* Version 3.8.7 and later */ +#define sqlite3_auto_extension sqlite3_api->auto_extension +#define sqlite3_bind_blob64 sqlite3_api->bind_blob64 +#define sqlite3_bind_text64 sqlite3_api->bind_text64 +#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension +#define sqlite3_load_extension sqlite3_api->load_extension +#define sqlite3_malloc64 sqlite3_api->malloc64 +#define sqlite3_msize sqlite3_api->msize +#define sqlite3_realloc64 sqlite3_api->realloc64 +#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension +#define sqlite3_result_blob64 sqlite3_api->result_blob64 +#define sqlite3_result_text64 sqlite3_api->result_text64 +#define sqlite3_strglob sqlite3_api->strglob #endif /* SQLITE_CORE */ -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +# define SQLITE_EXTENSION_INIT3 \ + extern const sqlite3_api_routines *sqlite3_api; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +# define SQLITE_EXTENSION_INIT3 /*no-op*/ +#endif #endif /* _SQLITE3EXT_H_ */ /************** End of sqlite3ext.h ******************************************/ /************** Continuing where we left off in loadext.c ********************/ +/* #include */ #ifndef SQLITE_OMIT_LOAD_EXTENSION @@ -87510,6 +100099,8 @@ struct sqlite3_api_routines { # define sqlite3_create_module 0 # define sqlite3_create_module_v2 0 # define sqlite3_declare_vtab 0 +# define sqlite3_vtab_config 0 +# define sqlite3_vtab_on_conflict 0 #endif #ifdef SQLITE_OMIT_SHARED_CACHE @@ -87533,6 +100124,7 @@ struct sqlite3_api_routines { #define sqlite3_blob_open 0 #define sqlite3_blob_read 0 #define sqlite3_blob_write 0 +#define sqlite3_blob_reopen 0 #endif /* @@ -87798,6 +100390,35 @@ static const sqlite3_api_routines sqlite3Apis = { 0, 0, #endif + sqlite3_blob_reopen, + sqlite3_vtab_config, + sqlite3_vtab_on_conflict, + sqlite3_close_v2, + sqlite3_db_filename, + sqlite3_db_readonly, + sqlite3_db_release_memory, + sqlite3_errstr, + sqlite3_stmt_busy, + sqlite3_stmt_readonly, + sqlite3_stricmp, + sqlite3_uri_boolean, + sqlite3_uri_int64, + sqlite3_uri_parameter, + sqlite3_vsnprintf, + sqlite3_wal_checkpoint_v2, + /* Version 3.8.7 and later */ + sqlite3_auto_extension, + sqlite3_bind_blob64, + sqlite3_bind_text64, + sqlite3_cancel_auto_extension, + sqlite3_load_extension, + sqlite3_malloc64, + sqlite3_msize, + sqlite3_realloc64, + sqlite3_reset_auto_extension, + sqlite3_result_blob64, + sqlite3_result_text64, + sqlite3_strglob }; /* @@ -87822,8 +100443,23 @@ static int sqlite3LoadExtension( void *handle; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; void **aHandle; - const int nMsg = 300; + int nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + if( pzErrMsg ) *pzErrMsg = 0; @@ -87840,11 +100476,17 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } - if( zProc==0 ){ - zProc = "sqlite3_extension_init"; - } + zEntry = zProc ? zProc : "sqlite3_extension_init"; handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); + } if( xInit==0 ){ if( pzErrMsg ){ + nMsg += sqlite3Strlen30(zEntry); *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, - "no entry point [%s] in shared library [%s]", zProc,zFile); + "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } - sqlite3OsDlClose(pVfs, handle); } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + } + sqlite3_free(zAltEntry); + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -88016,6 +100696,35 @@ SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ } } +/* +** Cancel a prior call to sqlite3_auto_extension. Remove xInit from the +** set of routines that is invoked for each new database connection, if it +** is currently on the list. If xInit is not on the list, then this +** routine is a no-op. +** +** Return 1 if xInit was found on the list and removed. Return 0 if xInit +** was not on the list. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void (*xInit)(void)){ +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + int i; + int n = 0; + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + for(i=wsdAutoext.nExt-1; i>=0; i--){ + if( wsdAutoext.aExt[i]==xInit ){ + wsdAutoext.nExt--; + wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; + n++; + break; + } + } + sqlite3_mutex_leave(mutex); + return n; +} + /* ** Reset the automatic extension loading mechanism. */ @@ -88044,6 +100753,7 @@ SQLITE_API void sqlite3_reset_auto_extension(void){ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ int i; int go = 1; + int rc; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); wsdAutoextInit; @@ -88066,8 +100776,8 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ } sqlite3_mutex_leave(mutex); zErrmsg = 0; - if( xInit && xInit(db, &zErrmsg, &sqlite3Apis) ){ - sqlite3Error(db, SQLITE_ERROR, + if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ + sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; } @@ -88091,17 +100801,479 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ** This file contains code used to implement the PRAGMA command. */ +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/*************************************************************************** +** The next block of code, including the PragTyp_XXXX macro definitions and +** the aPragmaName[] object is composed of generated code. DO NOT EDIT. +** +** To add new pragmas, edit the code in ../tool/mkpragmatab.tcl and rerun +** that script. Then copy/paste the output in place of the following: +*/ +#define PragTyp_HEADER_VALUE 0 +#define PragTyp_AUTO_VACUUM 1 +#define PragTyp_FLAG 2 +#define PragTyp_BUSY_TIMEOUT 3 +#define PragTyp_CACHE_SIZE 4 +#define PragTyp_CASE_SENSITIVE_LIKE 5 +#define PragTyp_COLLATION_LIST 6 +#define PragTyp_COMPILE_OPTIONS 7 +#define PragTyp_DATA_STORE_DIRECTORY 8 +#define PragTyp_DATABASE_LIST 9 +#define PragTyp_DEFAULT_CACHE_SIZE 10 +#define PragTyp_ENCODING 11 +#define PragTyp_FOREIGN_KEY_CHECK 12 +#define PragTyp_FOREIGN_KEY_LIST 13 +#define PragTyp_INCREMENTAL_VACUUM 14 +#define PragTyp_INDEX_INFO 15 +#define PragTyp_INDEX_LIST 16 +#define PragTyp_INTEGRITY_CHECK 17 +#define PragTyp_JOURNAL_MODE 18 +#define PragTyp_JOURNAL_SIZE_LIMIT 19 +#define PragTyp_LOCK_PROXY_FILE 20 +#define PragTyp_LOCKING_MODE 21 +#define PragTyp_PAGE_COUNT 22 +#define PragTyp_MMAP_SIZE 23 +#define PragTyp_PAGE_SIZE 24 +#define PragTyp_SECURE_DELETE 25 +#define PragTyp_SHRINK_MEMORY 26 +#define PragTyp_SOFT_HEAP_LIMIT 27 +#define PragTyp_STATS 28 +#define PragTyp_SYNCHRONOUS 29 +#define PragTyp_TABLE_INFO 30 +#define PragTyp_TEMP_STORE 31 +#define PragTyp_TEMP_STORE_DIRECTORY 32 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 +#define PragFlag_NeedSchema 0x01 +static const struct sPragmaNames { + const char *const zName; /* Name of pragma */ + u8 ePragTyp; /* PragTyp_XXX value */ + u8 mPragFlag; /* Zero or more PragFlag_XXX values */ + u32 iArg; /* Extra argument */ +} aPragmaNames[] = { +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) + { /* zName: */ "activate_extensions", + /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "application_id", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + { /* zName: */ "auto_vacuum", + /* ePragTyp: */ PragTyp_AUTO_VACUUM, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) + { /* zName: */ "automatic_index", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_AutoIndex }, +#endif +#endif + { /* zName: */ "busy_timeout", + /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "cache_size", + /* ePragTyp: */ PragTyp_CACHE_SIZE, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "cache_spill", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CacheSpill }, +#endif + { /* zName: */ "case_sensitive_like", + /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "checkpoint_fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CkptFullFSync }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "collation_list", + /* ePragTyp: */ PragTyp_COLLATION_LIST, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) + { /* zName: */ "compile_options", + /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "count_changes", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CountRows }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN + { /* zName: */ "data_store_directory", + /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "database_list", + /* ePragTyp: */ PragTyp_DATABASE_LIST, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + { /* zName: */ "default_cache_size", + /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + { /* zName: */ "defer_foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_DeferFKs }, +#endif +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "empty_result_callbacks", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_NullCallback }, +#endif +#if !defined(SQLITE_OMIT_UTF16) + { /* zName: */ "encoding", + /* ePragTyp: */ PragTyp_ENCODING, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + { /* zName: */ "foreign_key_check", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) + { /* zName: */ "foreign_key_list", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + { /* zName: */ "foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ForeignKeys }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "freelist_count", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "full_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_FullColNames }, + { /* zName: */ "fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_FullFSync }, +#endif +#if defined(SQLITE_HAS_CODEC) + { /* zName: */ "hexkey", + /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "hexrekey", + /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_CHECK) + { /* zName: */ "ignore_check_constraints", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_IgnoreChecks }, +#endif +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + { /* zName: */ "incremental_vacuum", + /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "index_info", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "index_list", + /* ePragTyp: */ PragTyp_INDEX_LIST, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + { /* zName: */ "integrity_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "journal_mode", + /* ePragTyp: */ PragTyp_JOURNAL_MODE, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "journal_size_limit", + /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_HAS_CODEC) + { /* zName: */ "key", + /* ePragTyp: */ PragTyp_KEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "legacy_file_format", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_LegacyFileFmt }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE + { /* zName: */ "lock_proxy_file", + /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + { /* zName: */ "lock_status", + /* ePragTyp: */ PragTyp_LOCK_STATUS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "locking_mode", + /* ePragTyp: */ PragTyp_LOCKING_MODE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "max_page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "mmap_size", + /* ePragTyp: */ PragTyp_MMAP_SIZE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, + { /* zName: */ "page_size", + /* ePragTyp: */ PragTyp_PAGE_SIZE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_DEBUG) + { /* zName: */ "parser_trace", + /* ePragTyp: */ PragTyp_PARSER_TRACE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "query_only", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_QueryOnly }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + { /* zName: */ "quick_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "read_uncommitted", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ReadUncommitted }, + { /* zName: */ "recursive_triggers", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_RecTriggers }, +#endif +#if defined(SQLITE_HAS_CODEC) + { /* zName: */ "rekey", + /* ePragTyp: */ PragTyp_REKEY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "reverse_unordered_selects", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ReverseOrder }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "schema_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "secure_delete", + /* ePragTyp: */ PragTyp_SECURE_DELETE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "short_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_ShortColNames }, +#endif + { /* zName: */ "shrink_memory", + /* ePragTyp: */ PragTyp_SHRINK_MEMORY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "soft_heap_limit", + /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + { /* zName: */ "sql_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_SqlTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "stats", + /* ePragTyp: */ PragTyp_STATS, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "synchronous", + /* ePragTyp: */ PragTyp_SYNCHRONOUS, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + { /* zName: */ "table_info", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + { /* zName: */ "temp_store", + /* ePragTyp: */ PragTyp_TEMP_STORE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "temp_store_directory", + /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif + { /* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "user_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + { /* zName: */ "vdbe_addoptrace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeAddopTrace }, + { /* zName: */ "vdbe_debug", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, + { /* zName: */ "vdbe_eqp", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeEQP }, + { /* zName: */ "vdbe_listing", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeListing }, + { /* zName: */ "vdbe_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_WAL) + { /* zName: */ "wal_autocheckpoint", + /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, + { /* zName: */ "wal_checkpoint", + /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, + /* ePragFlag: */ PragFlag_NeedSchema, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + { /* zName: */ "writable_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, +#endif +}; +/* Number of pragmas: 57 on by default, 70 total. */ +/* End of the automatically generated pragma table. +***************************************************************************/ + /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or -** unrecognized string argument. +** unrecognized string argument. The FULL option is disallowed +** if the omitFull parameter it 1. ** ** Note that the values returned are one less that the values that ** should be passed into sqlite3BtreeSetSafetyLevel(). The is done ** to support legacy SQL code. The safety level used to be boolean ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ -static u8 getSafetyLevel(const char *z){ +static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ /* 123456789 123456789 */ static const char zText[] = "onoffalseyestruefull"; static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; @@ -88112,19 +101284,19 @@ static u8 getSafetyLevel(const char *z){ return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); - for(i=0; iaDb[1].pBt); db->aDb[1].pBt = 0; - sqlite3ResetInternalSchema(db, -1); + sqlite3ResetAllSchemasOfConnection(db); } return SQLITE_OK; } @@ -88235,90 +101407,35 @@ static void returnSingleInt(Parse *pParse, const char *zLabel, i64 value){ sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); } -#ifndef SQLITE_OMIT_FLAG_PRAGMAS + /* -** Check to see if zRight and zLeft refer to a pragma that queries -** or changes one of the flags in db->flags. Return 1 if so and 0 if not. -** Also, implement the pragma. +** Set the safety_level and pager flags for pager iDb. Or if iDb<0 +** set these values for all pagers. */ -static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ - static const struct sPragmaType { - const char *zName; /* Name of the pragma */ - int mask; /* Mask for the db->flags value */ - } aPragma[] = { - { "full_column_names", SQLITE_FullColNames }, - { "short_column_names", SQLITE_ShortColNames }, - { "count_changes", SQLITE_CountRows }, - { "empty_result_callbacks", SQLITE_NullCallback }, - { "legacy_file_format", SQLITE_LegacyFileFmt }, - { "fullfsync", SQLITE_FullFSync }, - { "checkpoint_fullfsync", SQLITE_CkptFullFSync }, - { "reverse_unordered_selects", SQLITE_ReverseOrder }, -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX - { "automatic_index", SQLITE_AutoIndex }, -#endif -#ifdef SQLITE_DEBUG - { "sql_trace", SQLITE_SqlTrace }, - { "vdbe_listing", SQLITE_VdbeListing }, - { "vdbe_trace", SQLITE_VdbeTrace }, -#endif -#ifndef SQLITE_OMIT_CHECK - { "ignore_check_constraints", SQLITE_IgnoreChecks }, -#endif - /* The following is VERY experimental */ - { "writable_schema", SQLITE_WriteSchema|SQLITE_RecoveryMode }, - { "omit_readlock", SQLITE_NoReadlock }, - - /* TODO: Maybe it shouldn't be possible to change the ReadUncommitted - ** flag if there are any active statements. */ - { "read_uncommitted", SQLITE_ReadUncommitted }, - { "recursive_triggers", SQLITE_RecTriggers }, - - /* This flag may only be set if both foreign-key and trigger support - ** are present in the build. */ -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { "foreign_keys", SQLITE_ForeignKeys }, -#endif - }; - int i; - const struct sPragmaType *p; - for(i=0, p=aPragma; izName)==0 ){ - sqlite3 *db = pParse->db; - Vdbe *v; - v = sqlite3GetVdbe(pParse); - assert( v!=0 ); /* Already allocated by sqlite3Pragma() */ - if( ALWAYS(v) ){ - if( zRight==0 ){ - returnSingleInt(pParse, p->zName, (db->flags & p->mask)!=0 ); - }else{ - int mask = p->mask; /* Mask of bits to set or clear. */ - if( db->autoCommit==0 ){ - /* Foreign key support may not be enabled or disabled while not - ** in auto-commit mode. */ - mask &= ~(SQLITE_ForeignKeys); - } - - if( sqlite3GetBoolean(zRight) ){ - db->flags |= mask; - }else{ - db->flags &= ~mask; - } - - /* Many of the flag-pragmas modify the code generated by the SQL - ** compiler (eg. count_changes). So add an opcode to expire all - ** compiled SQL statements after modifying a pragma value. - */ - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); - } +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +static void setAllPagerFlags(sqlite3 *db){ + if( db->autoCommit ){ + Db *pDb = db->aDb; + int n = db->nDb; + assert( SQLITE_FullFSync==PAGER_FULLFSYNC ); + assert( SQLITE_CkptFullFSync==PAGER_CKPT_FULLFSYNC ); + assert( SQLITE_CacheSpill==PAGER_CACHESPILL ); + assert( (PAGER_FULLFSYNC | PAGER_CKPT_FULLFSYNC | PAGER_CACHESPILL) + == PAGER_FLAGS_MASK ); + assert( (pDb->safety_level & PAGER_SYNCHRONOUS_MASK)==pDb->safety_level ); + while( (n--) > 0 ){ + if( pDb->pBt ){ + sqlite3BtreeSetPagerFlags(pDb->pBt, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK) ); } - - return 1; + pDb++; } } - return 0; } -#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ +#else +# define setAllPagerFlags(X) /* no-op */ +#endif + /* ** Return a human-readable name for a constraint resolution action. @@ -88389,10 +101506,14 @@ SQLITE_PRIVATE void sqlite3Pragma( char *zRight = 0; /* Nul-terminated UTF-8 string , or NULL */ const char *zDb = 0; /* The database name */ Token *pId; /* Pointer to token */ + char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ int iDb; /* Database index for */ - sqlite3 *db = pParse->db; - Db *pDb; - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); + int lwr, upr, mid; /* Binary search bounds */ + int rc; /* return value form SQLITE_FCNTL_PRAGMA */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* The specific database being pragmaed */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ + if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; @@ -88423,8 +101544,62 @@ SQLITE_PRIVATE void sqlite3Pragma( if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } - -#ifndef SQLITE_OMIT_PAGER_PRAGMAS + + /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS + ** connection. If it returns SQLITE_OK, then assume that the VFS + ** handled the pragma and generate a no-op prepared statement. + */ + aFcntl[0] = 0; + aFcntl[1] = zLeft; + aFcntl[2] = zRight; + aFcntl[3] = 0; + db->busyHandler.nBusy = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + if( rc==SQLITE_OK ){ + if( aFcntl[0] ){ + int mem = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, mem, 0, aFcntl[0], 0); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); + sqlite3_free(aFcntl[0]); + } + goto pragma_out; + } + if( rc!=SQLITE_NOTFOUND ){ + if( aFcntl[0] ){ + sqlite3ErrorMsg(pParse, "%s", aFcntl[0]); + sqlite3_free(aFcntl[0]); + } + pParse->nErr++; + pParse->rc = rc; + goto pragma_out; + } + + /* Locate the pragma in the lookup table */ + lwr = 0; + upr = ArraySize(aPragmaNames)-1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + rc = sqlite3_stricmp(zLeft, aPragmaNames[mid].zName); + if( rc==0 ) break; + if( rc<0 ){ + upr = mid - 1; + }else{ + lwr = mid + 1; + } + } + if( lwr>upr ) goto pragma_out; + + /* Make sure the database schema is loaded if the pragma requires that */ + if( (aPragmaNames[mid].mPragFlag & PragFlag_NeedSchema)!=0 ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + } + + /* Jump to the appropriate pragma handler */ + switch( aPragmaNames[mid].ePragTyp ){ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* ** PRAGMA [database.]default_cache_size ** PRAGMA [database.]default_cache_size=N @@ -88441,25 +101616,26 @@ SQLITE_PRIVATE void sqlite3Pragma( ** size. But continue to take the absolute value of the default cache ** size of historical compatibility. */ - if( sqlite3StrICmp(zLeft,"default_cache_size")==0 ){ + case PragTyp_DEFAULT_CACHE_SIZE: { + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 2, 0}, { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; int addr; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); pParse->nMem += 2; - addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize); + addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+1, iDb); sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); @@ -88472,8 +101648,11 @@ SQLITE_PRIVATE void sqlite3Pragma( pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } - }else + break; + } +#endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* ** PRAGMA [database.]page_size ** PRAGMA [database.]page_size=N @@ -88483,7 +101662,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** database page size value. The value can only be set if ** the database has not yet been created. */ - if( sqlite3StrICmp(zLeft,"page_size")==0 ){ + case PragTyp_PAGE_SIZE: { Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ @@ -88494,11 +101673,12 @@ SQLITE_PRIVATE void sqlite3Pragma( ** buffer that the pager module resizes using sqlite3_realloc(). */ db->nextPagesize = sqlite3Atoi(zRight); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ db->mallocFailed = 1; } } - }else + break; + } /* ** PRAGMA [database.]secure_delete @@ -88508,12 +101688,12 @@ SQLITE_PRIVATE void sqlite3Pragma( ** secure_delete flag. The second form changes the secure_delete ** flag setting and reports thenew value. */ - if( sqlite3StrICmp(zLeft,"secure_delete")==0 ){ + case PragTyp_SECURE_DELETE: { Btree *pBt = pDb->pBt; int b = -1; assert( pBt!=0 ); if( zRight ){ - b = sqlite3GetBoolean(zRight); + b = sqlite3GetBoolean(zRight, 0); } if( pId2->n==0 && b>=0 ){ int ii; @@ -88523,7 +101703,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } b = sqlite3BtreeSecureDelete(pBt, b); returnSingleInt(pParse, "secure_delete", b); - }else + break; + } /* ** PRAGMA [database.]max_page_count @@ -88534,32 +101715,35 @@ SQLITE_PRIVATE void sqlite3Pragma( ** second form attempts to change this setting. Both ** forms return the current setting. ** + ** The absolute value of N is used. This is undocumented and might + ** change. The only purpose is to provide an easy way to test + ** the sqlite3AbsInt32() function. + ** ** PRAGMA [database.]page_count ** ** Return the number of pages in the specified database. */ - if( sqlite3StrICmp(zLeft,"page_count")==0 - || sqlite3StrICmp(zLeft,"max_page_count")==0 - ){ + case PragTyp_PAGE_COUNT: { int iReg; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3CodeVerifySchema(pParse, iDb); iReg = ++pParse->nMem; - if( zLeft[0]=='p' ){ + if( sqlite3Tolower(zLeft[0])=='p' ){ sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); }else{ - sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight)); + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, + sqlite3AbsInt32(sqlite3Atoi(zRight))); } sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); - }else + break; + } /* ** PRAGMA [database.]locking_mode ** PRAGMA [database.]locking_mode = (normal|exclusive) */ - if( sqlite3StrICmp(zLeft,"locking_mode")==0 ){ + case PragTyp_LOCKING_MODE: { const char *zRet = "normal"; int eMode = getLockingMode(zRight); @@ -88592,7 +101776,8 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = sqlite3PagerLockingMode(pPager, eMode); } - assert(eMode==PAGER_LOCKINGMODE_NORMAL||eMode==PAGER_LOCKINGMODE_EXCLUSIVE); + assert( eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } @@ -88600,23 +101785,18 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - }else + break; + } /* ** PRAGMA [database.]journal_mode ** PRAGMA [database.]journal_mode = ** (delete|persist|off|truncate|memory|wal|off) */ - if( sqlite3StrICmp(zLeft,"journal_mode")==0 ){ + case PragTyp_JOURNAL_MODE: { int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - /* Force the schema to be loaded on all databases. This cases all - ** database files to be opened and the journal_modes set. */ - if( sqlite3ReadSchema(pParse) ){ - goto pragma_out; - } - sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); @@ -88648,7 +101828,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } } sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - }else + break; + } /* ** PRAGMA [database.]journal_size_limit @@ -88656,16 +101837,17 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** Get or set the size limit on rollback journal files. */ - if( sqlite3StrICmp(zLeft,"journal_size_limit")==0 ){ + case PragTyp_JOURNAL_SIZE_LIMIT: { Pager *pPager = sqlite3BtreePager(pDb->pBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8); + sqlite3DecOrHexToI64(zRight, &iLimit); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); returnSingleInt(pParse, "journal_size_limit", iLimit); - }else + break; + } #endif /* SQLITE_OMIT_PAGER_PRAGMAS */ @@ -88677,57 +101859,48 @@ SQLITE_PRIVATE void sqlite3Pragma( ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( sqlite3StrICmp(zLeft,"auto_vacuum")==0 ){ + case PragTyp_AUTO_VACUUM: { Btree *pBt = pDb->pBt; assert( pBt!=0 ); - if( sqlite3ReadSchema(pParse) ){ - goto pragma_out; - } if( !zRight ){ - int auto_vacuum; - if( ALWAYS(pBt) ){ - auto_vacuum = sqlite3BtreeGetAutoVacuum(pBt); - }else{ - auto_vacuum = SQLITE_DEFAULT_AUTOVACUUM; - } - returnSingleInt(pParse, "auto_vacuum", auto_vacuum); + returnSingleInt(pParse, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); db->nextAutovac = (u8)eAuto; - if( ALWAYS(eAuto>=0) ){ - /* Call SetAutoVacuum() to set initialize the internal auto and - ** incr-vacuum flags. This is required in case this connection - ** creates the database file. It is important that it is created - ** as an auto-vacuum capable db. + /* Call SetAutoVacuum() to set initialize the internal auto and + ** incr-vacuum flags. This is required in case this connection + ** creates the database file. It is important that it is created + ** as an auto-vacuum capable db. + */ + rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ + /* When setting the auto_vacuum mode to either "full" or + ** "incremental", write the value of meta[6] in the database + ** file. Before writing to meta[6], check that meta[3] indicates + ** that this really is an auto-vacuum capable database. */ - int rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); - if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ - /* When setting the auto_vacuum mode to either "full" or - ** "incremental", write the value of meta[6] in the database - ** file. Before writing to meta[6], check that meta[3] indicates - ** that this really is an auto-vacuum capable database. - */ - static const VdbeOpList setMeta6[] = { - { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, - { OP_If, 1, 0, 0}, /* 2 */ - { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ - { OP_Integer, 0, 1, 0}, /* 4 */ - { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ - }; - int iAddr; - iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6); - sqlite3VdbeChangeP1(v, iAddr, iDb); - sqlite3VdbeChangeP1(v, iAddr+1, iDb); - sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); - sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); - sqlite3VdbeChangeP1(v, iAddr+5, iDb); - sqlite3VdbeUsesBtree(v, iDb); - } + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList setMeta6[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, + { OP_If, 1, 0, 0}, /* 2 */ + { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ + { OP_Integer, 0, 1, 0}, /* 4 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ + }; + int iAddr; + iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + sqlite3VdbeChangeP1(v, iAddr, iDb); + sqlite3VdbeChangeP1(v, iAddr+1, iDb); + sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); + sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); + sqlite3VdbeChangeP1(v, iAddr+5, iDb); + sqlite3VdbeUsesBtree(v, iDb); } } - }else + break; + } #endif /* @@ -88736,22 +101909,20 @@ SQLITE_PRIVATE void sqlite3Pragma( ** Do N steps of incremental vacuuming on a database. */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( sqlite3StrICmp(zLeft,"incremental_vacuum")==0 ){ + case PragTyp_INCREMENTAL_VACUUM: { int iLimit, addr; - if( sqlite3ReadSchema(pParse) ){ - goto pragma_out; - } if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ iLimit = 0x7fffffff; } sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); - addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); + addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_ResultRow, 1); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr); - }else + break; + } #endif #ifndef SQLITE_OMIT_PAGER_PRAGMAS @@ -88760,26 +101931,67 @@ SQLITE_PRIVATE void sqlite3Pragma( ** PRAGMA [database.]cache_size=N ** ** The first form reports the current local setting for the - ** page cache size. The local setting can be different from - ** the persistent cache size value that is stored in the database - ** file itself. The value returned is the maximum number of - ** pages in the page cache. The second form sets the local - ** page cache size value. It does not change the persistent - ** cache size stored on the disk so the cache size will revert - ** to its default value when the database is closed and reopened. - ** N should be a positive integer. + ** page cache size. The second form sets the local + ** page cache size value. If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. */ - if( sqlite3StrICmp(zLeft,"cache_size")==0 ){ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; + case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ - int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); + int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } - }else + break; + } + + /* + ** PRAGMA [database.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + case PragTyp_MMAP_SIZE: { + sqlite3_int64 sz; +#if SQLITE_MAX_MMAP_SIZE>0 + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3DecOrHexToI64(zRight, &sz); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_MMAP_SIZE, &sz); +#else + sz = 0; + rc = SQLITE_OK; +#endif + if( rc==SQLITE_OK ){ + returnSingleInt(pParse, "mmap_size", sz); + }else if( rc!=SQLITE_NOTFOUND ){ + pParse->nErr++; + pParse->rc = rc; + } + break; + } /* ** PRAGMA temp_store @@ -88792,13 +102004,14 @@ SQLITE_PRIVATE void sqlite3Pragma( ** Note that it is possible for the library compile-time options to ** override this setting */ - if( sqlite3StrICmp(zLeft, "temp_store")==0 ){ + case PragTyp_TEMP_STORE: { if( !zRight ){ returnSingleInt(pParse, "temp_store", db->temp_store); }else{ changeTempStorage(pParse, zRight); } - }else + break; + } /* ** PRAGMA temp_store_directory @@ -88810,7 +102023,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** If temporary directory is changed, then invalidateTempStorage. ** */ - if( sqlite3StrICmp(zLeft, "temp_store_directory")==0 ){ + case PragTyp_TEMP_STORE_DIRECTORY: { if( !zRight ){ if( sqlite3_temp_directory ){ sqlite3VdbeSetNumCols(v, 1); @@ -88822,7 +102035,6 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ - int rc; int res; rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ @@ -88844,30 +102056,69 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } - }else + break; + } -#if !defined(SQLITE_ENABLE_LOCKING_STYLE) -# if defined(__APPLE__) -# define SQLITE_ENABLE_LOCKING_STYLE 1 -# else -# define SQLITE_ENABLE_LOCKING_STYLE 0 -# endif +#if SQLITE_OS_WIN + /* + ** PRAGMA data_store_directory + ** PRAGMA data_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the data_store_directory flag. Changing + ** the value sets a specific directory to be used for database files that + ** were specified with a relative pathname. Setting to a null string reverts + ** to the default database directory, which for database files specified with + ** a relative path will probably be based on the current directory for the + ** process. Database file specified with an absolute path are not impacted + ** by this setting, regardless of its value. + ** + */ + case PragTyp_DATA_STORE_DIRECTORY: { + if( !zRight ){ + if( sqlite3_data_directory ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "data_store_directory", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_data_directory, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + sqlite3_free(sqlite3_data_directory); + if( zRight[0] ){ + sqlite3_data_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_data_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ + } + break; + } #endif + #if SQLITE_ENABLE_LOCKING_STYLE /* - ** PRAGMA [database.]lock_proxy_file - ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" - ** - ** Return or set the value of the lock_proxy_file flag. Changing - ** the value sets a specific file to be used for database access locks. - ** - */ - if( sqlite3StrICmp(zLeft, "lock_proxy_file")==0 ){ + ** PRAGMA [database.]lock_proxy_file + ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" + ** + ** Return or set the value of the lock_proxy_file flag. Changing + ** the value sets a specific file to be used for database access locks. + ** + */ + case PragTyp_LOCK_PROXY_FILE: { if( !zRight ){ Pager *pPager = sqlite3BtreePager(pDb->pBt); char *proxy_file_path = NULL; sqlite3_file *pFile = sqlite3PagerFile(pPager); - sqlite3OsFileControl(pFile, SQLITE_GET_LOCKPROXYFILE, + sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); if( proxy_file_path ){ @@ -88893,7 +102144,8 @@ SQLITE_PRIVATE void sqlite3Pragma( goto pragma_out; } } - }else + break; + } #endif /* SQLITE_ENABLE_LOCKING_STYLE */ /* @@ -88905,8 +102157,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** default value will be restored the next time the database is ** opened. */ - if( sqlite3StrICmp(zLeft,"synchronous")==0 ){ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; + case PragTyp_SYNCHRONOUS: { if( !zRight ){ returnSingleInt(pParse, "synchronous", pDb->safety_level-1); }else{ @@ -88914,17 +102165,49 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); }else{ - pDb->safety_level = getSafetyLevel(zRight)+1; + pDb->safety_level = getSafetyLevel(zRight,0,1)+1; + setAllPagerFlags(db); } } - }else + break; + } #endif /* SQLITE_OMIT_PAGER_PRAGMAS */ #ifndef SQLITE_OMIT_FLAG_PRAGMAS - if( flagPragma(pParse, zLeft, zRight) ){ - /* The flagPragma() subroutine also generates any necessary code - ** there is nothing more to do here */ - }else + case PragTyp_FLAG: { + if( zRight==0 ){ + returnSingleInt(pParse, aPragmaNames[mid].zName, + (db->flags & aPragmaNames[mid].iArg)!=0 ); + }else{ + int mask = aPragmaNames[mid].iArg; /* Mask of bits to set or clear. */ + if( db->autoCommit==0 ){ + /* Foreign key support may not be enabled or disabled while not + ** in auto-commit mode. */ + mask &= ~(SQLITE_ForeignKeys); + } +#if SQLITE_USER_AUTHENTICATION + if( db->auth.authLevel==UAUTH_User ){ + /* Do not allow non-admin users to modify the schema arbitrarily */ + mask &= ~(SQLITE_WriteSchema); + } +#endif + + if( sqlite3GetBoolean(zRight, 0) ){ + db->flags |= mask; + }else{ + db->flags &= ~mask; + if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + } + + /* Many of the flag-pragmas modify the code generated by the SQL + ** compiler (eg. count_changes). So add an opcode to expire all + ** compiled SQL statements after modifying a pragma value. + */ + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + setAllPagerFlags(db); + } + break; + } #endif /* SQLITE_OMIT_FLAG_PRAGMAS */ #ifndef SQLITE_OMIT_SCHEMA_PRAGMAS @@ -88940,16 +102223,17 @@ SQLITE_PRIVATE void sqlite3Pragma( ** notnull: True if 'NOT NULL' is part of column declaration ** dflt_value: The default value for the column, if any. */ - if( sqlite3StrICmp(zLeft, "table_info")==0 && zRight ){ + case PragTyp_TABLE_INFO: if( zRight ){ Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - int i; + int i, k; int nHidden = 0; Column *pCol; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); @@ -88972,27 +102256,67 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, 5); } - sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6); + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + } + sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } - }else + } + break; - if( sqlite3StrICmp(zLeft, "index_info")==0 && zRight ){ + case PragTyp_STATS: { + Index *pIdx; + HashElem *i; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + pParse->nMem = 4; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "index", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "width", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "height", SQLITE_STATIC); + for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pTab->zName, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, 2); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->szTabRow), 3); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + } + } + } + break; + + case PragTyp_INDEX_INFO: if( zRight ){ Index *pIdx; Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; pIdx = sqlite3FindIndex(db, zRight, zDb); if( pIdx ){ int i; pTab = pIdx->pTable; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); - for(i=0; inColumn; i++){ - int cnum = pIdx->aiColumn[i]; + for(i=0; inKeyCol; i++){ + i16 cnum = pIdx->aiColumn[i]; sqlite3VdbeAddOp2(v, OP_Integer, i, 1); sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); assert( pTab->nCol>cnum ); @@ -89000,38 +102324,34 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } - }else + } + break; - if( sqlite3StrICmp(zLeft, "index_list")==0 && zRight ){ + case PragTyp_INDEX_LIST: if( zRight ){ Index *pIdx; Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; + int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ v = sqlite3GetVdbe(pParse); - pIdx = pTab->pIndex; - if( pIdx ){ - int i = 0; - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); - while(pIdx){ - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); - ++i; - pIdx = pIdx->pNext; - } + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); + for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } - }else + } + break; - if( sqlite3StrICmp(zLeft, "database_list")==0 ){ + case PragTyp_DATABASE_LIST: { int i; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); @@ -89046,9 +102366,10 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } - }else + } + break; - if( sqlite3StrICmp(zLeft, "collation_list")==0 ){ + case PragTyp_COLLATION_LIST: { int i = 0; HashElem *p; sqlite3VdbeSetNumCols(v, 2); @@ -89061,14 +102382,14 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } - }else + } + break; #endif /* SQLITE_OMIT_SCHEMA_PRAGMAS */ #ifndef SQLITE_OMIT_FOREIGN_KEY - if( sqlite3StrICmp(zLeft, "foreign_key_list")==0 && zRight ){ + case PragTyp_FOREIGN_KEY_LIST: if( zRight ){ FKey *pFK; Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ v = sqlite3GetVdbe(pParse); @@ -89077,6 +102398,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); @@ -89107,59 +102429,191 @@ SQLITE_PRIVATE void sqlite3Pragma( } } } - }else + } + break; +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + case PragTyp_FOREIGN_KEY_CHECK: { + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + sqlite3CodeVerifySchema(pParse, iDb); + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || pTab->pFKey==0 ) continue; + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, + P4_TRANSIENT); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + if( pParent==0 ) continue; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + } + }else{ + k = 0; + break; + } + } + assert( pParse->nErr>0 || pFK==0 ); + if( pFK ) break; + if( pParse->nTabnTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); VdbeCoverage(v); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + pIdx = 0; + aiCols = 0; + if( pParent ){ + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 ); + } + addrOk = sqlite3VdbeMakeLabel(v); + if( pParent && pIdx==0 ){ + int iKey = pFK->aCol[0].iFrom; + assert( iKey>=0 && iKeynCol ); + if( iKey!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); + sqlite3ColumnDefault(v, pTab, iKey, regRow); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); + } + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else{ + for(j=0; jnCol; j++){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, + aiCols ? aiCols[j] : pFK->aCol[j].iFrom, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); + } + if( pParent ){ + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, + sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + VdbeCoverage(v); + } + } + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, + pFK->zTo, P4_TRANSIENT); + sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrTop); + } + } + break; +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ #ifndef NDEBUG - if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ + case PragTyp_PARSER_TRACE: { if( zRight ){ - if( sqlite3GetBoolean(zRight) ){ + if( sqlite3GetBoolean(zRight, 0) ){ sqlite3ParserTrace(stderr, "parser: "); }else{ sqlite3ParserTrace(0, 0); } } - }else + } + break; #endif /* Reinstall the LIKE and GLOB functions. The variant of LIKE ** used will be case sensitive or not depending on the RHS. */ - if( sqlite3StrICmp(zLeft, "case_sensitive_like")==0 ){ + case PragTyp_CASE_SENSITIVE_LIKE: { if( zRight ){ - sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight)); + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight, 0)); } - }else + } + break; #ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX # define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 #endif #ifndef SQLITE_OMIT_INTEGRITY_CHECK - /* Pragma "quick_check" is an experimental reduced version of + /* Pragma "quick_check" is reduced version of ** integrity_check designed to detect most database corruption ** without most of the overhead of a full integrity-check. */ - if( sqlite3StrICmp(zLeft, "integrity_check")==0 - || sqlite3StrICmp(zLeft, "quick_check")==0 - ){ + case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; /* Code that appears at the end of the integrity check. If no error ** messages have been generated, output OK. Otherwise output the ** error message */ + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { - { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_IfNeg, 1, 0, 0}, /* 1 */ - { OP_String8, 0, 3, 0}, /* 2 */ + { OP_IfNeg, 1, 0, 0}, /* 0 */ + { OP_String8, 0, 3, 0}, /* 1 */ { OP_ResultRow, 3, 1, 0}, }; - int isQuick = (zLeft[0]=='q'); + int isQuick = (sqlite3Tolower(zLeft[0])=='q'); + + /* If the PRAGMA command was of the form "PRAGMA .integrity_check", + ** then iDb is set to the index of the database identified by . + ** In this case, the integrity of database iDb only is verified by + ** the VDBE created below. + ** + ** Otherwise, if the command was simply "PRAGMA integrity_check" (or + ** "PRAGMA quick_check"), then iDb is set to 0. In this case, set iDb + ** to -1 here, to indicate that the VDBE should verify the integrity + ** of all attached databases. */ + assert( iDb>=0 ); + assert( iDb==0 || pId2->z ); + if( pId2->z==0 ) iDb = -1; /* Initialize the VDBE program */ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; pParse->nMem = 6; sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); @@ -89181,9 +102635,11 @@ SQLITE_PRIVATE void sqlite3Pragma( int cnt = 0; if( OMIT_TEMPDB && i==1 ) continue; + if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); @@ -89192,28 +102648,30 @@ SQLITE_PRIVATE void sqlite3Pragma( ** Begin by filling registers 2, 3, ... with the root pages numbers ** for all tables and indices in the database. */ - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( sqlite3SchemaMutexHeld(db, i, 0) ); pTbls = &db->aDb[i].pSchema->tblHash; for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); - cnt++; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); + VdbeComment((v, "%s", pTab->zName)); + cnt++; + } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); + VdbeComment((v, "%s", pIdx->zName)); cnt++; } } /* Make sure sufficient number of registers have been allocated */ - if( pParse->nMem < cnt+4 ){ - pParse->nMem = cnt+4; - } + pParse->nMem = MAX( pParse->nMem, cnt+8 ); /* Do the b-tree integrity checks */ sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); sqlite3VdbeChangeP5(v, (u8)i); - addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); + addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), P4_DYNAMIC); @@ -89226,77 +102684,127 @@ SQLITE_PRIVATE void sqlite3Pragma( */ for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); - Index *pIdx; + Index *pIdx, *pPk; + Index *pPrior = 0; int loopTop; + int iDataCur, iIdxCur; + int r1 = -1; if( pTab->pIndex==0 ) continue; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); - sqlite3OpenTableAndIndices(pParse, pTab, 1, OP_OpenRead); - sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */ - loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0); - sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */ + sqlite3ExprCacheClear(pParse); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, + 1, 0, &iDataCur, &iIdxCur); + sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp2; - int r1; - static const VdbeOpList idxErr[] = { - { OP_AddImm, 1, -1, 0}, - { OP_String8, 0, 3, 0}, /* 1 */ - { OP_Rowid, 1, 4, 0}, - { OP_String8, 0, 5, 0}, /* 3 */ - { OP_String8, 0, 6, 0}, /* 4 */ - { OP_Concat, 4, 3, 3}, - { OP_Concat, 5, 3, 3}, - { OP_Concat, 6, 3, 3}, - { OP_ResultRow, 3, 1, 0}, - { OP_IfPos, 1, 0, 0}, /* 9 */ - { OP_Halt, 0, 0, 0}, - }; - r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0); - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1); - addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr); - sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC); - sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC); - sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_TRANSIENT); - sqlite3VdbeJumpHere(v, addr+9); + sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ + } + pParse->nMem = MAX(pParse->nMem, 8+j); + sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); + loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + /* Verify that all NOT NULL columns really are NOT NULL */ + for(j=0; jnCol; j++){ + char *zErr; + int jmp2, jmp3; + if( j==pTab->iPKey ) continue; + if( pTab->aCol[j].notNull==0 ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pTab->aCol[j].zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); } - sqlite3VdbeAddOp2(v, OP_Next, 1, loopTop+1); - sqlite3VdbeJumpHere(v, loopTop); + /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - static const VdbeOpList cntIdx[] = { - { OP_Integer, 0, 3, 0}, - { OP_Rewind, 0, 0, 0}, /* 1 */ - { OP_AddImm, 3, 1, 0}, - { OP_Next, 0, 0, 0}, /* 3 */ - { OP_Eq, 2, 0, 3}, /* 4 */ - { OP_AddImm, 1, -1, 0}, - { OP_String8, 0, 2, 0}, /* 6 */ - { OP_String8, 0, 3, 0}, /* 7 */ - { OP_Concat, 3, 2, 2}, - { OP_ResultRow, 2, 1, 0}, - }; - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); - addr = sqlite3VdbeAddOpList(v, ArraySize(cntIdx), cntIdx); - sqlite3VdbeChangeP1(v, addr+1, j+2); - sqlite3VdbeChangeP2(v, addr+1, addr+4); - sqlite3VdbeChangeP1(v, addr+3, j+2); - sqlite3VdbeChangeP2(v, addr+3, addr+2); - sqlite3VdbeJumpHere(v, addr+4); - sqlite3VdbeChangeP4(v, addr+6, - "wrong # of entries in index ", P4_STATIC); - sqlite3VdbeChangeP4(v, addr+7, pIdx->zName, P4_TRANSIENT); + int jmp2, jmp3, jmp4, jmp5; + int ckUniq = sqlite3VdbeMakeLabel(v); + if( pPk==pIdx ) continue; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, + pPrior, r1); + pPrior = pIdx; + sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ + /* Verify that an index entry exists for the current table row */ + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + pIdx->nColumn); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + " missing from index ", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pIdx->zName, P4_TRANSIENT); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + sqlite3VdbeJumpHere(v, jmp2); + /* For UNIQUE indexes, verify that only one entry exists with the + ** current key. The entry is unique if (1) any column is NULL + ** or (2) the next entry has a different key */ + if( IsUniqueIndex(pIdx) ){ + int uniqOk = sqlite3VdbeMakeLabel(v); + int jmp6; + int kk; + for(kk=0; kknKeyCol; kk++){ + int iCol = pIdx->aiColumn[kk]; + assert( iCol>=0 && iColnCol ); + if( pTab->aCol[iCol].notNull ) continue; + sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + VdbeCoverage(v); + } + jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeJumpHere(v, jmp6); + sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, + pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + "non-unique entry in index ", P4_STATIC); + sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeResolveLabel(v, uniqOk); + } + sqlite3VdbeJumpHere(v, jmp4); + sqlite3ResolvePartIdxLabel(pParse, jmp3); } + sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, loopTop-1); +#ifndef SQLITE_OMIT_BTREECOUNT + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, + "wrong # of entries in index ", P4_STATIC); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( pPk==pIdx ) continue; + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); + sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); + sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); + sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + } +#endif /* SQLITE_OMIT_BTREECOUNT */ } } - addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode); - sqlite3VdbeChangeP2(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr+1); - sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); - }else + addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + sqlite3VdbeChangeP3(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); + } + break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ #ifndef SQLITE_OMIT_UTF16 @@ -89322,7 +102830,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** new database files created using this database handle. It is only ** useful if invoked immediately after the main database i */ - if( sqlite3StrICmp(zLeft, "encoding")==0 ){ + case PragTyp_ENCODING: { static const struct EncName { char *zName; u8 enc; @@ -89369,7 +102877,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } } } - }else + } + break; #endif /* SQLITE_OMIT_UTF16 */ #ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS @@ -89380,6 +102889,11 @@ SQLITE_PRIVATE void sqlite3Pragma( ** PRAGMA [database.]user_version ** PRAGMA [database.]user_version = ** + ** PRAGMA [database.]freelist_count = + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = + ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both ** the schema-version and the user-version are 32-bit signed integers @@ -89398,13 +102912,13 @@ SQLITE_PRIVATE void sqlite3Pragma( ** The user-version is not used internally by SQLite. It may be used by ** applications for any purpose. */ - if( sqlite3StrICmp(zLeft, "schema_version")==0 - || sqlite3StrICmp(zLeft, "user_version")==0 - || sqlite3StrICmp(zLeft, "freelist_count")==0 - ){ + case PragTyp_HEADER_VALUE: { int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; case 'f': case 'F': iCookie = BTREE_FREE_PAGE_COUNT; break; @@ -89423,7 +102937,7 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_Integer, 0, 1, 0}, /* 1 */ { OP_SetCookie, 0, 0, 1}, /* 2 */ }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie); + int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); sqlite3VdbeChangeP1(v, addr+2, iDb); @@ -89435,14 +102949,15 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, 0}, /* 1 */ { OP_ResultRow, 1, 1, 0} }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie); + int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+1, iDb); sqlite3VdbeChangeP3(v, addr+1, iCookie); sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); } - }else + } + break; #endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS @@ -89452,7 +102967,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** Return the names of all compile-time options used in this build, ** one option per row. */ - if( sqlite3StrICmp(zLeft, "compile_options")==0 ){ + case PragTyp_COMPILE_OPTIONS: { int i = 0; const char *zOpt; sqlite3VdbeSetNumCols(v, 1); @@ -89462,7 +102977,8 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } - }else + } + break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ #ifndef SQLITE_OMIT_WAL @@ -89471,7 +102987,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** Checkpoint the database. */ - if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){ + case PragTyp_WAL_CHECKPOINT: { int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ @@ -89481,7 +102997,6 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_RESTART; } } - if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); @@ -89490,7 +103005,8 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); - }else + } + break; /* ** PRAGMA wal_autocheckpoint @@ -89500,21 +103016,87 @@ SQLITE_PRIVATE void sqlite3Pragma( ** after accumulating N frames in the log. Or query for the current value ** of N. */ - if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){ + case PragTyp_WAL_AUTOCHECKPOINT: { if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } returnSingleInt(pParse, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); - }else + } + break; #endif + /* + ** PRAGMA shrink_memory + ** + ** This pragma attempts to free as much memory as possible from the + ** current database connection. + */ + case PragTyp_SHRINK_MEMORY: { + sqlite3_db_release_memory(db); + break; + } + + /* + ** PRAGMA busy_timeout + ** PRAGMA busy_timeout = N + ** + ** Call sqlite3_busy_timeout(db, N). Return the current timeout value + ** if one is set. If no busy handler or a different busy handler is set + ** then 0 is returned. Setting the busy_timeout to 0 or negative + ** disables the timeout. + */ + /*case PragTyp_BUSY_TIMEOUT*/ default: { + assert( aPragmaNames[mid].ePragTyp==PragTyp_BUSY_TIMEOUT ); + if( zRight ){ + sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + } + returnSingleInt(pParse, "timeout", db->busyTimeout); + break; + } + + /* + ** PRAGMA soft_heap_limit + ** PRAGMA soft_heap_limit = N + ** + ** Call sqlite3_soft_heap_limit64(N). Return the result. If N is omitted, + ** use -1. + */ + case PragTyp_SOFT_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_soft_heap_limit64(N); + } + returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + break; + } + + /* + ** PRAGMA threads + ** PRAGMA threads = N + ** + ** Configure the maximum number of worker threads. Return the new + ** maximum, which might be less than requested. + */ + case PragTyp_THREADS: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); + } + returnSingleInt(pParse, "threads", + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); + break; + } + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases */ - if( sqlite3StrICmp(zLeft, "lock_status")==0 ){ + case PragTyp_LOCK_STATUS: { static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; @@ -89525,13 +103107,12 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; - Pager *pPager; const char *zState = "unknown"; int j; if( db->aDb[i].zName==0 ) continue; sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; - if( pBt==0 || (pPager = sqlite3BtreePager(pBt))==0 ){ + if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ @@ -89540,35 +103121,39 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } - - }else + break; + } #endif #ifdef SQLITE_HAS_CODEC - if( sqlite3StrICmp(zLeft, "key")==0 && zRight ){ - sqlite3_key(db, zRight, sqlite3Strlen30(zRight)); - }else - if( sqlite3StrICmp(zLeft, "rekey")==0 && zRight ){ - sqlite3_rekey(db, zRight, sqlite3Strlen30(zRight)); - }else - if( zRight && (sqlite3StrICmp(zLeft, "hexkey")==0 || - sqlite3StrICmp(zLeft, "hexrekey")==0) ){ - int i, h1, h2; - char zKey[40]; - for(i=0; (h1 = zRight[i])!=0 && (h2 = zRight[i+1])!=0; i+=2){ - h1 += 9*(1&(h1>>6)); - h2 += 9*(1&(h2>>6)); - zKey[i/2] = (h2 & 0x0f) | ((h1 & 0xf)<<4); + case PragTyp_KEY: { + if( zRight ) sqlite3_key_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); + break; + } + case PragTyp_REKEY: { + if( zRight ) sqlite3_rekey_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); + break; + } + case PragTyp_HEXKEY: { + if( zRight ){ + u8 iByte; + int i; + char zKey[40]; + for(i=0, iByte=0; iautoCommit ){ - sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level, - (db->flags&SQLITE_FullFSync)!=0, - (db->flags&SQLITE_CkptFullFSync)!=0); } + break; #endif + + } /* End of the PRAGMA switch */ + pragma_out: sqlite3DbFree(db, zLeft); sqlite3DbFree(db, zRight); @@ -89740,7 +103314,9 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ int rc; int i; +#ifndef SQLITE_OMIT_DEPRECATED int size; +#endif Table *pTab; Db *pDb; char const *azArg[4]; @@ -89783,7 +103359,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ /* zMasterSchema and zInitScript are set to point at the master schema ** and initialisation script appropriate for the database being - ** initialised. zMasterName is the name of the master table. + ** initialized. zMasterName is the name of the master table. */ if( !OMIT_TEMPDB && iDb==1 ){ zMasterSchema = temp_master_schema; @@ -89863,12 +103439,15 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ */ if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ if( iDb==0 ){ +#ifndef SQLITE_OMIT_UTF16 u8 encoding; /* If opening the main database, set ENC(db). */ encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; if( encoding==0 ) encoding = SQLITE_UTF8; ENC(db) = encoding; - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); +#else + ENC(db) = SQLITE_UTF8; +#endif }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ @@ -89884,9 +103463,13 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ pDb->pSchema->enc = ENC(db); if( pDb->pSchema->cache_size==0 ){ +#ifndef SQLITE_OMIT_DEPRECATED size = sqlite3AbsInt32(meta[BTREE_DEFAULT_CACHE_SIZE-1]); if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } pDb->pSchema->cache_size = size; +#else + pDb->pSchema->cache_size = SQLITE_DEFAULT_CACHE_SIZE; +#endif sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } @@ -89925,7 +103508,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ db->aDb[iDb].zName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + sqlite3_xauth xAuth; xAuth = db->xAuth; db->xAuth = 0; #endif @@ -89944,7 +103527,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ } if( db->mallocFailed ){ rc = SQLITE_NOMEM; - sqlite3ResetInternalSchema(db, -1); + sqlite3ResetAllSchemasOfConnection(db); } if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ /* Black magic: If the SQLITE_RecoveryMode flag is set, then consider @@ -89991,26 +103574,27 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int commit_internal = !(db->flags&SQLITE_InternChanges); assert( sqlite3_mutex_held(db->mutex) ); + assert( db->init.busy==0 ); rc = SQLITE_OK; db->init.busy = 1; for(i=0; rc==SQLITE_OK && inDb; i++){ if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; rc = sqlite3InitOne(db, i, pzErrMsg); if( rc ){ - sqlite3ResetInternalSchema(db, i); + sqlite3ResetOneSchema(db, i); } } - /* Once all the other databases have been initialised, load the schema + /* Once all the other databases have been initialized, load the schema ** for the TEMP database. This is loaded last, as the TEMP database ** schema may contain references to objects in other databases. */ #ifndef SQLITE_OMIT_TEMPDB - if( rc==SQLITE_OK && ALWAYS(db->nDb>1) - && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ + assert( db->nDb>1 ); + if( rc==SQLITE_OK && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ rc = sqlite3InitOne(db, 1, pzErrMsg); if( rc ){ - sqlite3ResetInternalSchema(db, 1); + sqlite3ResetOneSchema(db, 1); } } #endif @@ -90024,7 +103608,7 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } /* -** This routine is a no-op if the database schema is already initialised. +** This routine is a no-op if the database schema is already initialized. ** Otherwise, the schema is loaded. An error code is returned. */ SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse){ @@ -90078,7 +103662,7 @@ static void schemaIsValid(Parse *pParse){ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ - sqlite3ResetInternalSchema(db, iDb); + sqlite3ResetOneSchema(db, iDb); pParse->rc = SQLITE_SCHEMA; } @@ -90121,6 +103705,17 @@ SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ return i; } +/* +** Free all memory allocations in the pParse object +*/ +SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ + if( pParse ){ + sqlite3 *db = pParse->db; + sqlite3DbFree(db, pParse->aLabel); + sqlite3ExprListDelete(db, pParse->pConstExpr); + } +} + /* ** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ @@ -90179,7 +103774,7 @@ static int sqlite3Prepare( rc = sqlite3BtreeSchemaLocked(pBt); if( rc ){ const char *zDb = db->aDb[i].zName; - sqlite3Error(db, rc, "database schema is locked: %s", zDb); + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; } @@ -90189,14 +103784,14 @@ static int sqlite3Prepare( sqlite3VtabUnlockList(db); pParse->db = db; - pParse->nQueryLoop = (double)1; + pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; testcase( nBytes==mxLen ); testcase( nBytes==mxLen+1 ); if( nBytes>mxLen ){ - sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); + sqlite3ErrorWithMsg(db, SQLITE_TOOBIG, "statement too long"); rc = sqlite3ApiExit(db, SQLITE_TOOBIG); goto end_prepare; } @@ -90211,7 +103806,7 @@ static int sqlite3Prepare( }else{ sqlite3RunParser(pParse, zSql, &zErrMsg); } - assert( 1==(int)pParse->nQueryLoop ); + assert( 0==pParse->nQueryLoop ); if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM; @@ -90251,7 +103846,6 @@ static int sqlite3Prepare( } #endif - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); @@ -90264,10 +103858,10 @@ static int sqlite3Prepare( } if( zErrMsg ){ - sqlite3Error(db, rc, "%s", zErrMsg); + sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); sqlite3DbFree(db, zErrMsg); }else{ - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } /* Delete any TriggerPrg structures allocated while parsing this statement. */ @@ -90279,6 +103873,7 @@ static int sqlite3Prepare( end_prepare: + sqlite3ParserReset(pParse); sqlite3StackFree(db, pParse); rc = sqlite3ApiExit(db, rc); assert( (rc&db->errMask)==rc ); @@ -90308,6 +103903,7 @@ static int sqlite3LockAndPrepare( } sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); + assert( rc==SQLITE_OK || *ppStmt==0 ); return rc; } @@ -90407,6 +104003,12 @@ static int sqlite3Prepare16( if( !sqlite3SafetyCheckOk(db) ){ return SQLITE_MISUSE_BKPT; } + if( nBytes>=0 ){ + int sz; + const char *z = (const char*)zSql; + for(sz=0; szmutex); zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); if( zSql8 ){ @@ -90480,6 +104082,48 @@ SQLITE_API int sqlite3_prepare16_v2( ** to handle SELECT statements in SQLite. */ +/* +** Trace output macros +*/ +#if SELECTTRACE_ENABLED +/***/ int sqlite3SelectTrace = 0; +# define SELECTTRACE(K,P,S,X) \ + if(sqlite3SelectTrace&(K)) \ + sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",(S)->zSelName,(S)),\ + sqlite3DebugPrintf X +#else +# define SELECTTRACE(K,P,S,X) +#endif + + +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* +** An instance of the following object is used to record information about +** the ORDER BY (or GROUP BY) clause of query is being coded. +*/ +typedef struct SortCtx SortCtx; +struct SortCtx { + ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ + int nOBSat; /* Number of ORDER BY terms satisfied by indices */ + int iECursor; /* Cursor number for the sorter */ + int regReturn; /* Register holding block-output return address */ + int labelBkOut; /* Start label for the block-output subroutine */ + int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +}; +#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure but do not deallocate @@ -90495,6 +104139,7 @@ static void clearSelect(sqlite3 *db, Select *p){ sqlite3SelectDelete(db, p->pPrior); sqlite3ExprDelete(db, p->pLimit); sqlite3ExprDelete(db, p->pOffset); + sqlite3WithDelete(db, p->pWith); } /* @@ -90502,10 +104147,10 @@ static void clearSelect(sqlite3 *db, Select *p){ */ SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ pDest->eDest = (u8)eDest; - pDest->iParm = iParm; - pDest->affinity = 0; - pDest->iMem = 0; - pDest->nMem = 0; + pDest->iSDParm = iParm; + pDest->affSdst = 0; + pDest->iSdst = 0; + pDest->nSdst = 0; } @@ -90521,7 +104166,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - int isDistinct, /* true if the DISTINCT keyword is present */ + u16 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ @@ -90531,6 +104176,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ if( pNew==0 ){ + assert( db->mallocFailed ); pNew = &standin; memset(pNew, 0, sizeof(*pNew)); } @@ -90538,27 +104184,42 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); } pNew->pEList = pEList; + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->selFlags = isDistinct ? SF_Distinct : 0; + pNew->selFlags = selFlags; pNew->op = TK_SELECT; pNew->pLimit = pLimit; pNew->pOffset = pOffset; assert( pOffset==0 || pLimit!=0 ); pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; if( db->mallocFailed ) { clearSelect(db, pNew); if( pNew!=&standin ) sqlite3DbFree(db, pNew); pNew = 0; + }else{ + assert( pNew->pSrc!=0 || pParse->nErr>0 ); } + assert( pNew!=&standin ); return pNew; } +#if SELECTTRACE_ENABLED +/* +** Set the name of a Select object +*/ +SQLITE_PRIVATE void sqlite3SelectSetName(Select *p, const char *zName){ + if( p && zName ){ + sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName); + } +} +#endif + + /* ** Delete the given Select structure and all of its substructures. */ @@ -90570,7 +104231,15 @@ SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ } /* -** Given 1 to 3 identifiers preceeding the JOIN keyword, determine the +** Return a pointer to the right-most SELECT statement in a compound. +*/ +static Select *findRightmost(Select *p){ + while( p->pNext ) p = p->pNext; + return p; +} + +/* +** Given 1 to 3 identifiers preceding the JOIN keyword, determine the ** type of join. Return an integer constant that expresses that type ** in terms of the following bit values: ** @@ -90725,8 +104394,8 @@ static void addWhereTerm( pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2, 0); if( pEq && isOuterJoin ){ ExprSetProperty(pEq, EP_FromJoin); - assert( !ExprHasAnyProperty(pEq, EP_TokenOnly|EP_Reduced) ); - ExprSetIrreducible(pEq); + assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pEq, EP_NoReduce); pEq->iRightJoinTable = (i16)pE2->iTable; } *ppWhere = sqlite3ExprAnd(db, *ppWhere, pEq); @@ -90761,8 +104430,8 @@ static void addWhereTerm( static void setJoinExpr(Expr *p, int iTable){ while( p ){ ExprSetProperty(p, EP_FromJoin); - assert( !ExprHasAnyProperty(p, EP_TokenOnly|EP_Reduced) ); - ExprSetIrreducible(p); + assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(p, EP_NoReduce); p->iRightJoinTable = (i16)iTable; setJoinExpr(p->pLeft, iTable); p = p->pRight; @@ -90871,28 +104540,93 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ return 0; } +/* Forward reference */ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +); + /* -** Insert code into "v" that will push the record on the top of the -** stack into the sorter. +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ - ExprList *pOrderBy, /* The ORDER BY clause */ + SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ - int regData /* Register holding data to be sorted */ + int regData, /* First register holding data to be sorted */ + int nData, /* Number of elements in the data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ ){ - Vdbe *v = pParse->pVdbe; - int nExpr = pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); - sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pOrderBy->iECursor, regRecord); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ + int regBase; /* Regs for sorter record */ + int regRecord = ++pParse->nMem; /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + + assert( bSeq==0 || bSeq==1 ); + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nExpr - bSeq; + }else{ + regBase = pParse->nMem + 1; + pParse->nMem += nBase; + } + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); + } + if( nPrefixReg==0 ){ + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); + } + + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); + if( nOBSat>0 ){ + int regPrevKey; /* The first nOBSat columns of the previous row */ + int addrFirst; /* Address of the OP_IfNot opcode */ + int addrJmp; /* Address of the OP_Jump opcode */ + VdbeOp *pOp; /* Opcode that opens the sorter */ + int nKey; /* Number of sorting key columns, including OP_Sequence */ + KeyInfo *pKI; /* Original KeyInfo on the sorter table */ + + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); + pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + if( pParse->db->mallocFailed ) return; + pOp->p2 = nKey + nData; + pKI = pOp->p4.pKeyInfo; + memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ + sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); + pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); + addrJmp = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(v); + pSort->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + sqlite3VdbeJumpHere(v, addrFirst); + sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); + sqlite3VdbeJumpHere(v, addrJmp); + } + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + op = OP_SorterInsert; + }else{ + op = OP_IdxInsert; + } + sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( pSelect->iLimit ){ int addr1, addr2; int iLimit; @@ -90901,12 +104635,12 @@ static void pushOntoSorter( }else{ iLimit = pSelect->iLimit; } - addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); + addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); addr2 = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); - sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); + sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr2); } } @@ -90916,13 +104650,12 @@ static void pushOntoSorter( */ static void codeOffset( Vdbe *v, /* Generate code into this VM */ - Select *p, /* The SELECT statement being coded */ + int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( p->iOffset && iContinue!=0 ){ + if( iOffset>0 ){ int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); + addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); @@ -90950,7 +104683,7 @@ static void codeDistinct( v = pParse->pVdbe; r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -90984,19 +104717,18 @@ static int checkForMultiColumnSelectError( ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** -** If srcTab and nColumn are both zero, then the pEList expressions -** are evaluated in order to get the data for this row. If nColumn>0 -** then data is pulled from srcTab and pEList is used only to get the -** datatypes for each column. +** If srcTab is negative, then the pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and pEList is used only +** to get number columns and the datatype for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - int nColumn, /* Number of columns in the source table */ - ExprList *pOrderBy, /* If not NULL, sort results using this key */ - int distinct, /* If >=0, make sure results are distinct */ + SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ + DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ int iBreak /* Jump here to break out of the inner loop */ @@ -91006,55 +104738,112 @@ static void selectInnerLoop( int hasDistinct; /* True if the DISTINCT keyword is present */ int regResult; /* Start of memory holding result set */ int eDest = pDest->eDest; /* How to dispose of results */ - int iParm = pDest->iParm; /* First argument to disposal method */ + int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ assert( v ); - if( NEVER(v==0) ) return; assert( pEList!=0 ); - hasDistinct = distinct>=0; - if( pOrderBy==0 && !hasDistinct ){ - codeOffset(v, p, iContinue); + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; + if( pSort && pSort->pOrderBy==0 ) pSort = 0; + if( pSort==0 && !hasDistinct ){ + assert( iContinue!=0 ); + codeOffset(v, p->iOffset, iContinue); } /* Pull the requested columns. */ - if( nColumn>0 ){ - nResultCol = nColumn; - }else{ - nResultCol = pEList->nExpr; - } - if( pDest->iMem==0 ){ - pDest->iMem = pParse->nMem+1; - pDest->nMem = nResultCol; + nResultCol = pEList->nExpr; + + if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; + pParse->nMem += nPrefixReg; + } + pDest->iSdst = pParse->nMem+1; + pParse->nMem += nResultCol; + }else if( pDest->iSdst+nResultCol > pParse->nMem ){ + /* This is an error condition that can result, for example, when a SELECT + ** on the right-hand side of an INSERT contains more result columns than + ** there are columns in the table on the left. The error will be caught + ** and reported later. But we need to make sure enough memory is allocated + ** to avoid other spurious errors in the meantime. */ pParse->nMem += nResultCol; - }else{ - assert( pDest->nMem==nResultCol ); } - regResult = pDest->iMem; - if( nColumn>0 ){ - for(i=0; inSdst = nResultCol; + regResult = pDest->iSdst; + if( srcTab>=0 ){ + for(i=0; ia[i].zName)); } }else if( eDest!=SRT_Exists ){ /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pEList, regResult, eDest==SRT_Output); + sqlite3ExprCodeExprList(pParse, pEList, regResult, + (eDest==SRT_Output||eDest==SRT_Coroutine)?SQLITE_ECEL_DUP:0); } - nColumn = nResultCol; /* If the DISTINCT keyword was present on the SELECT statement ** and this row has been seen before, then do not make this row ** part of the result. */ if( hasDistinct ){ - assert( pEList!=0 ); - assert( pEList->nExpr==nColumn ); - codeDistinct(pParse, distinct, iContinue, nColumn, regResult); - if( pOrderBy==0 ){ - codeOffset(v, p, iContinue); + switch( pDistinct->eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + regPrev = pParse->nMem+1; + pParse->nMem += nResultCol; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Null + ** sets the MEM_Cleared bit on the first register of the + ** previous value. This will cause the OP_Ne below to always + ** fail on the first iteration of the loop even if the first + ** row is all NULLs. + */ + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = regPrev; + + iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; + for(i=0; ia[i].pExpr); + if( idb->mallocFailed ); + sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1); + break; + } + + case WHERE_DISTINCT_UNIQUE: { + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + break; + } + + default: { + assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); + break; + } + } + if( pSort==0 ){ + codeOffset(v, p->iOffset, iContinue); } } @@ -91066,7 +104855,7 @@ static void selectInnerLoop( case SRT_Union: { int r1; r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; @@ -91077,21 +104866,36 @@ static void selectInnerLoop( ** the temporary table iParm. */ case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); break; } -#endif +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* Store the result as data using a unique key. */ + case SRT_Fifo: + case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open + ** on an ephemeral index. If the current row is already present + ** in the index, do not write it to the output. If not, add the + ** current row to the index and proceed with writing it to the + ** output table as well. */ + int addr = sqlite3VdbeCurrentAddr(v) + 4; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + assert( pSort==0 ); + } +#endif + if( pSort ){ + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -91099,7 +104903,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); } - sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); break; } @@ -91109,17 +104913,18 @@ static void selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { - assert( nColumn==1 ); - p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affinity); - if( pOrderBy ){ + assert( nResultCol==1 ); + pDest->affSdst = + sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); + if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pOrderBy, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, 1, r1, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regResult, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -91140,39 +104945,81 @@ static void selectInnerLoop( ** of the scan loop. */ case SRT_Mem: { - assert( nColumn==1 ); - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, regResult); + assert( nResultCol==1 ); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ - sqlite3ExprCodeMove(pParse, regResult, iParm, 1); + assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ } break; } #endif /* #ifndef SQLITE_OMIT_SUBQUERY */ - /* Send the data to the callback function or to a subroutine. In the - ** case of a subroutine, the subroutine itself is responsible for - ** popping the data from the stack. - */ - case SRT_Coroutine: - case SRT_Output: { + case SRT_Coroutine: /* Send data to a co-routine */ + case SRT_Output: { /* Return the results */ testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); - if( pOrderBy ){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); - pushOntoSorter(pParse, pOrderBy, p, r1); - sqlite3ReleaseTempReg(pParse, r1); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ - sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); } break; } +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + regResult, nResultCol); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + + #if !defined(SQLITE_OMIT_TRIGGER) /* Discard the results. This is used for SELECT statements inside ** the body of a TRIGGER. The purpose of such selects is to call @@ -91190,11 +105037,63 @@ static void selectInnerLoop( ** there is a sorter, in which case the sorter has already limited ** the output for us. */ - if( pOrderBy==0 && p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); + if( pSort==0 && p->iLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); } } +/* +** Allocate a KeyInfo object sufficient for an index of N key columns and +** X extra columns. +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ + KeyInfo *p = sqlite3DbMallocZero(0, + sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1)); + if( p ){ + p->aSortOrder = (u8*)&p->aColl[N+X]; + p->nField = (u16)N; + p->nXField = (u16)X; + p->enc = ENC(db); + p->db = db; + p->nRef = 1; + }else{ + db->mallocFailed = 1; + } + return p; +} + +/* +** Deallocate a KeyInfo object +*/ +SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef--; + if( p->nRef==0 ) sqlite3DbFree(0, p); + } +} + +/* +** Make a new pointer to a KeyInfo object +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef++; + } + return p; +} + +#ifdef SQLITE_DEBUG +/* +** Return TRUE if a KeyInfo object can be change. The KeyInfo object +** can only be changed if this is just a single reference to the object. +** +** This routine is used only inside of assert() statements. +*/ +SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } +#endif /* SQLITE_DEBUG */ + /* ** Given an expression list, generate a KeyInfo structure that records ** the collating sequence for each expression in that expression list. @@ -91205,33 +105104,32 @@ static void selectInnerLoop( ** then the KeyInfo structure is appropriate for initializing a virtual ** index to implement a DISTINCT test. ** -** Space to hold the KeyInfo structure is obtain from malloc. The calling +** Space to hold the KeyInfo structure is obtained from malloc. The calling ** function is responsible for seeing that this structure is eventually -** freed. Add the KeyInfo structure to the P4 field of an opcode using -** P4_KEYINFO_HANDOFF is the usual way of dealing with this. +** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ - sqlite3 *db = pParse->db; +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; + sqlite3 *db = pParse->db; int i; nExpr = pList->nExpr; - pInfo = sqlite3DbMallocZero(db, sizeof(*pInfo) + nExpr*(sizeof(CollSeq*)+1) ); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); if( pInfo ){ - pInfo->aSortOrder = (u8*)&pInfo->aColl[nExpr]; - pInfo->nField = (u16)nExpr; - pInfo->enc = ENC(db); - pInfo->db = db; - for(i=0, pItem=pList->a; ia+iStart; ipExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } - pInfo->aColl[i] = pColl; - pInfo->aSortOrder[i] = pItem->sortOrder; + if( !pColl ) pColl = db->pDfltColl; + pInfo->aColl[i-iStart] = pColl; + pInfo->aSortOrder[i-iStart] = pItem->sortOrder; } } return pInfo; @@ -91333,35 +105231,69 @@ static void explainComposite( static void generateSortTail( Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ - Vdbe *v, /* Generate code into this VDBE */ + SortCtx *pSort, /* Information on the ORDER BY clause */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ + Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; + int addrOnce = 0; int iTab; - int pseudoTab = 0; - ExprList *pOrderBy = p->pOrderBy; - + ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; - int iParm = pDest->iParm; - + int iParm = pDest->iSDParm; int regRow; int regRowid; + int nKey; + int iSortTab; /* Sorter cursor to read from */ + int nSortData; /* Trailing values to read from sorter */ + int i; + int bSeq; /* True if sorter record includes seq. no. */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + struct ExprList_item *aOutEx = p->pEList->a; +#endif - iTab = pOrderBy->iECursor; - regRow = sqlite3GetTempReg(pParse); + if( pSort->labelBkOut ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeResolveLabel(v, pSort->labelBkOut); + } + iTab = pSort->iECursor; if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; + regRow = pDest->iSdst; + nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); + regRow = sqlite3GetTempReg(pParse); + nSortData = 1; + } + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + int regSortOut = ++pParse->nMem; + iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); + bSeq = 0; + }else{ + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + iSortTab = iTab; + bSeq = 1; + } + for(i=0; inExpr + 1, regRow); switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -91375,7 +105307,8 @@ static void generateSortTail( #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { assert( nColumn==1 ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, + &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regRow, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; @@ -91388,43 +105321,41 @@ static void generateSortTail( } #endif default: { - int i; assert( eDest==SRT_Output || eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); testcase( eDest==SRT_Coroutine ); - for(i=0; iiMem+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iMem+i); - if( i==0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); - } - } if( eDest==SRT_Output ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iMem, nColumn); - sqlite3ExprCacheAffinityChange(pParse, pDest->iMem, nColumn); + sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); + sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); }else{ - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); } break; } } - sqlite3ReleaseTempReg(pParse, regRow); - sqlite3ReleaseTempReg(pParse, regRowid); - + if( regRowid ){ + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); + } /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); - sqlite3VdbeResolveLabel(v, addrBreak); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); + sqlite3VdbeResolveLabel(v, addrBreak); } /* ** Return a pointer to a string containing the 'declaration type' of the ** expression pExpr. The string may be treated as static by the caller. ** +** Also try to estimate the size of the returned value and return that +** result in *pEstWidth. +** ** The declaration type is the exact datatype definition extracted from the ** original CREATE TABLE statement if the expression is a column. The ** declaration type for a ROWID field is INTEGER. Exactly when an expression @@ -91438,21 +105369,36 @@ static void generateSortTail( ** SELECT abc FROM (SELECT col AS abc FROM tbl); ** ** The declaration type for any expression other than a column is NULL. +** +** This routine has either 3 or 6 parameters depending on whether or not +** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used. */ -static const char *columnType( +#ifdef SQLITE_ENABLE_COLUMN_METADATA +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F) +static const char *columnTypeImpl( NameContext *pNC, Expr *pExpr, - const char **pzOriginDb, - const char **pzOriginTab, - const char **pzOriginCol + const char **pzOrigDb, + const char **pzOrigTab, + const char **pzOrigCol, + u8 *pEstWidth ){ + char const *zOrigDb = 0; + char const *zOrigTab = 0; + char const *zOrigCol = 0; +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) +static const char *columnTypeImpl( + NameContext *pNC, + Expr *pExpr, + u8 *pEstWidth +){ +#endif /* !defined(SQLITE_ENABLE_COLUMN_METADATA) */ char const *zType = 0; - char const *zOriginDb = 0; - char const *zOriginTab = 0; - char const *zOriginCol = 0; int j; - if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; + u8 estWidth = 1; + if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; switch( pExpr->op ){ case TK_AGG_COLUMN: case TK_COLUMN: { @@ -91513,25 +105459,35 @@ static const char *columnType( sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); + zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); } - }else if( ALWAYS(pTab->pSchema) ){ + }else if( pTab->pSchema ){ /* A real table */ assert( !pS ); if( iCol<0 ) iCol = pTab->iPKey; assert( iCol==-1 || (iCol>=0 && iColnCol) ); +#ifdef SQLITE_ENABLE_COLUMN_METADATA if( iCol<0 ){ zType = "INTEGER"; - zOriginCol = "rowid"; + zOrigCol = "rowid"; }else{ zType = pTab->aCol[iCol].zType; - zOriginCol = pTab->aCol[iCol].zName; + zOrigCol = pTab->aCol[iCol].zName; + estWidth = pTab->aCol[iCol].szEst; } - zOriginTab = pTab->zName; + zOrigTab = pTab->zName; if( pNC->pParse ){ int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); - zOriginDb = pNC->pParse->db->aDb[iDb].zName; + zOrigDb = pNC->pParse->db->aDb[iDb].zName; } +#else + if( iCol<0 ){ + zType = "INTEGER"; + }else{ + zType = pTab->aCol[iCol].zType; + estWidth = pTab->aCol[iCol].szEst; + } +#endif } break; } @@ -91548,18 +105504,21 @@ static const char *columnType( sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, &estWidth); break; } #endif } - - if( pzOriginDb ){ - assert( pzOriginTab && pzOriginCol ); - *pzOriginDb = zOriginDb; - *pzOriginTab = zOriginTab; - *pzOriginCol = zOriginCol; + +#ifdef SQLITE_ENABLE_COLUMN_METADATA + if( pzOrigDb ){ + assert( pzOrigTab && pzOrigCol ); + *pzOrigDb = zOrigDb; + *pzOrigTab = zOrigTab; + *pzOrigCol = zOrigCol; } +#endif + if( pEstWidth ) *pEstWidth = estWidth; return zType; } @@ -91585,7 +105544,7 @@ static void generateColumnTypes( const char *zOrigDb = 0; const char *zOrigTab = 0; const char *zOrigCol = 0; - zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, 0); /* The vdbe must make its own copy of the column-type and other ** column specific strings, in case the schema is reset before this @@ -91595,11 +105554,11 @@ static void generateColumnTypes( sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); #else - zType = columnType(&sNC, p, 0, 0, 0); + zType = columnType(&sNC, p, 0, 0, 0, 0); #endif sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } -#endif /* SQLITE_OMIT_DECLTYPE */ +#endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } /* @@ -91663,15 +105622,16 @@ static void generateColumnNames( sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); } }else{ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, - sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); + const char *z = pEList->a[i].zSpan; + z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); } } generateColumnTypes(pParse, pTabList, pEList); } /* -** Given a an expression list (which is really the list of expressions +** Given an expression list (which is really the list of expressions ** that form the result set of a SELECT statement) compute appropriate ** column names for a table that would hold the expression list. ** @@ -91686,7 +105646,7 @@ static void generateColumnNames( static int selectColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ - int *pnCol, /* Write the number of columns here */ + i16 *pnCol, /* Write the number of columns here */ Column **paCol /* Write the new column list here */ ){ sqlite3 *db = pParse->db; /* Database connection */ @@ -91698,22 +105658,31 @@ static int selectColumnsFromExprList( char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList->nExpr; - aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); - if( aCol==0 ) return SQLITE_NOMEM; + if( pEList ){ + nCol = pEList->nExpr; + aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + testcase( aCol==0 ); + }else{ + nCol = 0; + aCol = 0; + } + *pnCol = nCol; + *paCol = aCol; + for(i=0, pCol=aCol; ia[i].pExpr; - assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue) - || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 ); + p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS " phrase, use as the name */ zName = sqlite3DbStrDup(db, zName); }else{ Expr *pColExpr = p; /* The expression that is the result column name */ Table *pTab; /* Table associated with this expression */ - while( pColExpr->op==TK_DOT ) pColExpr = pColExpr->pRight; + while( pColExpr->op==TK_DOT ){ + pColExpr = pColExpr->pRight; + assert( pColExpr!=0 ); + } if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; @@ -91735,12 +105704,15 @@ static int selectColumnsFromExprList( } /* Make sure the column name is unique. If the name is not unique, - ** append a integer to the name so that it becomes unique. + ** append an integer to the name so that it becomes unique. */ nName = sqlite3Strlen30(zName); for(j=cnt=0; j1 && sqlite3Isdigit(zName[k]); k--){} + if( k>=0 && zName[k]==':' ) nName = k; zName[nName] = 0; zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); sqlite3DbFree(db, zName); @@ -91776,8 +105748,7 @@ static int selectColumnsFromExprList( */ static void selectAddColumnTypeAndCollation( Parse *pParse, /* Parsing contexts */ - int nCol, /* Number of columns */ - Column *aCol, /* List of columns */ + Table *pTab, /* Add column type information to this table */ Select *pSelect /* SELECT used to determine types and collations */ ){ sqlite3 *db = pParse->db; @@ -91787,17 +105758,19 @@ static void selectAddColumnTypeAndCollation( int i; Expr *p; struct ExprList_item *a; + u64 szAll = 0; assert( pSelect!=0 ); assert( (pSelect->selFlags & SF_Resolved)!=0 ); - assert( nCol==pSelect->pEList->nExpr || db->mallocFailed ); + assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); if( db->mallocFailed ) return; memset(&sNC, 0, sizeof(sNC)); sNC.pSrcList = pSelect->pSrc; a = pSelect->pEList->a; - for(i=0, pCol=aCol; iaCol; inCol; i++, pCol++){ p = a[i].pExpr; - pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p, 0, 0, 0)); + pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p,0,0,0, &pCol->szEst)); + szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; pColl = sqlite3ExprCollSeq(pParse, p); @@ -91805,6 +105778,7 @@ static void selectAddColumnTypeAndCollation( pCol->zColl = sqlite3DbStrDup(db, pColl->zName); } } + pTab->szTabRow = sqlite3LogEst(szAll*4); } /* @@ -91832,9 +105806,9 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1000000; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); + selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -91850,12 +105824,14 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ Vdbe *v = pParse->pVdbe; if( v==0 ){ - v = pParse->pVdbe = sqlite3VdbeCreate(pParse->db); -#ifndef SQLITE_OMIT_TRACE - if( v ){ - sqlite3VdbeAddOp0(v, OP_Trace); + v = pParse->pVdbe = sqlite3VdbeCreate(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Init); + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; } -#endif + } return v; } @@ -91872,8 +105848,13 @@ SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ ** ** This routine changes the values of iLimit and iOffset only if ** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values -** (usually but not always -1) prior to calling this routine. +** iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** ** Only if pLimit!=0 or pOffset!=0 do the limit registers get ** redefined. The UNION ALL operator uses this property to force ** the reuse of the same limit and offset registers across multiple @@ -91888,7 +105869,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ /* ** "LIMIT -1" always shows all rows. There is some - ** contraversy about what the correct behavior should be. + ** controversy about what the correct behavior should be. ** The current implementation interprets "LIMIT 0" to mean ** no rows. */ @@ -91897,33 +105878,33 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ if( p->pLimit ){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ + assert( v!=0 ); if( sqlite3ExprIsInteger(p->pLimit, &n) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); - }else{ - if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n; + }else if( n>=0 && p->nSelectRow>(u64)n ){ + p->nSelectRow = n; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); - sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); VdbeComment((v, "LIMIT counter")); - sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); + sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); VdbeCoverage(v); } if( p->pOffset ){ p->iOffset = iOffset = ++pParse->nMem; pParse->nMem++; /* Allocate an extra register for limit+offset */ sqlite3ExprCode(pParse, p->pOffset, iOffset); - sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); sqlite3VdbeJumpHere(v, addr1); } @@ -91952,9 +105933,210 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ } return pRet; } -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ -/* Forward reference */ +/* +** The select statement passed as the second parameter is a compound SELECT +** with an ORDER BY clause. This function allocates and returns a KeyInfo +** structure suitable for implementing the ORDER BY. +** +** Space to hold the KeyInfo structure is obtained from malloc. The calling +** function is responsible for ensuring that this structure is eventually +** freed. +*/ +static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ + ExprList *pOrderBy = p->pOrderBy; + int nOrderBy = p->pOrderBy->nExpr; + sqlite3 *db = pParse->db; + KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); + if( pRet ){ + int i; + for(i=0; ia[i]; + Expr *pTerm = pItem->pExpr; + CollSeq *pColl; + + if( pTerm->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pTerm); + }else{ + pColl = multiSelectCollSeq(pParse, p, pItem->u.x.iOrderByCol-1); + if( pColl==0 ) pColl = db->pDfltColl; + pOrderBy->a[i].pExpr = + sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName); + } + assert( sqlite3KeyInfoIsWriteable(pRet) ); + pRet->aColl[i] = pColl; + pRet->aSortOrder[i] = pOrderBy->a[i].sortOrder; + } + } + + return pRet; +} + +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from Queue is output to pDest. Then the single +** extracted row (now in the iCurrent table) becomes the content of the +** recursive-table for a recursive-query run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup = p->pPrior; /* The setup query */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent = 0; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Fifo; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + pOffset = p->pOffset; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = p->pOffset = 0; + p->iLimit = p->iOffset = 0; + pOrderBy = p->pOrderBy; + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo; + iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Fifo; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Detach the ORDER BY clause from the compound SELECT */ + p->pOrderBy = 0; + + /* Store the results of the setup-query in Queue. */ + pSetup->pNext = 0; + rc = sqlite3Select(pParse, pSetup, &destQueue); + pSetup->pNext = p; + if( rc ) goto end_of_recursive_query; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(v); + codeOffset(v, regOffset, addrCont); + selectInnerLoop(pParse, p, p->pEList, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + if( regLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + VdbeCoverage(v); + } + sqlite3VdbeResolveLabel(v, addrCont); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + sqlite3ExprListDelete(pParse->db, p->pOrderBy); + p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + p->pOffset = pOffset; + return; +} +#endif /* SQLITE_OMIT_CTE */ + +/* Forward references */ static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ @@ -91962,7 +106144,6 @@ static int multiSelectOrderBy( ); -#ifndef SQLITE_OMIT_COMPOUND_SELECT /* ** This routine is called to process a compound query form from ** two or more separate queries using UNION, UNION ALL, EXCEPT, or @@ -92006,18 +106187,17 @@ static int multiSelect( Select *pDelete = 0; /* Chain of simple selects to delete */ sqlite3 *db; /* Database connection */ #ifndef SQLITE_OMIT_EXPLAIN - int iSub1; /* EQP id of left-hand query */ - int iSub2; /* EQP id of right-hand query */ + int iSub1 = 0; /* EQP id of left-hand query */ + int iSub2 = 0; /* EQP id of right-hand query */ #endif /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ assert( p && p->pPrior ); /* Calling function guarantees this much */ + assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); db = pParse->db; pPrior = p->pPrior; - assert( pPrior->pRightmost!=pPrior ); - assert( pPrior->pRightmost==p->pRightmost ); dest = *pDest; if( pPrior->pOrderBy ){ sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", @@ -92039,7 +106219,7 @@ static int multiSelect( */ if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } @@ -92049,17 +106229,27 @@ static int multiSelect( */ assert( p->pEList && pPrior->pEList ); if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } rc = 1; goto multi_select_end; } +#ifndef SQLITE_OMIT_CTE + if( p->selFlags & SF_Recursive ){ + generateWithRecursiveQuery(pParse, p, &dest); + }else +#endif + /* Compound SELECTs that have an ORDER BY clause are handled separately. */ if( p->pOrderBy ){ return multiSelectOrderBy(pParse, p, pDest); - } + }else /* Generate code for the left and right SELECT statements. */ @@ -92068,6 +106258,8 @@ static int multiSelect( int addr = 0; int nLimit; assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; explainSetInteger(iSub1, pParse->iNextSelectId); @@ -92081,7 +106273,7 @@ static int multiSelect( p->iLimit = pPrior->iLimit; p->iOffset = pPrior->iOffset; if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); + addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); } explainSetInteger(iSub2, pParse->iNextSelectId); @@ -92092,9 +106284,9 @@ static int multiSelect( p->nSelectRow += pPrior->nSelectRow; if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && p->nSelectRow > (double)nLimit + && nLimit>0 && p->nSelectRow > (u64)nLimit ){ - p->nSelectRow = (double)nLimit; + p->nSelectRow = nLimit; } if( addr ){ sqlite3VdbeJumpHere(v, addr); @@ -92113,15 +106305,13 @@ static int multiSelect( testcase( p->op==TK_EXCEPT ); testcase( p->op==TK_UNION ); priorOp = SRT_Union; - if( dest.eDest==priorOp && ALWAYS(!p->pLimit &&!p->pOffset) ){ + if( dest.eDest==priorOp ){ /* We can reuse a temporary table generated by a SELECT to our ** right. */ - assert( p->pRightmost!=p ); /* Can only happen for leftward elements - ** of a 3-way or more compound */ assert( p->pLimit==0 ); /* Not allowed on leftward elements */ assert( p->pOffset==0 ); /* Not allowed on leftward elements */ - unionTab = dest.iParm; + unionTab = dest.iSDParm; }else{ /* We will need to create our own temporary table to hold the ** intermediate results. @@ -92131,7 +106321,7 @@ static int multiSelect( addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); assert( p->addrOpenEphm[0] == -1 ); p->addrOpenEphm[0] = addr; - p->pRightmost->selFlags |= SF_UsesEphemeral; + findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); } @@ -92178,7 +106368,7 @@ static int multiSelect( /* Convert the data in the temporary table into whatever form ** it is that we currently need. */ - assert( unionTab==dest.iParm || dest.eDest!=priorOp ); + assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); if( dest.eDest!=priorOp ){ int iCont, iBreak, iStart; assert( p->pEList ); @@ -92190,12 +106380,12 @@ static int multiSelect( iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + selectInnerLoop(pParse, p, p->pEList, unionTab, + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); } @@ -92220,7 +106410,7 @@ static int multiSelect( addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); assert( p->addrOpenEphm[0] == -1 ); p->addrOpenEphm[0] = addr; - p->pRightmost->selFlags |= SF_UsesEphemeral; + findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); /* Code the SELECTs to our left into temporary table "tab1". @@ -92242,7 +106432,7 @@ static int multiSelect( p->pLimit = 0; pOffset = p->pOffset; p->pOffset = 0; - intersectdest.iParm = tab2; + intersectdest.iSDParm = tab2; explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); @@ -92265,15 +106455,15 @@ static int multiSelect( iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); - sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + selectInnerLoop(pParse, p, p->pEList, tab1, + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); @@ -92299,18 +106489,13 @@ static int multiSelect( CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ int nCol; /* Number of columns in result set */ - assert( p->pRightmost==p ); + assert( p->pNext==0 ); nCol = p->pEList->nExpr; - pKeyInfo = sqlite3DbMallocZero(db, - sizeof(*pKeyInfo)+nCol*(sizeof(CollSeq*) + 1)); + pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); if( !pKeyInfo ){ rc = SQLITE_NOMEM; goto multi_select_end; } - - pKeyInfo->enc = ENC(db); - pKeyInfo->nField = (u16)nCol; - for(i=0, apColl=pKeyInfo->aColl; iaddrOpenEphm[i] = -1; } } - sqlite3DbFree(db, pKeyInfo); + sqlite3KeyInfoUnref(pKeyInfo); } multi_select_end: - pDest->iMem = dest.iMem; - pDest->nMem = dest.nMem; + pDest->iSdst = dest.iSdst; + pDest->nSdst = dest.nSdst; sqlite3SelectDelete(db, pDelete); return rc; } @@ -92347,8 +106533,8 @@ multi_select_end: ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. ** -** The data to be output is contained in pIn->iMem. There are -** pIn->nMem columns to be output. pDest is where the output should +** The data to be output is contained in pIn->iSdst. There are +** pIn->nSdst columns to be output. pDest is where the output should ** be sent. ** ** regReturn is the number of the register holding the subroutine @@ -92371,7 +106557,6 @@ static int generateOutputSubroutine( int regReturn, /* The return address register */ int regPrev, /* Previous result register. No uniqueness if 0 */ KeyInfo *pKeyInfo, /* For comparing with previous entry */ - int p4type, /* The p4 type for pKeyInfo */ int iBreak /* Jump here if we hit the LIMIT */ ){ Vdbe *v = pParse->pVdbe; @@ -92385,19 +106570,19 @@ static int generateOutputSubroutine( */ if( regPrev ){ int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iMem, regPrev+1, pIn->nMem, - (char*)pKeyInfo, p4type); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); + j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); sqlite3VdbeJumpHere(v, j1); - sqlite3ExprCodeCopy(pParse, pIn->iMem, regPrev+1, pIn->nMem); + sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } if( pParse->db->mallocFailed ) return 0; - /* Suppress the the first OFFSET entries if there is an OFFSET clause + /* Suppress the first OFFSET entries if there is an OFFSET clause */ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); switch( pDest->eDest ){ /* Store the result as data using a unique key. @@ -92408,9 +106593,9 @@ static int generateOutputSubroutine( int r2 = sqlite3GetTempReg(pParse); testcase( pDest->eDest==SRT_Table ); testcase( pDest->eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iMem, pIn->nMem, r1); - sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, pDest->iParm, r1, r2); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); sqlite3ReleaseTempReg(pParse, r1); @@ -92424,13 +106609,13 @@ static int generateOutputSubroutine( */ case SRT_Set: { int r1; - assert( pIn->nMem==1 ); - p->affinity = - sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affinity); + assert( pIn->nSdst==1 ); + pDest->affSdst = + sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iMem, 1, r1, &p->affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iParm, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; } @@ -92439,7 +106624,7 @@ static int generateOutputSubroutine( /* If any row exist in the result set, record that fact and abort. */ case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iParm); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); /* The LIMIT clause will terminate the loop for us */ break; } @@ -92450,23 +106635,23 @@ static int generateOutputSubroutine( ** of the scan loop. */ case SRT_Mem: { - assert( pIn->nMem==1 ); - sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iParm, 1); + assert( pIn->nSdst==1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); /* The LIMIT clause will jump out of the loop for us */ break; } #endif /* #ifndef SQLITE_OMIT_SUBQUERY */ /* The results are stored in a sequence of registers - ** starting at pDest->iMem. Then the co-routine yields. + ** starting at pDest->iSdst. Then the co-routine yields. */ case SRT_Coroutine: { - if( pDest->iMem==0 ){ - pDest->iMem = sqlite3GetTempRange(pParse, pIn->nMem); - pDest->nMem = pIn->nMem; + if( pDest->iSdst==0 ){ + pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst); + pDest->nSdst = pIn->nSdst; } - sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iMem, pDest->nMem); - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pDest->nSdst); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); break; } @@ -92480,8 +106665,8 @@ static int generateOutputSubroutine( */ default: { assert( pDest->eDest==SRT_Output ); - sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iMem, pIn->nMem); - sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, pIn->nMem); + sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); break; } } @@ -92489,7 +106674,7 @@ static int generateOutputSubroutine( /* Jump to the end of the loop if the LIMIT is reached. */ if( p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); } /* Generate the subroutine return @@ -92597,9 +106782,7 @@ static int multiSelectOrderBy( SelectDest destA; /* Destination for coroutine A */ SelectDest destB; /* Destination for coroutine B */ int regAddrA; /* Address register for select-A coroutine */ - int regEofA; /* Flag to indicate when select-A is complete */ int regAddrB; /* Address register for select-B coroutine */ - int regEofB; /* Flag to indicate when select-B is complete */ int addrSelectA; /* Address of the select-A coroutine */ int addrSelectB; /* Address of the select-B coroutine */ int regOutA; /* Address register for the output-A subroutine */ @@ -92607,6 +106790,7 @@ static int multiSelectOrderBy( int addrOutA; /* Address of the output-A subroutine */ int addrOutB = 0; /* Address of the output-B subroutine */ int addrEofA; /* Address of the select-A-exhausted subroutine */ + int addrEofA_noB; /* Alternate addrEofA if B is uninitialized */ int addrEofB; /* Address of the select-B-exhausted subroutine */ int addrAltB; /* Address of the AmallocFailed==0 && i<=p->pEList->nExpr; i++){ struct ExprList_item *pItem; for(j=0, pItem=pOrderBy->a; jiCol>0 ); - if( pItem->iCol==i ) break; + assert( pItem->u.x.iOrderByCol>0 ); + if( pItem->u.x.iOrderByCol==i ) break; } if( j==nOrderBy ){ Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); @@ -92666,7 +106850,7 @@ static int multiSelectOrderBy( pNew->flags |= EP_IntValue; pNew->u.iValue = i; pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); - pOrderBy->a[nOrderBy++].iCol = (u16)i; + if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; } } } @@ -92682,29 +106866,11 @@ static int multiSelectOrderBy( if( aPermute ){ struct ExprList_item *pItem; for(i=0, pItem=pOrderBy->a; iiCol>0 && pItem->iCol<=p->pEList->nExpr ); - aPermute[i] = pItem->iCol - 1; - } - pKeyMerge = - sqlite3DbMallocRaw(db, sizeof(*pKeyMerge)+nOrderBy*(sizeof(CollSeq*)+1)); - if( pKeyMerge ){ - pKeyMerge->aSortOrder = (u8*)&pKeyMerge->aColl[nOrderBy]; - pKeyMerge->nField = (u16)nOrderBy; - pKeyMerge->enc = ENC(db); - for(i=0; ia[i].pExpr; - if( pTerm->flags & EP_ExpCollate ){ - pColl = pTerm->pColl; - }else{ - pColl = multiSelectCollSeq(pParse, p, aPermute[i]); - pTerm->flags |= EP_ExpCollate; - pTerm->pColl = pColl; - } - pKeyMerge->aColl[i] = pColl; - pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder; - } + assert( pItem->u.x.iOrderByCol>0 + && pItem->u.x.iOrderByCol<=p->pEList->nExpr ); + aPermute[i] = pItem->u.x.iOrderByCol - 1; } + pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); }else{ pKeyMerge = 0; } @@ -92723,14 +106889,12 @@ static int multiSelectOrderBy( }else{ int nExpr = p->pEList->nExpr; assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite3GetTempRange(pParse, nExpr+1); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); - pKeyDup = sqlite3DbMallocZero(db, - sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); + pKeyDup = sqlite3KeyInfoAlloc(db, nExpr, 1); if( pKeyDup ){ - pKeyDup->aSortOrder = (u8*)&pKeyDup->aColl[nExpr]; - pKeyDup->nField = (u16)nExpr; - pKeyDup->enc = ENC(db); + assert( sqlite3KeyInfoIsWriteable(pKeyDup) ); for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); pKeyDup->aSortOrder[i] = 0; @@ -92741,6 +106905,7 @@ static int multiSelectOrderBy( /* Separate the left and the right query from one another */ p->pPrior = 0; + pPrior->pNext = 0; sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); if( pPrior->pPrior==0 ){ sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); @@ -92763,37 +106928,30 @@ static int multiSelectOrderBy( p->pOffset = 0; regAddrA = ++pParse->nMem; - regEofA = ++pParse->nMem; regAddrB = ++pParse->nMem; - regEofB = ++pParse->nMem; regOutA = ++pParse->nMem; regOutB = ++pParse->nMem; sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); - /* Jump past the various subroutines and coroutines to the main - ** merge loop - */ - j1 = sqlite3VdbeAddOp0(v, OP_Goto); - addrSelectA = sqlite3VdbeCurrentAddr(v); - - /* Generate a coroutine to evaluate the SELECT statement to the ** left of the compound operator - the "A" select. */ - VdbeNoopComment((v, "Begin coroutine for left SELECT")); + addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - VdbeNoopComment((v, "End coroutine for left SELECT")); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); + sqlite3VdbeJumpHere(v, j1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ - addrSelectB = sqlite3VdbeCurrentAddr(v); - VdbeNoopComment((v, "Begin coroutine for right SELECT")); + addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; @@ -92802,9 +106960,7 @@ static int multiSelectOrderBy( sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofB); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); - VdbeNoopComment((v, "End coroutine for right SELECT")); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB); /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. @@ -92812,7 +106968,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "Output routine for A")); addrOutA = generateOutputSubroutine(pParse, p, &destA, pDest, regOutA, - regPrev, pKeyDup, P4_KEYINFO_HANDOFF, labelEnd); + regPrev, pKeyDup, labelEnd); /* Generate a subroutine that outputs the current row of the B ** select as the next output row of the compound select. @@ -92821,19 +106977,20 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "Output routine for B")); addrOutB = generateOutputSubroutine(pParse, p, &destB, pDest, regOutB, - regPrev, pKeyDup, P4_KEYINFO_STATIC, labelEnd); + regPrev, pKeyDup, labelEnd); } + sqlite3KeyInfoUnref(pKeyDup); /* Generate a subroutine to run when the results from select A ** are exhausted and only data in select B remains. */ - VdbeNoopComment((v, "eof-A subroutine")); if( op==TK_EXCEPT || op==TK_INTERSECT ){ - addrEofA = sqlite3VdbeAddOp2(v, OP_Goto, 0, labelEnd); + addrEofA_noB = addrEofA = labelEnd; }else{ - addrEofA = sqlite3VdbeAddOp2(v, OP_If, regEofB, labelEnd); - sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + VdbeNoopComment((v, "eof-A subroutine")); + addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); p->nSelectRow += pPrior->nSelectRow; } @@ -92846,9 +107003,8 @@ static int multiSelectOrderBy( if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ VdbeNoopComment((v, "eof-B subroutine")); - addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd); - sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); + addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); } @@ -92856,8 +107012,7 @@ static int multiSelectOrderBy( */ VdbeNoopComment((v, "A-lt-B subroutine")); addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); /* Generate code to handle the case of A==B @@ -92870,8 +107025,7 @@ static int multiSelectOrderBy( }else{ VdbeNoopComment((v, "A-eq-B subroutine")); addrAeqB = - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); } @@ -92882,33 +107036,23 @@ static int multiSelectOrderBy( if( op==TK_ALL || op==TK_UNION ){ sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); - sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); /* This code runs once to initialize everything. */ sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofA); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofB); - sqlite3VdbeAddOp2(v, OP_Gosub, regAddrA, addrSelectA); - sqlite3VdbeAddOp2(v, OP_Gosub, regAddrB, addrSelectB); - sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); - sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); /* Implement the main merge loop */ sqlite3VdbeResolveLabel(v, labelCmpr); sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); - sqlite3VdbeAddOp4(v, OP_Compare, destA.iMem, destB.iMem, nOrderBy, - (char*)pKeyMerge, P4_KEYINFO_HANDOFF); - sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - - /* Release temporary registers - */ - if( regPrev ){ - sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } + sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, + (char*)pKeyMerge, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); /* Jump to the this point in order to terminate the query. */ @@ -92928,6 +107072,7 @@ static int multiSelectOrderBy( sqlite3SelectDelete(db, p->pPrior); } p->pPrior = pPrior; + pPrior->pNext = p; /*** TBD: Insert subroutine calls to close cursors on incomplete **** subqueries ****/ @@ -92969,9 +107114,6 @@ static Expr *substExpr( assert( pEList!=0 && pExpr->iColumnnExpr ); assert( pExpr->pLeft==0 && pExpr->pRight==0 ); pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); - if( pNew && pExpr->pColl ){ - pNew->pColl = pExpr->pColl; - } sqlite3ExprDelete(db, pExpr); pExpr = pNew; } @@ -93026,9 +107168,8 @@ static void substSelect( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* -** This routine attempts to flatten subqueries in order to speed -** execution. It returns 1 if it makes changes and 0 if no flattening -** occurs. +** This routine attempts to flatten subqueries as a performance optimization. +** This routine returns 1 if it makes changes and 0 if no flattening occurs. ** ** To understand the concept of flattening, consider the following ** query: @@ -93047,7 +107188,7 @@ static void substSelect( ** ** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 ** -** The code generated for this simpification gives the same result +** The code generated for this simplification gives the same result ** but only has to scan the data once. And because indices might ** exist on the table t1, a complete scan of the data might be ** avoided. @@ -93070,15 +107211,20 @@ static void substSelect( ** (6) The subquery does not use aggregates or the outer query is not ** DISTINCT. ** -** (7) The subquery has a FROM clause. +** (7) The subquery has a FROM clause. TODO: For subqueries without +** A FROM clause, consider adding a FROM close with the special +** table sqlite_once that consists of a single row containing a +** single NULL. ** ** (8) The subquery does not use LIMIT or the outer query is not a join. ** ** (9) The subquery does not use LIMIT or the outer query does not use ** aggregates. ** -** (10) The subquery does not use aggregates or the outer query does not -** use LIMIT. +** (**) Restriction (10) was removed from the code on 2005-02-05 but we +** accidently carried the comment forward until 2014-09-15. Original +** text: "The subquery does not use aggregates or the outer query does not +** use LIMIT." ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** @@ -93103,11 +107249,20 @@ static void substSelect( ** ** * is not itself part of a compound select, ** * is not an aggregate or DISTINCT query, and -** * has no other tables or sub-selects in the FROM clause. +** * is not a join ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, -** LIMIT and OFFSET clauses. +** LIMIT and OFFSET clauses. The subquery cannot use any compound +** operator other than UNION ALL because all the other compound +** operators have an implied DISTINCT which is disallowed by +** restriction (4). +** +** Also, each component of the sub-query must return the same number +** of result columns. This is actually a requirement for any compound +** SELECT statement, but all the code here does is make sure that no +** such (illegal) sub-query is flattened. The caller will detect the +** syntax error and return a detailed message. ** ** (18) If the sub-query is a compound select, then all terms of the ** ORDER by clause of the parent must be simple references to @@ -93119,12 +107274,25 @@ static void substSelect( ** (20) If the sub-query is a compound select, then it must not use ** an ORDER BY clause. Ticket #3773. We could relax this constraint ** somewhat by saying that the terms of the ORDER BY clause must -** appear as unmodified result columns in the outer query. But +** appear as unmodified result columns in the outer query. But we ** have other optimizations in mind to deal with that case. ** ** (21) The subquery does not use LIMIT or the outer query is not ** DISTINCT. (See ticket [752e1646fc]). ** +** (22) The subquery is not a recursive CTE. +** +** (23) The parent is not a recursive CTE, or the sub-query is not a +** compound query. This restriction is because transforming the +** parent to a compound query confuses the code that handles +** recursive queries in multiSelect(). +** +** (24) The subquery is not an aggregate that uses the built-in min() or +** or max() functions. (Without this restriction, a query like: +** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily +** return the value X for which Y was maximal.) +** +** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query ** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. @@ -93159,7 +107327,7 @@ static int flattenSubquery( */ assert( p!=0 ); assert( p->pPrior==0 ); /* Unable to flatten compound queries */ - if( db->flags & SQLITE_QueryFlattener ) return 0; + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; @@ -93171,13 +107339,13 @@ static int flattenSubquery( pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, - ** not arbitrary expresssions, we allowed some combining of LIMIT and OFFSET + ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET ** because they could be computed at compile-time. But when LIMIT and OFFSET ** became arbitrary expressions, we were forced to add restrictions (13) ** and (14). */ if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ if( pSub->pOffset ) return 0; /* Restriction (14) */ - if( p->pRightmost && pSub->pLimit ){ + if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){ return 0; /* Restriction (15) */ } if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ @@ -93196,6 +107364,14 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } + testcase( pSub->selFlags & SF_Recursive ); + testcase( pSub->selFlags & SF_MinMaxAgg ); + if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){ + return 0; /* Restrictions (22) and (24) */ + } + if( (p->selFlags & SF_Recursive) && pSub->pPrior ){ + return 0; /* Restriction (23) */ + } /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is @@ -93248,28 +107424,34 @@ static int flattenSubquery( for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + assert( pSub->pSrc!=0 ); if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 || (pSub1->pPrior && pSub1->op!=TK_ALL) - || NEVER(pSub1->pSrc==0) || pSub1->pSrc->nSrc!=1 + || pSub1->pSrc->nSrc<1 + || pSub->pEList->nExpr!=pSub1->pEList->nExpr ){ return 0; } + testcase( pSub1->pSrc->nSrc>1 ); } /* Restriction 18. */ if( p->pOrderBy ){ int ii; for(ii=0; iipOrderBy->nExpr; ii++){ - if( p->pOrderBy->a[ii].iCol==0 ) return 0; + if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; } } } /***** If we reach this point, flattening is permitted. *****/ + SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", + pSub->zSelName, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; - sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + testcase( i==SQLITE_DENY ); pParse->zAuthContext = zSavedAuthContext; /* If the sub-query is a compound SELECT statement, then (by restrictions @@ -93309,24 +107491,31 @@ static int flattenSubquery( Select *pNew; ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; + Expr *pOffset = p->pOffset; Select *pPrior = p->pPrior; p->pOrderBy = 0; p->pSrc = 0; p->pPrior = 0; p->pLimit = 0; + p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + sqlite3SelectSetName(pNew, pSub->zSelName); + p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; p->op = TK_ALL; - p->pRightmost = 0; if( pNew==0 ){ - pNew = pPrior; + p->pPrior = pPrior; }else{ pNew->pPrior = pPrior; - pNew->pRightmost = 0; + if( pPrior ) pPrior->pNext = pNew; + pNew->pNext = p; + p->pPrior = pNew; + SELECTTRACE(2,pParse,p, + ("compound-subquery flattener creates %s.%p as peer\n", + pNew->zSelName, pNew)); } - p->pPrior = pNew; if( db->mallocFailed ) return 1; } @@ -93444,10 +107633,9 @@ static int flattenSubquery( pList = pParent->pEList; for(i=0; inExpr; i++){ if( pList->a[i].zName==0 ){ - const char *zSpan = pList->a[i].zSpan; - if( ALWAYS(zSpan) ){ - pList->a[i].zName = sqlite3DbStrDup(db, zSpan); - } + char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); + sqlite3Dequote(zName); + pList->a[i].zName = zName; } } substExprList(db, pParent->pEList, iParent, pSub->pEList); @@ -93456,8 +107644,23 @@ static int flattenSubquery( pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); } if( pSub->pOrderBy ){ + /* At this point, any non-zero iOrderByCol values indicate that the + ** ORDER BY column expression is identical to the iOrderByCol'th + ** expression returned by SELECT statement pSub. Since these values + ** do not necessarily correspond to columns in SELECT statement pParent, + ** zero them before transfering the ORDER BY clause. + ** + ** Not doing this may cause an error if a subsequent call to this + ** function attempts to flatten a compound sub-query into pParent + ** (the only way this can happen is if the compound sub-query is + ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ExprList *pOrderBy = pSub->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } assert( pParent->pOrderBy==0 ); - pParent->pOrderBy = pSub->pOrderBy; + assert( pSub->pPrior==0 ); + pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; }else if( pParent->pOrderBy ){ substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); @@ -93503,44 +107706,60 @@ static int flattenSubquery( */ sqlite3SelectDelete(db, pSub1); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + sqlite3DebugPrintf("After flattening:\n"); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + return 1; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Analyze the SELECT statement passed as an argument to see if it -** is a min() or max() query. Return WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX if -** it is, or 0 otherwise. At present, a query is considered to be -** a min()/max() query if: +** Based on the contents of the AggInfo structure indicated by the first +** argument, this function checks if the following are true: ** -** 1. There is a single object in the FROM clause. +** * the query contains just a single aggregate function, +** * the aggregate function is either min() or max(), and +** * the argument to the aggregate function is a column value. ** -** 2. There is a single expression in the result set, and it is -** either min(x) or max(x), where x is a column reference. +** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX +** is returned as appropriate. Also, *ppMinMax is set to point to the +** list of arguments passed to the aggregate before returning. +** +** Or, if the conditions above are not met, *ppMinMax is set to 0 and +** WHERE_ORDERBY_NORMAL is returned. */ -static u8 minMaxQuery(Select *p){ - Expr *pExpr; - ExprList *pEList = p->pEList; +static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ - if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; - pExpr = pEList->a[0].pExpr; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; - pEList = pExpr->x.pList; - if( pEList==0 || pEList->nExpr!=1 ) return 0; - if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken,"min")==0 ){ - return WHERE_ORDERBY_MIN; - }else if( sqlite3StrICmp(pExpr->u.zToken,"max")==0 ){ - return WHERE_ORDERBY_MAX; + *ppMinMax = 0; + if( pAggInfo->nFunc==1 ){ + Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ + ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ + + assert( pExpr->op==TK_AGG_FUNCTION ); + if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ + const char *zFunc = pExpr->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + *ppMinMax = pEList; + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + *ppMinMax = pEList; + } + } } - return WHERE_ORDERBY_NORMAL; + + assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + return eRet; } /* ** The select statement passed as the first argument is an aggregate query. -** The second argment is the associated aggregate-info object. This +** The second argument is the associated aggregate-info object. This ** function tests if the SELECT is of the form: ** ** SELECT count(*) FROM @@ -93566,7 +107785,8 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( IsVirtual(pTab) ) return 0; if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( (pAggInfo->aFunc[0].pFunc->flags&SQLITE_FUNC_COUNT)==0 ) return 0; + if( NEVER(pAggInfo->nFunc==0) ) return 0; + if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0; if( pExpr->flags&EP_Distinct ) return 0; return pTab; @@ -93597,6 +107817,265 @@ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pF } return SQLITE_OK; } +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + p->pNext = 0; + p->selFlags &= ~SF_Compound; + assert( pNew->pPrior!=0 ); + pNew->pPrior->pNext = pNew; + pNew->pLimit = 0; + pNew->pOffset = 0; + return WRC_Continue; +} + +#ifndef SQLITE_OMIT_CTE +/* +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) +** then return a pointer to the CTE definition for that table. Otherwise +** return NULL. +** +** If a non-NULL value is returned, set *ppContext to point to the With +** object that the returned CTE belongs to. +*/ +static struct Cte *searchWith( + With *pWith, /* Current outermost WITH clause */ + struct SrcList_item *pItem, /* FROM clause element to resolve */ + With **ppContext /* OUT: WITH clause return value belongs to */ +){ + const char *zName; + if( pItem->zDatabase==0 && (zName = pItem->zName)!=0 ){ + With *p; + for(p=pWith; p; p=p->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + *ppContext = p; + return &p->a[i]; + } + } + } + } + return 0; +} + +/* The code generator maintains a stack of active WITH clauses +** with the inner-most WITH clause being at the top of the stack. +** +** This routine pushes the WITH clause passed as the second argument +** onto the top of the stack. If argument bFree is true, then this +** WITH clause will never be popped from the stack. In this case it +** should be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT +** statement with which it is associated. +*/ +SQLITE_PRIVATE void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ + assert( bFree==0 || pParse->pWith==0 ); + if( pWith ){ + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + pParse->bFreeWith = bFree; + } +} + +/* +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser. And, +** if currently processing a CTE expression, if it is a recursive +** reference to the current CTE. +** +** If pFrom falls into either of the two categories above, pFrom->pTab +** and other fields are populated accordingly. The caller should check +** (pFrom->pTab!=0) to determine whether or not a successful match +** was found. +** +** Whether or not a match is found, SQLITE_OK is returned if no error +** occurs. If an error does occur, an error message is stored in the +** parser and some error code other than SQLITE_OK returned. +*/ +static int withExpand( + Walker *pWalker, + struct SrcList_item *pFrom +){ + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + struct Cte *pCte; /* Matched CTE (or NULL if no match) */ + With *pWith; /* WITH clause that pCte belongs to */ + + assert( pFrom->pTab==0 ); + + pCte = searchWith(pParse->pWith, pFrom, &pWith); + if( pCte ){ + Table *pTab; + ExprList *pEList; + Select *pSel; + Select *pLeft; /* Left-most SELECT statement */ + int bMayRecursive; /* True if compound joined by UNION [ALL] */ + With *pSavedWith; /* Initial value of pParse->pWith */ + + /* If pCte->zErr is non-NULL at this point, then this is an illegal + ** recursive reference to CTE pCte. Leave an error in pParse and return + ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** In this case, proceed. */ + if( pCte->zErr ){ + sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + return SQLITE_ERROR; + } + + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->nRef = 1; + pTab->zName = sqlite3DbStrDup(db, pCte->zName); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral; + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( db->mallocFailed ) return SQLITE_NOMEM; + assert( pFrom->pSelect ); + + /* Check if this is a recursive CTE. */ + pSel = pFrom->pSelect; + bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); + if( bMayRecursive ){ + int i; + SrcList *pSrc = pFrom->pSelect->pSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pSrc->a[i]; + if( pItem->zDatabase==0 + && pItem->zName!=0 + && 0==sqlite3StrICmp(pItem->zName, pCte->zName) + ){ + pItem->pTab = pTab; + pItem->isRecursive = 1; + pTab->nRef++; + pSel->selFlags |= SF_Recursive; + } + } + } + + /* Only one recursive reference is permitted. */ + if( pTab->nRef>2 ){ + sqlite3ErrorMsg( + pParse, "multiple references to recursive table: %s", pCte->zName + ); + return SQLITE_ERROR; + } + assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); + + pCte->zErr = "circular reference: %s"; + pSavedWith = pParse->pWith; + pParse->pWith = pWith; + sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; + if( pCte->pCols ){ + if( pEList->nExpr!=pCte->pCols->nExpr ){ + sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", + pCte->zName, pEList->nExpr, pCte->pCols->nExpr + ); + pParse->pWith = pSavedWith; + return SQLITE_ERROR; + } + pEList = pCte->pCols; + } + + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + if( bMayRecursive ){ + if( pSel->selFlags & SF_Recursive ){ + pCte->zErr = "multiple recursive references: %s"; + }else{ + pCte->zErr = "recursive reference in a subquery: %s"; + } + sqlite3WalkSelect(pWalker, pSel); + } + pCte->zErr = 0; + pParse->pWith = pSavedWith; + } + + return SQLITE_OK; +} +#endif + +#ifndef SQLITE_OMIT_CTE +/* +** If the SELECT passed as the second argument has an associated WITH +** clause, pop it from the stack stored as part of the Parse object. +** +** This function is used as the xSelectCallback2() callback by +** sqlite3SelectExpand() when walking a SELECT tree to resolve table +** names and other FROM clause elements. +*/ +static void selectPopWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith ); + pParse->pWith = pWith->pOuter; + } +} +#else +#define selectPopWith 0 +#endif /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -93610,10 +108089,10 @@ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pF ** fill pTabList->a[].pSelect with a copy of the SELECT statement ** that implements the view. A copy is made of the view's SELECT ** statement so that we can freely modify or delete that statement -** without worrying about messing up the presistent representation +** without worrying about messing up the persistent representation ** of the view. ** -** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword +** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword ** on joins and the ON and USING clause of joins. ** ** (4) Scan the list of columns in the result set (pEList) looking @@ -93629,16 +108108,19 @@ static int selectExpander(Walker *pWalker, Select *p){ ExprList *pEList; struct SrcList_item *pFrom; sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + p->selFlags |= SF_Expanded; if( db->mallocFailed ){ return WRC_Abort; } - if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){ + if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } - p->selFlags |= SF_Expanded; pTabList = p->pSrc; pEList = p->pEList; + sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. @@ -93651,12 +108133,21 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; + assert( pFrom->isRecursive==0 || pFrom->pTab ); + if( pFrom->isRecursive ) continue; if( pFrom->pTab!=0 ){ /* This statement has already been prepared. There is no need ** to go further. */ assert( i==0 ); +#ifndef SQLITE_OMIT_CTE + selectPopWith(pWalker, p); +#endif return WRC_Prune; } +#ifndef SQLITE_OMIT_CTE + if( withExpand(pWalker, pFrom) ) return WRC_Abort; + if( pFrom->pTab ) {} else +#endif if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY Select *pSel = pFrom->pSelect; @@ -93667,19 +108158,24 @@ static int selectExpander(Walker *pWalker, Select *p){ pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; - pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); + pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1000000; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; #endif }else{ /* An ordinary table or view name in the FROM clause */ assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; + if( pTab->nRef==0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ @@ -93687,6 +108183,7 @@ static int selectExpander(Walker *pWalker, Select *p){ if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); + sqlite3SelectSetName(pFrom->pSelect, pTab->zName); sqlite3WalkSelect(pWalker, pFrom->pSelect); } #endif @@ -93715,7 +108212,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** that need expanding. */ for(k=0; knExpr; k++){ - Expr *pE = pEList->a[k].pExpr; + pE = pEList->a[k].pExpr; if( pE->op==TK_ALL ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); @@ -93733,10 +108230,18 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; + /* When processing FROM-clause subqueries, it is always the case + ** that full_column_names=OFF and short_column_names=ON. The + ** sqlite3ResultSetOfSelect() routine makes it so. */ + assert( (p->selFlags & SF_NestedFrom)==0 + || ((flags & SQLITE_FullColNames)==0 && + (flags & SQLITE_ShortColNames)!=0) ); + for(k=0; knExpr; k++){ - Expr *pE = a[k].pExpr; - assert( pE->op!=TK_DOT || pE->pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pE->pRight->op!=TK_ALL) ){ + pE = a[k].pExpr; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -93751,32 +108256,43 @@ static int selectExpander(Walker *pWalker, Select *p){ /* This expression is a "*" or a "TABLE.*" and needs to be ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ - char *zTName; /* text of name of TABLE */ + char *zTName = 0; /* text of name of TABLE */ if( pE->op==TK_DOT ){ assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; - }else{ - zTName = 0; } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; if( zTabName==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ - continue; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; } - tableSeen = 1; for(j=0; jnCol; j++){ - Expr *pExpr, *pRight; char *zName = pTab->aCol[j].zName; char *zColname; /* The computed column name */ char *zToFree; /* Malloced string that needs to be freed */ Token sColname; /* Computed column name as a token */ + assert( zName ); + if( zTName && pSub + && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 + ){ + continue; + } + /* If a column is marked as 'hidden' (currently only possible ** for virtual tables), do not include it in the expanded ** result-set list. @@ -93785,6 +108301,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert(IsVirtual(pTab)); continue; } + tableSeen = 1; if( i>0 && zTName==0 ){ if( (pFrom->jointype & JT_NATURAL)!=0 @@ -93807,6 +108324,10 @@ static int selectExpander(Walker *pWalker, Select *p){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); + } if( longNames ){ zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); zToFree = zColname; @@ -93818,6 +108339,18 @@ static int selectExpander(Walker *pWalker, Select *p){ sColname.z = zColname; sColname.n = sqlite3Strlen30(zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + if( pSub ){ + pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); + testcase( pX->zSpan==0 ); + }else{ + pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zSpan==0 ); + } + pX->bSpanIsTab = 1; + } sqlite3DbFree(db, zToFree); } } @@ -93870,9 +108403,15 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; - w.xSelectCallback = selectExpander; + memset(&w, 0, sizeof(w)); w.xExprCallback = exprWalkNoop; w.pParse = pParse; + if( pParse->hasCompound ){ + w.xSelectCallback = convertCompoundSelectToSubquery; + sqlite3WalkSelect(&w, pSelect); + } + w.xSelectCallback = selectExpander; + w.xSelectCallback2 = selectPopWith; sqlite3WalkSelect(&w, pSelect); } @@ -93891,7 +108430,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ ** at that point because identifiers had not yet been resolved. This ** routine is called after identifier resolution. */ -static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ +static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; int i; SrcList *pTabList; @@ -93907,13 +108446,13 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ /* A sub-query in the FROM clause of a SELECT */ Select *pSel = pFrom->pSelect; - assert( pSel ); - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSel); + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); + } } } } - return WRC_Continue; } #endif @@ -93928,7 +108467,8 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; - w.xSelectCallback = selectAddSubqueryTypeInfo; + memset(&w, 0, sizeof(w)); + w.xSelectCallback2 = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); @@ -93937,7 +108477,7 @@ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ /* -** This routine sets of a SELECT statement for processing. The +** This routine sets up a SELECT statement for processing. The ** following is accomplished: ** ** * VDBE Cursor numbers are assigned to all FROM-clause terms. @@ -93956,6 +108496,7 @@ SQLITE_PRIVATE void sqlite3SelectPrep( sqlite3 *db; if( NEVER(p==0) ) return; db = pParse->db; + if( db->mallocFailed ) return; if( p->selFlags & SF_HasTypeInfo ) return; sqlite3SelectExpand(pParse, p); if( pParse->nErr || db->mallocFailed ) return; @@ -93969,20 +108510,30 @@ SQLITE_PRIVATE void sqlite3SelectPrep( ** ** The aggregate accumulator is a set of memory cells that hold ** intermediate results while calculating an aggregate. This -** routine simply stores NULLs in all of those memory cells. +** routine generates code that stores NULLs in all of those memory +** cells. */ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; struct AggInfo_func *pFunc; - if( pAggInfo->nFunc+pAggInfo->nColumn==0 ){ - return; - } + int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + if( nReg==0 ) return; +#ifdef SQLITE_DEBUG + /* Verify that all AggInfo registers are within the range specified by + ** AggInfo.mnReg..AggInfo.mxReg */ + assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); for(i=0; inColumn; i++){ - sqlite3VdbeAddOp2(v, OP_Null, 0, pAggInfo->aCol[i].iMem); + assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg + && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); } + for(i=0; inFunc; i++){ + assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg + && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); + } +#endif + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ - sqlite3VdbeAddOp2(v, OP_Null, 0, pFunc->iMem); if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pExpr; assert( !ExprHasProperty(pE, EP_xIsSelect) ); @@ -93991,9 +108542,9 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + (char*)pKeyInfo, P4_KEYINFO); } } } @@ -94022,11 +108573,12 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; + int regHit = 0; + int addrHitTest = 0; struct AggInfo_func *pF; struct AggInfo_col *pC; pAggInfo->directMode = 1; - sqlite3ExprCacheClear(pParse); for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; int addrNext = 0; @@ -94036,7 +108588,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, 1); + sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; @@ -94046,7 +108598,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( nArg==1 ); codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); } - if( pF->pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ CollSeq *pColl = 0; struct ExprList_item *pItem; int j; @@ -94057,7 +108609,8 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( !pColl ){ pColl = pParse->db->pDfltColl; } - sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, (void*)pF->pFunc, P4_FUNCDEF); @@ -94080,12 +108633,18 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ ** Another solution would be to change the OP_SCopy used to copy cached ** values to an OP_Copy. */ + if( regHit ){ + addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); + } sqlite3ExprCacheClear(pParse); for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); } pAggInfo->directMode = 0; sqlite3ExprCacheClear(pParse); + if( addrHitTest ){ + sqlite3VdbeJumpHere(v, addrHitTest); + } } /* @@ -94099,11 +108658,11 @@ static void explainSimpleCount( Index *pIdx /* Index used to optimize scan, or NULL */ ){ if( pParse->explain==2 ){ - char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s %s%s(~%d rows)", - pTab->zName, - pIdx ? "USING COVERING INDEX " : "", - pIdx ? pIdx->zName : "", - pTab->nRowEst + int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx))); + char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s%s%s", + pTab->zName, + bCover ? " USING COVERING INDEX " : "", + bCover ? pIdx->zName : "" ); sqlite3VdbeAddOp4( pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC @@ -94117,49 +108676,8 @@ static void explainSimpleCount( /* ** Generate code for the SELECT statement given in the p argument. ** -** The results are distributed in various ways depending on the -** contents of the SelectDest structure pointed to by argument pDest -** as follows: -** -** pDest->eDest Result -** ------------ ------------------------------------------- -** SRT_Output Generate a row of output (using the OP_ResultRow -** opcode) for each row in the result set. -** -** SRT_Mem Only valid if the result is a single column. -** Store the first column of the first result row -** in register pDest->iParm then abandon the rest -** of the query. This destination implies "LIMIT 1". -** -** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iParm. -** Apply the affinity pDest->affinity before storing -** results. Used to implement "IN (SELECT ...)". -** -** SRT_Union Store results as a key in a temporary table pDest->iParm. -** -** SRT_Except Remove results from the temporary table pDest->iParm. -** -** SRT_Table Store results in temporary table pDest->iParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. -** -** SRT_EphemTab Create an temporary table pDest->iParm and store -** the result there. The cursor is left open after -** returning. This is like SRT_Table except that -** this destination uses OP_OpenEphemeral to create -** the table first. -** -** SRT_Coroutine Generate a co-routine that returns a new row of -** results each time it is invoked. The entry point -** of the co-routine is stored in register pDest->iParm. -** -** SRT_Exists Store a 1 in memory cell pDest->iParm if the result -** set is not empty. -** -** SRT_Discard Throw the results away. This is used by SELECT -** statements within triggers whose only purpose is -** the side-effects of functions. +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. ** ** This routine returns the number of errors. If any errors are ** encountered, then an appropriate error message is left in @@ -94180,13 +108698,11 @@ SQLITE_PRIVATE int sqlite3Select( ExprList *pEList; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ - ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ - int isDistinct; /* True if the DISTINCT keyword is present */ - int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ - int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ + DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ + SortCtx sSort; /* Info on how to code the ORDER BY clause */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -94202,10 +108718,23 @@ SQLITE_PRIVATE int sqlite3Select( } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); +#if SELECTTRACE_ENABLED + pParse->nSelectIndent++; + SELECTTRACE(1,pParse,p, ("begin processing:\n")); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableOrderby(pDest) ){ assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard); + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistFifo || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo); /* If ORDER BY makes no difference in the output then neither does ** DISTINCT so it can be removed too. */ sqlite3ExprListDelete(db, p->pOrderBy); @@ -94213,7 +108742,8 @@ SQLITE_PRIVATE int sqlite3Select( p->selFlags &= ~SF_Distinct; } sqlite3SelectPrep(pParse, p, 0); - pOrderBy = p->pOrderBy; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ @@ -94245,10 +108775,23 @@ SQLITE_PRIVATE int sqlite3Select( Select *pSub = pItem->pSelect; int isAggSub; - if( pSub==0 || pItem->isPopulated ) continue; + if( pSub==0 ) continue; + + /* Sometimes the code for a subquery will be generated more than + ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, + ** for example. In that case, do not regenerate the code to manifest + ** a view or the co-routine to implement a view. The first instance + ** is sufficient, though the subroutine to manifest the view does need + ** to be invoked again. */ + if( pItem->addrFillSub ){ + if( pItem->viaCoroutine==0 ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + } + continue; + } /* Increment Parse.nHeight by the height of the largest expression - ** tree refered to by this, the parent select. The child select + ** tree referred to by this, the parent select. The child select ** may contain expression trees of at most ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit ** more conservative than necessary, but much easier than enforcing @@ -94256,21 +108799,65 @@ SQLITE_PRIVATE int sqlite3Select( */ pParse->nHeight += sqlite3SelectExprHeight(p); - /* Check to see if the subquery can be absorbed into the parent. */ isAggSub = (pSub->selFlags & SF_Aggregate)!=0; if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + /* This subquery can be absorbed into its parent. */ if( isAggSub ){ isAgg = 1; p->selFlags |= SF_Aggregate; } i = -1; - }else{ - sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - assert( pItem->isPopulated==0 ); + }else if( pTabList->nSrc==1 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop = sqlite3VdbeCurrentAddr(v)+1; + pItem->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + VdbeComment((v, "%s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->isPopulated = 1; - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->viaCoroutine = 1; + pItem->regResult = dest.iSdst; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); + }else{ + /* Generate a subroutine that will fill an ephemeral table with + ** the content of this subquery. pItem->addrFillSub will point + ** to the address of the generated subroutine. pItem->regReturn + ** is a register allocated to hold the subroutine return address + */ + int topAddr; + int onceAddr = 0; + int retAddr; + assert( pItem->addrFillSub==0 ); + pItem->regReturn = ++pParse->nMem; + topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); + pItem->addrFillSub = topAddr+1; + if( pItem->isCorrelated==0 ){ + /* If the subquery is not correlated and if we are not inside of + ** a trigger, then we only need to compute the value of the subquery + ** once. */ + onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); + }else{ + VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); + } + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); + retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeChangeP1(v, topAddr, retAddr); + sqlite3ClearTempRegCache(pParse); } if( /*pParse->nErr ||*/ db->mallocFailed ){ goto select_end; @@ -94278,7 +108865,7 @@ SQLITE_PRIVATE int sqlite3Select( pParse->nHeight -= sqlite3SelectExprHeight(p); pTabList = p->pSrc; if( !IgnorableOrderby(pDest) ){ - pOrderBy = p->pOrderBy; + sSort.pOrderBy = p->pOrderBy; } } pEList = p->pEList; @@ -94286,53 +108873,48 @@ SQLITE_PRIVATE int sqlite3Select( pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; - isDistinct = (p->selFlags & SF_Distinct)!=0; + sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #ifndef SQLITE_OMIT_COMPOUND_SELECT /* If there is are a sequence of queries, do the earlier ones first. */ if( p->pPrior ){ - if( p->pRightmost==0 ){ - Select *pLoop, *pRight = 0; - int cnt = 0; - int mxSelect; - for(pLoop=p; pLoop; pLoop=pLoop->pPrior, cnt++){ - pLoop->pRightmost = p; - pLoop->pNext = pRight; - pRight = pLoop; - } - mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; - if( mxSelect && cnt>mxSelect ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - goto select_end; - } - } rc = multiSelect(pParse, p, pDest); explainSetInteger(pParse->iSelectId, iRestoreSelectId); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end compound-select processing\n")); + pParse->nSelectIndent--; +#endif return rc; } #endif - /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. - ** GROUP BY might use an index, DISTINCT never does. + /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + ** if the select-list is the same as the ORDER BY list, then this query + ** can be rewritten as a GROUP BY. In other words, this: + ** + ** SELECT DISTINCT xyz FROM ... ORDER BY xyz + ** + ** is transformed to: + ** + ** SELECT xyz FROM ... GROUP BY xyz + ** + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER + ** BY and DISTINCT, and an index or separate temp-table for the other. */ - assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 + ){ + p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - p->selFlags &= ~SF_Distinct; - } - - /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then disable the ORDER BY clause since the GROUP BY - ** will cause elements to come out in the correct order. This is - ** an optimization - the correct answer should result regardless. - ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER - ** to disable this optimization for testing purposes. - */ - if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 - && (db->flags & SQLITE_GroupByOrder)==0 ){ - pOrderBy = 0; + sSort.pOrderBy = 0; + /* Notice that even thought SF_Distinct has been cleared from p->selFlags, + ** the sDistinct.isTnct is still set. Hence, isTnct represents the + ** original setting of the SF_Distinct flag, not the current setting */ + assert( sDistinct.isTnct ); } /* If there is an ORDER BY clause, then this sorting @@ -94342,73 +108924,89 @@ SQLITE_PRIVATE int sqlite3Select( ** we figure out that the sorting index is not needed. The addrSortIndex ** variable is used to facilitate that change. */ - if( pOrderBy ){ + if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); - pOrderBy->iECursor = pParse->nTab++; - p->addrOpenEphm[2] = addrSortIndex = + pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, 0); + sSort.iECursor = pParse->nTab++; + sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - pOrderBy->iECursor, pOrderBy->nExpr+2, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); }else{ - addrSortIndex = -1; + sSort.addrSortIndex = -1; } /* If the output is destined for a temporary table, open that table. */ if( pDest->eDest==SRT_EphemTab ){ - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iParm, pEList->nExpr); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); } /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = (double)LARGEST_INT64; + p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); + if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sSort.sortFlags |= SORTFLAG_UseSorter; + } /* Open a virtual index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ - KeyInfo *pKeyInfo; - assert( isAgg || pGroupBy ); - distinct = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sDistinct.tabTnct = pParse->nTab++; + sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ - distinct = -1; + sDistinct.eTnctType = WHERE_DISTINCT_NOOP; } - /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - /* This case is for non-aggregate queries - ** Begin the database scan - */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); + /* No aggregate functions and no GROUP BY clause */ + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + + /* Begin the database scan. */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + p->pEList, wctrlFlags, 0); if( pWInfo==0 ) goto select_end; - if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; + if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ + p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + } + if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ + sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); + } + if( sSort.pOrderBy ){ + sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ + sSort.pOrderBy = 0; + } + } /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ - if( addrSortIndex>=0 && pOrderBy==0 ){ - sqlite3VdbeChangeToNoop(v, addrSortIndex, 1); - p->addrOpenEphm[2] = -1; + if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } - /* Use the standard inner loop - */ - assert(!isDistinct); - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, - pWInfo->iContinue, pWInfo->iBreak); + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, pEList, -1, &sSort, &sDistinct, pDest, + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); /* End the database scan loop. */ sqlite3WhereEnd(pWInfo); }else{ - /* This is the processing for aggregate queries */ + /* This case when there exist aggregate functions or a GROUP BY clause + ** or both */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -94418,6 +109016,9 @@ SQLITE_PRIVATE int sqlite3Select( int iAbortFlag; /* Mem address which causes query abort if positive */ int groupBySort; /* Rows come from source in GROUP BY order */ int addrEnd; /* End of processing for this SELECT */ + int sortPTab = 0; /* Pseudotable used to decode sorting results */ + int sortOut = 0; /* Output register from the sorter */ + int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */ /* Remove any and all aliases between the result set and the ** GROUP BY clause. @@ -94427,16 +109028,28 @@ SQLITE_PRIVATE int sqlite3Select( struct ExprList_item *pItem; /* For looping over expression in a list */ for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){ - pItem->iAlias = 0; + pItem->u.x.iAlias = 0; } for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ - pItem->iAlias = 0; + pItem->u.x.iAlias = 0; } - if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100; + if( p->nSelectRow>100 ) p->nSelectRow = 100; }else{ - p->nSelectRow = (double)1; + p->nSelectRow = 1; } + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY may use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } /* Create a label to jump to when we want to abort the query */ addrEnd = sqlite3VdbeMakeLabel(v); @@ -94449,18 +109062,22 @@ SQLITE_PRIVATE int sqlite3Select( sNC.pParse = pParse; sNC.pSrcList = pTabList; sNC.pAggInfo = &sAggInfo; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.mnReg = pParse->nMem+1; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); - sqlite3ExprAnalyzeAggList(&sNC, pOrderBy); + sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } sAggInfo.nAccumulator = sAggInfo.nColumn; for(i=0; ix.pList); + sNC.ncFlags &= ~NC_InAggFunc; } + sAggInfo.mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; /* Processing for aggregates with GROUP BY is very different and @@ -94479,14 +109096,14 @@ SQLITE_PRIVATE int sqlite3Select( /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out - ** that we do not need it after all, the OpenEphemeral instruction + ** that we do not need it after all, the OP_SorterOpen instruction ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); - addrSortingIdx = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, 0); + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, - 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + 0, (char*)pKeyInfo, P4_KEYINFO); /* Initialize memory locations used by GROUP BY aggregate processing */ @@ -94504,6 +109121,7 @@ SQLITE_PRIVATE int sqlite3Select( VdbeComment((v, "clear abort flag")); sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); VdbeComment((v, "indicate accumulator empty")); + sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); /* Begin a loop that will extract all source rows in GROUP BY order. ** This might involve two separate loops with an OP_Sort in between, or @@ -94511,14 +109129,15 @@ SQLITE_PRIVATE int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + ); if( pWInfo==0 ) goto select_end; - if( pGroupBy==0 ){ + if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo */ - pGroupBy = p->pGroupBy; groupBySort = 0; }else{ /* Rows are coming out in undetermined order. We have to push @@ -94532,12 +109151,13 @@ SQLITE_PRIVATE int sqlite3Select( int nGroupBy; explainTempTable(pParse, - isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); + (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? + "DISTINCT" : "GROUP BY"); groupBySort = 1; nGroupBy = pGroupBy->nExpr; - nCol = nGroupBy + 1; - j = nGroupBy+1; + nCol = nGroupBy; + j = nGroupBy; for(i=0; i=j ){ nCol++; @@ -94547,8 +109167,7 @@ SQLITE_PRIVATE int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); - j = nGroupBy+1; + j = nGroupBy; for(i=0; iiSorterColumn>=j ){ @@ -94556,7 +109175,7 @@ SQLITE_PRIVATE int sqlite3Select( int r2; r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); if( r1!=r2 ){ sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); } @@ -94565,14 +109184,32 @@ SQLITE_PRIVATE int sqlite3Select( } regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, sAggInfo.sortingIdx, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, sAggInfo.sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sqlite3VdbeAddOp2(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); - VdbeComment((v, "GROUP BY sort")); + sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++; + sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); + sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); + VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); sAggInfo.useSortingIdx = 1; sqlite3ExprCacheClear(pParse); + + } + + /* If the index or temporary table used by the GROUP BY sort + ** will naturally deliver rows in the order required by the ORDER BY + ** clause, cancel the ephemeral table open coded earlier. + ** + ** This is an optimization - the correct answer should result regardless. + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** disable this optimization for testing purposes. */ + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + && (groupBySort || sqlite3WhereIsSorted(pWInfo)) + ){ + sSort.pOrderBy = 0; + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } /* Evaluate the current GROUP BY terms and store in b0, b1, b2... @@ -94582,18 +109219,21 @@ SQLITE_PRIVATE int sqlite3Select( */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3ExprCacheClear(pParse); + if( groupBySort ){ + sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, sortOut,sortPTab); + } for(j=0; jnExpr; j++){ if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_Column, sAggInfo.sortingIdx, j, iBMem+j); + sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); }else{ sAggInfo.directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, - (char*)pKeyInfo, P4_KEYINFO); + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); + sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code @@ -94607,7 +109247,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); VdbeComment((v, "output one row")); - sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); + sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); VdbeComment((v, "reset accumulator")); @@ -94623,10 +109263,11 @@ SQLITE_PRIVATE int sqlite3Select( /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); + VdbeCoverage(v); }else{ sqlite3WhereEnd(pWInfo); - sqlite3VdbeChangeToNoop(v, addrSortingIdx, 1); + sqlite3VdbeChangeToNoop(v, addrSortingIdx); } /* Output the final row of result @@ -94651,13 +109292,13 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); sqlite3VdbeResolveLabel(v, addrOutputRow); addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, - distinct, pDest, + selectInnerLoop(pParse, p, p->pEList, -1, &sSort, + &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); VdbeComment((v, "end groupby result generator")); @@ -94697,33 +109338,34 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* Search for the index that has the least amount of columns. If - ** there is such an index, and it has less columns than the table - ** does, then we can assume that it consumes less space on disk and - ** will therefore be cheaper to scan to determine the query result. - ** In this case set iRoot to the root page number of the index b-tree - ** and pKeyInfo to the KeyInfo structure required to navigate the - ** index. + /* Search for the index that has the lowest scan cost. ** ** (2011-04-15) Do not do a full scan of an unordered index. ** + ** (2013-10-03) Do not count the entries in a partial index. + ** ** In practice the KeyInfo structure will not be used. It is only ** passed to keep OP_OpenRead happy. */ + if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->bUnordered==0 && (!pBest || pIdx->nColumnnColumn) ){ + if( pIdx->bUnordered==0 + && pIdx->szIdxRowszTabRow + && pIdx->pPartIdxWhere==0 + && (!pBest || pIdx->szIdxRowszIdxRow) + ){ pBest = pIdx; } } - if( pBest && pBest->nColumnnCol ){ + if( pBest ){ iRoot = pBest->tnum; - pKeyInfo = sqlite3IndexKeyinfo(pParse, pBest); + pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pBest); } /* Open a read-only cursor, execute the OP_Count, close the cursor. */ - sqlite3VdbeAddOp3(v, OP_OpenRead, iCsr, iRoot, iDb); + sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, iRoot, iDb, 1); if( pKeyInfo ){ - sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); sqlite3VdbeAddOp1(v, OP_Close, iCsr); @@ -94745,7 +109387,7 @@ SQLITE_PRIVATE int sqlite3Select( ** value of x, the only row required). ** ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behaviour as follows: + ** modify behavior as follows: ** ** + If the query is a "SELECT min(x)", then the loop coded by ** where.c should not iterate over any values with a NULL value @@ -94757,10 +109399,17 @@ SQLITE_PRIVATE int sqlite3Select( ** Refer to code and comments in where.c for details. */ ExprList *pMinMax = 0; - u8 flag = minMaxQuery(p); + u8 flag = WHERE_ORDERBY_NORMAL; + + assert( p->pGroupBy==0 ); + assert( flag==0 ); + if( p->pHaving==0 ){ + flag = minMaxQuery(&sAggInfo, &pMinMax); + } + assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); + if( flag ){ - assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) ); - pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0); + pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; if( pMinMax && !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; @@ -94773,14 +109422,15 @@ SQLITE_PRIVATE int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); - if( !pMinMax && flag ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); + assert( pMinMax==0 || pMinMax->nExpr==1 ); + if( sqlite3WhereIsOrdered(pWInfo)>0 ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } @@ -94788,9 +109438,9 @@ SQLITE_PRIVATE int sqlite3Select( finalizeAggFunctions(pParse, &sAggInfo); } - pOrderBy = 0; + sSort.pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -94798,16 +109448,16 @@ SQLITE_PRIVATE int sqlite3Select( } /* endif aggregate query */ - if( distinct>=0 ){ + if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ explainTempTable(pParse, "DISTINCT"); } /* If there is an ORDER BY clause, then we need to sort the results ** and send them to the callback one by one. */ - if( pOrderBy ){ - explainTempTable(pParse, "ORDER BY"); - generateSortTail(pParse, p, v, pEList->nExpr, pDest); + if( sSort.pOrderBy ){ + explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } /* Jump here to skip this query @@ -94833,104 +109483,106 @@ select_end: sqlite3DbFree(db, sAggInfo.aCol); sqlite3DbFree(db, sAggInfo.aFunc); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end processing\n")); + pParse->nSelectIndent--; +#endif return rc; } -#if defined(SQLITE_DEBUG) +#ifdef SQLITE_DEBUG /* -******************************************************************************* -** The following code is used for testing and debugging only. The code -** that follows does not appear in normal builds. -** -** These routines are used to print out the content of all or part of a -** parse structures such as Select or Expr. Such printouts are useful -** for helping to understand what is happening inside the code generator -** during the execution of complex SELECT statements. -** -** These routine are not called anywhere from within the normal -** code base. Then are intended to be called from within the debugger -** or from temporary "printf" statements inserted for debugging. +** Generate a human-readable description of a the Select object. */ -SQLITE_PRIVATE void sqlite3PrintExpr(Expr *p){ - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - sqlite3DebugPrintf("(%s", p->u.zToken); - }else{ - sqlite3DebugPrintf("(%d", p->op); - } - if( p->pLeft ){ - sqlite3DebugPrintf(" "); - sqlite3PrintExpr(p->pLeft); - } - if( p->pRight ){ - sqlite3DebugPrintf(" "); - sqlite3PrintExpr(p->pRight); - } - sqlite3DebugPrintf(")"); -} -SQLITE_PRIVATE void sqlite3PrintExprList(ExprList *pList){ - int i; - for(i=0; inExpr; i++){ - sqlite3PrintExpr(pList->a[i].pExpr); - if( inExpr-1 ){ - sqlite3DebugPrintf(", "); - } - } -} -SQLITE_PRIVATE void sqlite3PrintSelect(Select *p, int indent){ - sqlite3DebugPrintf("%*sSELECT(%p) ", indent, "", p); - sqlite3PrintExprList(p->pEList); - sqlite3DebugPrintf("\n"); - if( p->pSrc ){ - char *zPrefix; +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewLine(pView, "SELECT%s%s", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : "") + ); + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + if( p->pPrior ) n++; + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pSrc && p->pSrc->nSrc ){ int i; - zPrefix = "FROM"; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; - sqlite3DebugPrintf("%*s ", indent+6, zPrefix); - zPrefix = ""; - if( pItem->pSelect ){ - sqlite3DebugPrintf("(\n"); - sqlite3PrintSelect(pItem->pSelect, indent+10); - sqlite3DebugPrintf("%*s)", indent+8, ""); + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); }else if( pItem->zName ){ - sqlite3DebugPrintf("%s", pItem->zName); + sqlite3XPrintf(&x, 0, " %s", pItem->zName); } if( pItem->pTab ){ - sqlite3DebugPrintf("(table: %s)", pItem->pTab->zName); + sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); } if( pItem->zAlias ){ - sqlite3DebugPrintf(" AS %s", pItem->zAlias); + sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); } - if( ipSrc->nSrc-1 ){ - sqlite3DebugPrintf(","); + if( pItem->jointype & JT_LEFT ){ + sqlite3XPrintf(&x, 0, " LEFT-JOIN"); } - sqlite3DebugPrintf("\n"); + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + sqlite3TreeViewPop(pView); } + sqlite3TreeViewPop(pView); } if( p->pWhere ){ - sqlite3DebugPrintf("%*s WHERE ", indent, ""); - sqlite3PrintExpr(p->pWhere); - sqlite3DebugPrintf("\n"); + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); } if( p->pGroupBy ){ - sqlite3DebugPrintf("%*s GROUP BY ", indent, ""); - sqlite3PrintExprList(p->pGroupBy); - sqlite3DebugPrintf("\n"); + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); } if( p->pHaving ){ - sqlite3DebugPrintf("%*s HAVING ", indent, ""); - sqlite3PrintExpr(p->pHaving); - sqlite3DebugPrintf("\n"); + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); } if( p->pOrderBy ){ - sqlite3DebugPrintf("%*s ORDER BY ", indent, ""); - sqlite3PrintExprList(p->pOrderBy); - sqlite3DebugPrintf("\n"); + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOffset ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, (n--)>0); + sqlite3TreeViewSelect(pView, p->pPrior, 0); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); } -/* End of the structure debug printing code -*****************************************************************************/ -#endif /* defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ +#endif /* SQLITE_DEBUG */ /************** End of select.c **********************************************/ /************** Begin file table.c *******************************************/ @@ -94952,6 +109604,8 @@ SQLITE_PRIVATE void sqlite3PrintSelect(Select *p, int indent){ ** These routines are in a separate files so that they will not be linked ** if they are not used. */ +/* #include */ +/* #include */ #ifndef SQLITE_OMIT_GET_TABLE @@ -94962,10 +109616,10 @@ SQLITE_PRIVATE void sqlite3PrintSelect(Select *p, int indent){ typedef struct TabResult { char **azResult; /* Accumulated output */ char *zErrMsg; /* Error message text, if an error occurs */ - int nAlloc; /* Slots allocated for azResult[] */ - int nRow; /* Number of rows in the result */ - int nColumn; /* Number of columns in the result */ - int nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ + u32 nAlloc; /* Slots allocated for azResult[] */ + u32 nRow; /* Number of rows in the result */ + u32 nColumn; /* Number of columns in the result */ + u32 nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ int rc; /* Return code from sqlite3_exec() */ } TabResult; @@ -94991,7 +109645,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( p->nData + need > p->nAlloc ){ char **azNew; p->nAlloc = p->nAlloc*2 + need; - azNew = sqlite3_realloc( p->azResult, sizeof(char*)*p->nAlloc ); + azNew = sqlite3_realloc64( p->azResult, sizeof(char*)*p->nAlloc ); if( azNew==0 ) goto malloc_failed; p->azResult = azNew; } @@ -95006,7 +109660,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( z==0 ) goto malloc_failed; p->azResult[p->nData++] = z; } - }else if( p->nColumn!=nCol ){ + }else if( (int)p->nColumn!=nCol ){ sqlite3_free(p->zErrMsg); p->zErrMsg = sqlite3_mprintf( "sqlite3_get_table() called with two or more incompatible queries" @@ -95115,7 +109769,7 @@ SQLITE_API int sqlite3_get_table( ** This routine frees the space the sqlite3_get_table() malloced. */ SQLITE_API void sqlite3_free_table( - char **azResult /* Result returned from from sqlite3_get_table() */ + char **azResult /* Result returned from sqlite3_get_table() */ ){ if( azResult ){ int i, n; @@ -95243,21 +109897,34 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( iDb = 1; pName = pName1; }else{ - /* Figure out the db that the the trigger will be created in */ + /* Figure out the db that the trigger will be created in */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ){ goto trigger_cleanup; } } + if( !pTableName || db->mallocFailed ){ + goto trigger_cleanup; + } + + /* A long-standing parser bug is that this syntax was allowed: + ** + ** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab .... + ** ^^^^^^^^ + ** + ** To maintain backwards compatibility, ignore the database + ** name on pTableName if we are reparsing out of SQLITE_MASTER. + */ + if( db->init.busy && iDb!=1 ){ + sqlite3DbFree(db, pTableName->a[0].zDatabase); + pTableName->a[0].zDatabase = 0; + } /* If the trigger name was unqualified, and the table is a temp table, ** then set iDb to 1 to create the trigger in the temporary database. ** If sqlite3SrcListLookup() returns 0, indicating the table does not ** exist, the error is caught by the block below. */ - if( !pTableName || db->mallocFailed ){ - goto trigger_cleanup; - } pTab = sqlite3SrcListLookup(pParse, pTableName); if( db->init.busy==0 && pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ @@ -95267,8 +109934,8 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( /* Ensure the table name matches database name and that the table exists */ if( db->mallocFailed ) goto trigger_cleanup; assert( pTableName->nSrc==1 ); - if( sqlite3FixInit(&sFix, pParse, iDb, "trigger", pName) && - sqlite3FixSrcList(&sFix, pTableName) ){ + sqlite3FixInit(&sFix, pParse, iDb, "trigger", pName); + if( sqlite3FixSrcList(&sFix, pTableName) ){ goto trigger_cleanup; } pTab = sqlite3SrcListLookup(pParse, pTableName); @@ -95299,8 +109966,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash), - zName, sqlite3Strlen30(zName)) ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ if( !noErr ){ sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); }else{ @@ -95410,8 +110076,10 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( } nameToken.z = pTrig->zName; nameToken.n = sqlite3Strlen30(nameToken.z); - if( sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken) - && sqlite3FixTriggerStep(&sFix, pTrig->step_list) ){ + sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); + if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) + || sqlite3FixExpr(&sFix, pTrig->pWhen) + ){ goto triggerfinish_cleanup; } @@ -95441,13 +110109,12 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig); + pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ db->mallocFailed = 1; }else if( pLink->pSchema==pLink->pTabSchema ){ Table *pTab; - int n = sqlite3Strlen30(pLink->table); - pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table, n); + pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); assert( pTab!=0 ); pLink->pNext = pTab->pTrigger; pTab->pTrigger = pLink; @@ -95514,25 +110181,21 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( sqlite3 *db, /* The database connection */ Token *pTableName, /* Name of the table into which we insert */ IdList *pColumn, /* List of columns in pTableName to insert into */ - ExprList *pEList, /* The VALUE clause: a list of values to be inserted */ Select *pSelect, /* A SELECT statement that supplies values */ u8 orconf /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ ){ TriggerStep *pTriggerStep; - assert(pEList == 0 || pSelect == 0); - assert(pEList != 0 || pSelect != 0 || db->mallocFailed); + assert(pSelect != 0 || db->mallocFailed); pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName); if( pTriggerStep ){ pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); pTriggerStep->pIdList = pColumn; - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); pTriggerStep->orconf = orconf; }else{ sqlite3IdListDelete(db, pColumn); } - sqlite3ExprListDelete(db, pEList); sqlite3SelectDelete(db, pSelect); return pTriggerStep; @@ -95610,7 +110273,6 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) int i; const char *zDb; const char *zName; - int nName; sqlite3 *db = pParse->db; if( db->mallocFailed ) goto drop_trigger_cleanup; @@ -95621,13 +110283,12 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) assert( pName->nSrc==1 ); zDb = pName->a[0].zDatabase; zName = pName->a[0].zName; - nName = sqlite3Strlen30(zName); assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); + pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); if( pTrigger ) break; } if( !pTrigger ){ @@ -95650,8 +110311,7 @@ drop_trigger_cleanup: ** is set on. */ static Table *tableOfTrigger(Trigger *pTrigger){ - int n = sqlite3Strlen30(pTrigger->table); - return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table, n); + return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table); } @@ -95687,6 +110347,7 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ int base; + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList dropTrigger[] = { { OP_Rewind, 0, ADDR(9), 0}, { OP_String8, 0, 1, 0}, /* 1 */ @@ -95701,7 +110362,7 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3OpenMasterTable(pParse, iDb); - base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger); + base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger, iLn); sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, P4_TRANSIENT); sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); sqlite3ChangeCookie(pParse, iDb); @@ -95722,7 +110383,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const ch assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &(db->aDb[iDb].pSchema->trigHash); - pTrigger = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), 0); + pTrigger = sqlite3HashInsert(pHash, zName, 0); if( ALWAYS(pTrigger) ){ if( pTrigger->pSchema==pTrigger->pTabSchema ){ Table *pTab = tableOfTrigger(pTrigger); @@ -95847,6 +110508,7 @@ static int codeTriggerProgram( ** INSERT OR IGNORE INTO t1 ... ; -- insert into t2 uses IGNORE policy */ pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; + assert( pParse->okConstFactor==0 ); switch( pStep->op ){ case TK_UPDATE: { @@ -95861,7 +110523,6 @@ static int codeTriggerProgram( case TK_INSERT: { sqlite3Insert(pParse, targetSrcList(pParse, pStep), - sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3SelectDup(db, pStep->pSelect, 0), sqlite3IdListDup(db, pStep->pIdList), pParse->eOrconf @@ -95892,7 +110553,7 @@ static int codeTriggerProgram( return 0; } -#ifdef SQLITE_DEBUG +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS /* ** This function is used to add VdbeComment() annotations to a VDBE ** program. It is not used in production code, only for debugging. @@ -96023,6 +110684,7 @@ static TriggerPrg *codeRowTrigger( } pProgram->nMem = pSubParse->nMem; pProgram->nCsr = pSubParse->nTab; + pProgram->nOnce = pSubParse->nOnce; pProgram->token = (void *)pTrigger; pPrg->aColmask[0] = pSubParse->oldmask; pPrg->aColmask[1] = pSubParse->newmask; @@ -96031,6 +110693,7 @@ static TriggerPrg *codeRowTrigger( assert( !pSubParse->pAinc && !pSubParse->pZombieTab ); assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg ); + sqlite3ParserReset(pSubParse); sqlite3StackFree(db, pSubParse); return pPrg; @@ -96111,7 +110774,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( /* ** This is called to code the required FOR EACH ROW triggers for an operation ** on table pTab. The operation to code triggers for (INSERT, UPDATE or DELETE) -** is given by the op paramater. The tr_tm parameter determines whether the +** is given by the op parameter. The tr_tm parameter determines whether the ** BEFORE or AFTER triggers are coded. If the operation is an UPDATE, then ** parameter pChanges is passed the list of columns being modified. ** @@ -96305,7 +110968,7 @@ static void updateVirtualTable( SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ assert( pTab!=0 ); if( !pTab->pSelect ){ - sqlite3_value *pValue; + sqlite3_value *pValue = 0; u8 enc = ENC(sqlite3VdbeDb(v)); Column *pCol = &pTab->aCol[i]; VdbeComment((v, "%s.%s", pTab->zName, pCol->zName)); @@ -96316,7 +110979,7 @@ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ sqlite3VdbeChangeP4(v, -1, (const char *)pValue, P4_MEM); } #ifndef SQLITE_OMIT_FLOATING_POINT - if( iReg>=0 && pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif @@ -96339,25 +111002,32 @@ SQLITE_PRIVATE void sqlite3Update( ){ int i, j; /* Loop counters */ Table *pTab; /* The table to be updated */ - int addr = 0; /* VDBE instruction address of the start of the loop */ + int addrTop = 0; /* VDBE instruction address of the start of the loop */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Vdbe *v; /* The virtual database engine */ Index *pIdx; /* For looping over indices */ + Index *pPk; /* The PRIMARY KEY index for WITHOUT ROWID tables */ int nIdx; /* Number of indices that need updating */ - int iCur; /* VDBE Cursor number of pTab */ + int iBaseCur; /* Base cursor number */ + int iDataCur; /* Cursor for the canonical data btree */ + int iIdxCur; /* Cursor for the first index */ sqlite3 *db; /* The database structure */ int *aRegIdx = 0; /* One register assigned to each index to be updated */ int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the ** an expression for the i-th column of the table. ** aXRef[i]==-1 if the i-th column is not changed. */ - int chngRowid; /* True if the record number is being changed */ + u8 *aToOpen; /* 1 for tables and indices to be opened */ + u8 chngPk; /* PRIMARY KEY changed in a WITHOUT ROWID table */ + u8 chngRowid; /* Rowid changed in a normal table */ + u8 chngKey; /* Either chngPk or chngRowid */ Expr *pRowidExpr = 0; /* Expression defining the new record number */ - int openAll = 0; /* True if all indices need to be opened */ AuthContext sContext; /* The authorization context */ NameContext sNC; /* The name-context to resolve expressions in */ int iDb; /* Database containing the table being updated */ int okOnePass; /* True for one-pass algorithm without the FIFO */ int hasFK; /* True if foreign key processing is required */ + int labelBreak; /* Jump here to break out of UPDATE loop */ + int labelContinue; /* Jump here to continue next step of UPDATE loop */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True when updating a view (INSTEAD OF trigger) */ @@ -96365,14 +111035,18 @@ SQLITE_PRIVATE void sqlite3Update( int tmask; /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ #endif int newmask; /* Mask of NEW.* columns accessed by BEFORE triggers */ + int iEph = 0; /* Ephemeral table holding all primary key values */ + int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ + int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ int regOldRowid; /* The old rowid */ int regNewRowid; /* The new rowid */ - int regNew; - int regOld = 0; + int regNew; /* Content of the NEW.* table in triggers */ + int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ + int regKey = 0; /* composite PRIMARY KEY value */ memset(&sContext, 0, sizeof(sContext)); db = pParse->db; @@ -96410,20 +111084,34 @@ SQLITE_PRIVATE void sqlite3Update( if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ goto update_cleanup; } - aXRef = sqlite3DbMallocRaw(db, sizeof(int) * pTab->nCol ); - if( aXRef==0 ) goto update_cleanup; - for(i=0; inCol; i++) aXRef[i] = -1; /* Allocate a cursors for the main database table and for all indices. ** The index cursors might not be used, but if they are used they ** need to occur right after the database cursor. So go ahead and ** allocate enough space, just in case. */ - pTabList->a[0].iCursor = iCur = pParse->nTab++; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + pTabList->a[0].iCursor = iBaseCur = iDataCur = pParse->nTab++; + iIdxCur = iDataCur+1; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + if( IsPrimaryKeyIndex(pIdx) && pPk!=0 ){ + iDataCur = pParse->nTab; + pTabList->a[0].iCursor = iDataCur; + } pParse->nTab++; } + /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. + ** Initialize aXRef[] and aToOpen[] to their default values. + */ + aXRef = sqlite3DbMallocRaw(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + if( aXRef==0 ) goto update_cleanup; + aRegIdx = aXRef+pTab->nCol; + aToOpen = (u8*)(aRegIdx+nIdx); + memset(aToOpen, 1, nIdx+1); + aToOpen[nIdx+1] = 0; + for(i=0; inCol; i++) aXRef[i] = -1; + /* Initialize the name-context */ memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; @@ -96435,7 +111123,7 @@ SQLITE_PRIVATE void sqlite3Update( ** column to be updated, make sure we have authorization to change ** that column. */ - chngRowid = 0; + chngRowid = chngPk = 0; for(i=0; inExpr; i++){ if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ goto update_cleanup; @@ -96445,13 +111133,16 @@ SQLITE_PRIVATE void sqlite3Update( if( j==pTab->iPKey ){ chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; + }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ + chngPk = 1; } aXRef[j] = i; break; } } if( j>=pTab->nCol ){ - if( sqlite3IsRowid(pChanges->a[i].zName) ){ + if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zName) ){ + j = -1; chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else{ @@ -96464,7 +111155,8 @@ SQLITE_PRIVATE void sqlite3Update( { int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, - pTab->aCol[j].zName, db->aDb[iDb].zName); + j<0 ? "ROWID" : pTab->aCol[j].zName, + db->aDb[iDb].zName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ @@ -96473,32 +111165,36 @@ SQLITE_PRIVATE void sqlite3Update( } #endif } + assert( (chngRowid & chngPk)==0 ); + assert( chngRowid==0 || chngRowid==1 ); + assert( chngPk==0 || chngPk==1 ); + chngKey = chngRowid + chngPk; - hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngRowid); - - /* Allocate memory for the array aRegIdx[]. There is one entry in the - ** array for each index associated with table being updated. Fill in - ** the value with a register number for indices that are to be used - ** and with zero for unused indices. + /* The SET expressions are not actually used inside the WHERE loop. + ** So reset the colUsed mask + */ + pTabList->a[0].colUsed = 0; + + hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngKey); + + /* There is one entry in the aRegIdx[] array for each index on the table + ** being updated. Fill in aRegIdx[] with a register number that will hold + ** the key for accessing each index. */ - for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){} - if( nIdx>0 ){ - aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx ); - if( aRegIdx==0 ) goto update_cleanup; - } for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; - if( hasFK || chngRowid ){ + if( chngKey || hasFK || pIdx->pPartIdxWhere || pIdx==pPk ){ reg = ++pParse->nMem; }else{ reg = 0; - for(i=0; inColumn; i++){ + for(i=0; inKeyCol; i++){ if( aXRef[pIdx->aiColumn[i]]>=0 ){ reg = ++pParse->nMem; break; } } } + if( reg==0 ) aToOpen[j+1] = 0; aRegIdx[j] = reg; } @@ -96520,12 +111216,13 @@ SQLITE_PRIVATE void sqlite3Update( #endif /* Allocate required registers. */ + regRowSet = ++pParse->nMem; regOldRowid = regNewRowid = ++pParse->nMem; - if( pTrigger || hasFK ){ + if( chngPk || pTrigger || hasFK ){ regOld = pParse->nMem + 1; pParse->nMem += pTab->nCol; } - if( chngRowid || pTrigger || hasFK ){ + if( chngKey || pTrigger || hasFK ){ regNewRowid = ++pParse->nMem; } regNew = pParse->nMem + 1; @@ -96537,11 +111234,11 @@ SQLITE_PRIVATE void sqlite3Update( } /* If we are trying to update a view, realize that view into - ** a ephemeral table. + ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iCur); + sqlite3MaterializeView(pParse, pTab, pWhere, iDataCur); } #endif @@ -96554,23 +111251,58 @@ SQLITE_PRIVATE void sqlite3Update( /* Begin the database scan */ - sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = pWInfo->okOnePass; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur + ); + if( pWInfo==0 ) goto update_cleanup; + okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + + /* Remember the rowid of every item to be updated. + */ + sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); + if( !okOnePass ){ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); + } + + /* End the database scan loop. + */ + sqlite3WhereEnd(pWInfo); + }else{ + int iPk; /* First of nPk memory cells holding PRIMARY KEY value */ + i16 nPk; /* Number of components of the PRIMARY KEY */ + int addrOpen; /* Address of the OpenEphemeral instruction */ - /* Remember the rowid of every item to be updated. - */ - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regOldRowid); - if( !okOnePass ){ - regRowSet = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + regKey = ++pParse->nMem; + iEph = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Null, 0, iPk); + addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, + WHERE_ONEPASS_DESIRED, iIdxCur); + if( pWInfo==0 ) goto update_cleanup; + okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + for(i=0; iaiColumn[i], + iPk+i); + } + if( okOnePass ){ + sqlite3VdbeChangeToNoop(v, addrOpen); + nKey = nPk; + regKey = iPk; + }else{ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, + sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); + } + sqlite3WhereEnd(pWInfo); } - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); - /* Initialize the count of updated rows */ if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ @@ -96578,6 +111310,7 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); } + labelBreak = sqlite3VdbeMakeLabel(v); if( !isView ){ /* ** Open every index that needs updating. Note that if any @@ -96585,73 +111318,84 @@ SQLITE_PRIVATE void sqlite3Update( ** action, then we need to open all indices because we might need ** to be deleting some records. */ - if( !okOnePass ) sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenWrite); if( onError==OE_Replace ){ - openAll = 1; + memset(aToOpen, 1, nIdx+1); }else{ - openAll = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_Replace ){ - openAll = 1; + memset(aToOpen, 1, nIdx+1); break; } } } - for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - if( openAll || aRegIdx[i]>0 ){ - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iCur+i+1, pIdx->tnum, iDb, - (char*)pKey, P4_KEYINFO_HANDOFF); - assert( pParse->nTab>iCur+i+1 ); - } + if( okOnePass ){ + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iBaseCur, aToOpen, + 0, 0); } /* Top of the update loop */ if( okOnePass ){ - int a1 = sqlite3VdbeAddOp1(v, OP_NotNull, regOldRowid); - addr = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, a1); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); + VdbeCoverageNeverTaken(v); + } + labelContinue = labelBreak; + sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); + VdbeCoverageIf(v, pPk==0); + VdbeCoverageIf(v, pPk!=0); + }else if( pPk ){ + labelContinue = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); + addrTop = sqlite3VdbeAddOp2(v, OP_RowKey, iEph, regKey); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0); + VdbeCoverage(v); }else{ - addr = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, 0, regOldRowid); + labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, labelBreak, + regOldRowid); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid); + VdbeCoverage(v); } - /* Make cursor iCur point to the record that is being updated. If - ** this record does not exist for some reason (deleted by a trigger, - ** for example, then jump to the next iteration of the RowSet loop. */ - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid); - /* If the record number will change, set register regNewRowid to ** contain the new value. If the record number is not being modified, ** then regNewRowid is the same register as regOldRowid, which is ** already populated. */ - assert( chngRowid || pTrigger || hasFK || regOldRowid==regNewRowid ); + assert( chngKey || pTrigger || hasFK || regOldRowid==regNewRowid ); if( chngRowid ){ sqlite3ExprCode(pParse, pRowidExpr, regNewRowid); - sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v); } - /* If there are triggers on this table, populate an array of registers - ** with the required old.* column data. */ - if( hasFK || pTrigger ){ + /* Compute the old pre-UPDATE content of the row being changed, if that + ** information is needed */ + if( chngPk || hasFK || pTrigger ){ u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); oldmask |= sqlite3TriggerColmask(pParse, pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError ); for(i=0; inCol; i++){ - if( aXRef[i]<0 || oldmask==0xffffffff || (i<32 && (oldmask & (1<aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 + ){ + testcase( oldmask!=0xffffffff && i==31 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regOld+i); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, regOld+i); } } - if( chngRowid==0 ){ + if( chngRowid==0 && pPk==0 ){ sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } /* Populate the array of registers beginning at regNew with the new - ** row data. This array is used to check constaints, create the new + ** row data. This array is used to check constants, create the new ** table and index records, and as the values for any new.* references ** made by triggers. ** @@ -96666,6 +111410,7 @@ SQLITE_PRIVATE void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); + /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ for(i=0; inCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); @@ -96673,7 +111418,7 @@ SQLITE_PRIVATE void sqlite3Update( j = aXRef[i]; if( j>=0 ){ sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); - }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask&(1<31 || (newmask & MASKBIT32(i)) ){ /* This branch loads the value of a column that will not be changed ** into a register. This is done if there are no BEFORE triggers, or ** if there are one or more BEFORE triggers that use this value via @@ -96681,8 +111426,9 @@ SQLITE_PRIVATE void sqlite3Update( */ testcase( i==31 ); testcase( i==32 ); - sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i); - sqlite3ColumnDefault(v, pTab, i, regNew+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); } } } @@ -96691,18 +111437,23 @@ SQLITE_PRIVATE void sqlite3Update( ** verified. One could argue that this is wrong. */ if( tmask&TRIGGER_BEFORE ){ - sqlite3VdbeAddOp2(v, OP_Affinity, regNew, pTab->nCol); - sqlite3TableAffinityStr(v, pTab); + sqlite3TableAffinity(v, pTab, regNew); sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, - TRIGGER_BEFORE, pTab, regOldRowid, onError, addr); + TRIGGER_BEFORE, pTab, regOldRowid, onError, labelContinue); /* The row-trigger may have deleted the row being updated. In this ** case, jump to the next row. No updates or AFTER triggers are - ** required. This behaviour - what happens when the row being updated + ** required. This behavior - what happens when the row being updated ** is deleted or renamed by a BEFORE trigger - is left undefined in the ** documentation. */ - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid); + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue,regKey,nKey); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid); + VdbeCoverage(v); + } /* If it did not delete it, the row-trigger may still have modified ** some of the columns of the row being updated. Load the values for @@ -96711,46 +111462,57 @@ SQLITE_PRIVATE void sqlite3Update( */ for(i=0; inCol; i++){ if( aXRef[i]<0 && i!=pTab->iPKey ){ - sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i); - sqlite3ColumnDefault(v, pTab, i, regNew+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); } } } if( !isView ){ - int j1; /* Address of jump instruction */ + int j1 = 0; /* Address of jump instruction */ + int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ - sqlite3GenerateConstraintChecks(pParse, pTab, iCur, regNewRowid, - aRegIdx, (chngRowid?regOldRowid:0), 1, onError, addr, 0); + assert( regOldRowid>0 ); + sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, + regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace); /* Do FK constraint checks. */ if( hasFK ){ - sqlite3FkCheck(pParse, pTab, regOldRowid, 0); + sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey); } /* Delete the index entries associated with the current record. */ - j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regOldRowid); - sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, aRegIdx); + if( bReplace || chngKey ){ + if( pPk ){ + j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + }else{ + j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + } + VdbeCoverageNeverTaken(v); + } + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); /* If changing the record number, delete the old record. */ - if( hasFK || chngRowid ){ - sqlite3VdbeAddOp2(v, OP_Delete, iCur, 0); + if( hasFK || chngKey || pPk!=0 ){ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); + } + if( bReplace || chngKey ){ + sqlite3VdbeJumpHere(v, j1); } - sqlite3VdbeJumpHere(v, j1); if( hasFK ){ - sqlite3FkCheck(pParse, pTab, 0, regNewRowid); + sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey); } /* Insert the new index entries and the new record. */ - sqlite3CompleteInsertion(pParse, pTab, iCur, regNewRowid, aRegIdx, 1, 0, 0); + sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, + regNewRowid, aRegIdx, 1, 0, 0); /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to ** handle rows (possibly in other tables) that refer via a foreign key ** to the row just updated. */ if( hasFK ){ - sqlite3FkActions(pParse, pTab, pChanges, regOldRowid); + sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey); } } @@ -96761,21 +111523,29 @@ SQLITE_PRIVATE void sqlite3Update( } sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, - TRIGGER_AFTER, pTab, regOldRowid, onError, addr); + TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); /* Repeat the above with the next record to be updated, until ** all record selected by the WHERE clause have been updated. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); - sqlite3VdbeJumpHere(v, addr); + if( okOnePass ){ + /* Nothing to do at end-of-loop for a single-pass */ + }else if( pPk ){ + sqlite3VdbeResolveLabel(v, labelContinue); + sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); + } + sqlite3VdbeResolveLabel(v, labelBreak); /* Close all tables */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - if( openAll || aRegIdx[i]>0 ){ - sqlite3VdbeAddOp2(v, OP_Close, iCur+i+1, 0); + assert( aRegIdx ); + if( aToOpen[i+1] ){ + sqlite3VdbeAddOp2(v, OP_Close, iIdxCur+i, 0); } } - sqlite3VdbeAddOp2(v, OP_Close, iCur, 0); + if( iDataCurnMem; pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); + addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); for(i=0; inCol; i++){ @@ -96902,7 +111671,7 @@ static void updateVirtualTable( sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr); sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); @@ -96958,7 +111727,7 @@ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ return sqlite3_errcode(db); } VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW ); + assert( rc!=SQLITE_ROW || (db->flags&SQLITE_CountRows) ); return vacuumFinalize(db, pStmt, pzErrMsg); } @@ -96985,19 +111754,40 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ } /* -** The non-standard VACUUM command is used to clean up the database, +** The VACUUM command is used to clean up the database, ** collapse free space, etc. It is modelled after the VACUUM command -** in PostgreSQL. +** in PostgreSQL. The VACUUM command works as follows: ** -** In version 1.0.x of SQLite, the VACUUM command would call -** gdbm_reorganize() on all the database tables. But beginning -** with 2.0.0, SQLite no longer uses GDBM so this command has -** become a no-op. +** (1) Create a new transient database file +** (2) Copy all content from the database being vacuumed into +** the new transient database file +** (3) Copy content from the transient database back into the +** original database. +** +** The transient database requires temporary disk space approximately +** equal to the size of the original database. The copy operation of +** step (3) requires additional temporary disk space approximately equal +** to the size of the original database for the rollback journal. +** Hence, temporary disk space that is approximately 2x the size of the +** original database is required. Every page of the database is written +** approximately 3 times: Once for step (2) and twice for step (3). +** Two writes per page are required in step (3) because the original +** database content must be written into the rollback journal prior to +** overwriting the database with the vacuumed content. +** +** Only 1x temporary space and only 1x writes would be required if +** the copy of step (3) were replace by deleting the original database +** and renaming the transient database as the original. But that will +** not work if other processes are attached to the original database. +** And a power loss in between deleting the original and renaming the +** transient would cause the database file to appear to be deleted +** following reboot. */ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ Vdbe *v = sqlite3GetVdbe(pParse); if( v ){ sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); + sqlite3VdbeUsesBtree(v, 0); } return; } @@ -97023,7 +111813,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); return SQLITE_ERROR; } - if( db->activeVdbeCnt>1 ){ + if( db->nVdbeActive>1 ){ sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); return SQLITE_ERROR; } @@ -97089,6 +111879,18 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ } #endif + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Begin a transaction and take an exclusive lock on the main database + ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, + ** to ensure that we do not try to change the page-size on a WAL database. + */ + rc = execSql(db, pzErrMsg, "BEGIN;"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = sqlite3BtreeBeginTrans(pMain, 2); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + /* Do not attempt to change the page size for a WAL database */ if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) ==PAGER_JOURNALMODE_WAL ){ @@ -97102,27 +111904,19 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ rc = SQLITE_NOMEM; goto end_of_vacuum; } - rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); - if( rc!=SQLITE_OK ){ - goto end_of_vacuum; - } #ifndef SQLITE_OMIT_AUTOVACUUM sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : sqlite3BtreeGetAutoVacuum(pMain)); #endif - /* Begin a transaction */ - rc = execSql(db, pzErrMsg, "BEGIN EXCLUSIVE;"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ rc = execExecSql(db, pzErrMsg, "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" - " AND rootpage>0" + " AND coalesce(rootpage,1)>0" ); if( rc!=SQLITE_OK ) goto end_of_vacuum; rc = execExecSql(db, pzErrMsg, @@ -97143,7 +111937,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ "|| ' SELECT * FROM main.' || quote(name) || ';'" "FROM main.sqlite_master " "WHERE type = 'table' AND name!='sqlite_sequence' " - " AND rootpage>0" + " AND coalesce(rootpage,1)>0" ); if( rc!=SQLITE_OK ) goto end_of_vacuum; @@ -97176,13 +111970,11 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ); if( rc ) goto end_of_vacuum; - /* At this point, unless the main db was completely empty, there is now a - ** transaction open on the vacuum database, but not on the main database. - ** Open a btree level transaction on the main database. This allows a - ** call to sqlite3BtreeCopyFile(). The main database btree level - ** transaction is then committed, so the SQL level never knows it was - ** opened for writing. This way, the SQL transaction used to create the - ** temporary database never needs to be committed. + /* At this point, there is a write transaction open on both the + ** vacuum database and the main database. Assuming no error occurs, + ** both transactions are closed by this block - the main database + ** transaction by sqlite3BtreeCopyFile() and the other by an explicit + ** call to sqlite3BtreeCommit(). */ { u32 meta; @@ -97199,6 +111991,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); @@ -97250,7 +112043,7 @@ end_of_vacuum: /* This both clears the schemas and reduces the size of the db->aDb[] ** array. */ - sqlite3ResetInternalSchema(db, -1); + sqlite3ResetAllSchemasOfConnection(db); return rc; } @@ -97282,8 +112075,8 @@ end_of_vacuum: ** are invoked only from within xCreate and xConnect methods. */ struct VtabCtx { - Table *pTab; - VTable *pVTable; + VTable *pVTable; /* The virtual table being constructed */ + Table *pTab; /* The Table object to which the virtual table belongs */ }; /* @@ -97298,33 +112091,35 @@ static int createModule( void *pAux, /* Context pointer for xCreate/xConnect */ void (*xDestroy)(void *) /* Module destructor function */ ){ - int rc, nName; - Module *pMod; + int rc = SQLITE_OK; + int nName; sqlite3_mutex_enter(db->mutex); nName = sqlite3Strlen30(zName); - pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); - if( pMod ){ - Module *pDel; - char *zCopy = (char *)(&pMod[1]); - memcpy(zCopy, zName, nName+1); - pMod->zName = zCopy; - pMod->pModule = pModule; - pMod->pAux = pAux; - pMod->xDestroy = xDestroy; - pDel = (Module *)sqlite3HashInsert(&db->aModule, zCopy, nName, (void*)pMod); - if( pDel && pDel->xDestroy ){ - sqlite3ResetInternalSchema(db, -1); - pDel->xDestroy(pDel->pAux); + if( sqlite3HashFind(&db->aModule, zName) ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + Module *pMod; + pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); + if( pMod ){ + Module *pDel; + char *zCopy = (char *)(&pMod[1]); + memcpy(zCopy, zName, nName+1); + pMod->zName = zCopy; + pMod->pModule = pModule; + pMod->pAux = pAux; + pMod->xDestroy = xDestroy; + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); + assert( pDel==0 || pDel==pMod ); + if( pDel ){ + db->mallocFailed = 1; + sqlite3DbFree(db, pDel); + } } - sqlite3DbFree(db, pDel); - if( pDel==pMod ){ - db->mallocFailed = 1; - } - }else if( xDestroy ){ - xDestroy(pAux); } - rc = sqlite3ApiExit(db, SQLITE_OK); + rc = sqlite3ApiExit(db, rc); + if( rc!=SQLITE_OK && xDestroy ) xDestroy(pAux); + sqlite3_mutex_leave(db->mutex); return rc; } @@ -97389,7 +112184,7 @@ SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ assert( db ); assert( pVTab->nRef>0 ); - assert( sqlite3SafetyCheckOk(db) ); + assert( db->magic==SQLITE_MAGIC_OPEN || db->magic==SQLITE_MAGIC_ZOMBIE ); pVTab->nRef--; if( pVTab->nRef==0 ){ @@ -97440,6 +112235,31 @@ static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){ return pRet; } +/* +** Table *p is a virtual table. This function removes the VTable object +** for table *p associated with database connection db from the linked +** list in p->pVTab. It also decrements the VTable ref count. This is +** used when closing database connection db to free all of its VTable +** objects without disturbing the rest of the Schema object (which may +** be being used by other shared-cache connections). +*/ +SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p){ + VTable **ppVTab; + + assert( IsVirtual(p) ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3_mutex_held(db->mutex) ); + + for(ppVTab=&p->pVTable; *ppVTab; ppVTab=&(*ppVTab)->pNext){ + if( (*ppVTab)->db==db ){ + VTable *pVTab = *ppVTab; + *ppVTab = pVTab->pNext; + sqlite3VtabUnlock(pVTab); + break; + } + } +} + /* ** Disconnect all the virtual table objects in the sqlite3.pDisconnect list. @@ -97497,7 +112317,7 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ if( p->azModuleArg ){ int i; for(i=0; inModuleArg; i++){ - sqlite3DbFree(db, p->azModuleArg[i]); + if( i!=1 ) sqlite3DbFree(db, p->azModuleArg[i]); } sqlite3DbFree(db, p->azModuleArg); } @@ -97538,13 +112358,14 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( Parse *pParse, /* Parsing context */ Token *pName1, /* Name of new table, or database name */ Token *pName2, /* Name of new table or NULL */ - Token *pModuleName /* Name of the module for the virtual table */ + Token *pModuleName, /* Name of the module for the virtual table */ + int ifNotExists /* No error if the table already exists */ ){ int iDb; /* The database the table is being created in */ Table *pTable; /* The new virtual table */ sqlite3 *db; /* Database connection */ - sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, 0); + sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, ifNotExists); pTable = pParse->pNewTable; if( pTable==0 ) return; assert( 0==pTable->pIndex ); @@ -97556,7 +112377,7 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( pTable->tabFlags |= TF_Virtual; pTable->nModuleArg = 0; addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, db->aDb[iDb].zName)); + addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); @@ -97579,7 +112400,7 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( ** virtual table currently under construction in pParse->pTable. */ static void addArgumentToVtab(Parse *pParse){ - if( pParse->sArg.z && ALWAYS(pParse->pNewTable) ){ + if( pParse->sArg.z && pParse->pNewTable ){ const char *z = (const char*)pParse->sArg.z; int n = pParse->sArg.n; sqlite3 *db = pParse->db; @@ -97657,9 +112478,8 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ Table *pOld; Schema *pSchema = pTab->pSchema; const char *zName = pTab->zName; - int nName = sqlite3Strlen30(zName); assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, zName, nName, pTab); + pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ db->mallocFailed = 1; assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ @@ -97706,13 +112526,14 @@ static int vtabCallConstructor( int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ - VtabCtx sCtx; + VtabCtx sCtx, *pPriorCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; int nArg = pTab->nModuleArg; char *zErr = 0; char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + int iDb; if( !zModuleName ){ return SQLITE_NOMEM; @@ -97726,14 +112547,18 @@ static int vtabCallConstructor( pVTable->db = db; pVTable->pMod = pMod; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pTab->azModuleArg[1] = db->aDb[iDb].zName; + /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); assert( xConstruct ); sCtx.pTab = pTab; sCtx.pVTable = pVTable; + pPriorCtx = db->pVtabCtx; db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - db->pVtabCtx = 0; + db->pVtabCtx = pPriorCtx; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; if( SQLITE_OK!=rc ){ @@ -97747,6 +112572,7 @@ static int vtabCallConstructor( }else if( ALWAYS(pVTable->pVtab) ){ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ + memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; if( sCtx.pTab ){ @@ -97759,7 +112585,7 @@ static int vtabCallConstructor( /* If everything went according to plan, link the new VTable structure ** into the linked list headed by pTab->pVTable. Then loop through the ** columns of the table to see if any of them contain the token "hidden". - ** If so, set the Column.isHidden flag and remove the token from + ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from ** the type string. */ pVTable->pNext = pTab->pVTable; pTab->pVTable = pVTable; @@ -97790,7 +112616,7 @@ static int vtabCallConstructor( assert(zType[i-1]==' '); zType[i-1] = '\0'; } - pTab->aCol[iCol].isHidden = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_HIDDEN; } } } @@ -97820,7 +112646,7 @@ SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); if( !pMod ){ const char *zModule = pTab->azModuleArg[0]; @@ -97888,7 +112714,7 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); /* If the module has been registered and includes a Create method, ** invoke it now. If the module has not been registered, return an @@ -97927,7 +112753,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3_mutex_enter(db->mutex); if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ - sqlite3Error(db, SQLITE_MISUSE, 0); + sqlite3Error(db, SQLITE_MISUSE); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } @@ -97955,7 +112781,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ } db->pVtabCtx->pTab = 0; }else{ - sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } @@ -97965,6 +112791,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3VdbeFinalize(pParse->pVdbe); } sqlite3DeleteTable(db, pParse->pNewTable); + sqlite3ParserReset(pParse); sqlite3StackFree(db, pParse); } @@ -98037,10 +112864,9 @@ static void callFinaliser(sqlite3 *db, int offset){ ** array. Return the error code for the first error that occurs, or ** SQLITE_OK if all xSync operations are successful. ** -** Set *pzErrmsg to point to a buffer that should be released using -** sqlite3DbFree() containing an error message, if one is available. +** If an error message is available, leave it in p->zErrMsg. */ -SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ +SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, Vdbe *p){ int i; int rc = SQLITE_OK; VTable **aVTrans = db->aVTrans; @@ -98051,9 +112877,7 @@ SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ sqlite3_vtab *pVtab = aVTrans[i]->pVtab; if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ rc = x(pVtab); - sqlite3DbFree(db, *pzErrmsg); - *pzErrmsg = sqlite3DbStrDup(db, pVtab->zErrMsg); - sqlite3_free(pVtab->zErrMsg); + sqlite3VtabImportErrmsg(p, pVtab); } } db->aVTrans = aVTrans; @@ -98151,7 +112975,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ for(i=0; rc==SQLITE_OK && inVTrans; i++){ VTable *pVTab = db->aVTrans[i]; const sqlite3_module *pMod = pVTab->pMod->pModule; - if( pMod->iVersion>=2 ){ + if( pVTab->pVtab && pMod->iVersion>=2 ){ int (*xMethod)(sqlite3_vtab *, int); switch( op ){ case SAVEPOINT_BEGIN: @@ -98166,7 +112990,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ break; } if( xMethod && pVTab->iSavepoint>iSavepoint ){ - rc = xMethod(db->aVTrans[i]->pVtab, iSavepoint); + rc = xMethod(pVTab->pVtab, iSavepoint); } } } @@ -98243,7 +113067,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); pNew->xFunc = xFunc; pNew->pUserData = pArg; - pNew->flags |= SQLITE_FUNC_EPHEM; + pNew->funcFlags |= SQLITE_FUNC_EPHEM; return pNew; } @@ -98318,7 +113142,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ } va_end(ap); - if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + if( rc!=SQLITE_OK ) sqlite3Error(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -98345,27 +113169,197 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** so is applicable. Because this module is responsible for selecting ** indices, you might also think of this module as the "query optimizer". */ - +/************** Include whereInt.h in the middle of where.c ******************/ +/************** Begin file whereInt.h ****************************************/ +/* +** 2013-11-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains structure and macro definitions for the query +** planner logic in "where.c". These definitions are broken out into +** a separate source file for easier editing. +*/ /* ** Trace output macros */ #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3WhereTrace = 0; +/***/ int sqlite3WhereTrace = 0; #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# define WHERETRACE(X) if(sqlite3WhereTrace) sqlite3DebugPrintf X +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +# define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X +# define WHERETRACE_ENABLED 1 #else -# define WHERETRACE(X) +# define WHERETRACE(K,X) #endif -/* Forward reference +/* Forward references */ typedef struct WhereClause WhereClause; typedef struct WhereMaskSet WhereMaskSet; typedef struct WhereOrInfo WhereOrInfo; typedef struct WhereAndInfo WhereAndInfo; -typedef struct WhereCost WhereCost; +typedef struct WhereLevel WhereLevel; +typedef struct WhereLoop WhereLoop; +typedef struct WherePath WherePath; +typedef struct WhereTerm WhereTerm; +typedef struct WhereLoopBuilder WhereLoopBuilder; +typedef struct WhereScan WhereScan; +typedef struct WhereOrCost WhereOrCost; +typedef struct WhereOrSet WhereOrSet; + +/* +** This object contains information needed to implement a single nested +** loop in WHERE clause. +** +** Contrast this object with WhereLoop. This object describes the +** implementation of the loop. WhereLoop describes the algorithm. +** This object contains a pointer to the WhereLoop algorithm as one of +** its elements. +** +** The WhereInfo object contains a single instance of this object for +** each term in the FROM clause (which is to say, for each of the +** nested loops as implemented). The order of WhereLevel objects determines +** the loop nested order, with WhereInfo.a[0] being the outer loop and +** WhereInfo.a[WhereInfo.nLevel-1] being the inner loop. +*/ +struct WhereLevel { + int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ + int iTabCur; /* The VDBE cursor used to access the table */ + int iIdxCur; /* The VDBE cursor used to access pIdx */ + int addrBrk; /* Jump here to break out of the loop */ + int addrNxt; /* Jump here to start the next IN combination */ + int addrSkip; /* Jump here for next iteration of skip-scan */ + int addrCont; /* Jump here to continue with the next loop cycle */ + int addrFirst; /* First instruction of interior of the loop */ + int addrBody; /* Beginning of the body of this loop */ + u8 iFrom; /* Which entry in the FROM clause */ + u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ + int p1, p2; /* Operands of the opcode used to ends the loop */ + union { /* Information that depends on pWLoop->wsFlags */ + struct { + int nIn; /* Number of entries in aInLoop[] */ + struct InLoop { + int iCur; /* The VDBE cursor used by this IN operator */ + int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ + } *aInLoop; /* Information about each nested IN operator */ + } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ + Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ + } u; + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ + Bitmask notReady; /* FROM entries not usable at this level */ +}; + +/* +** Each instance of this object represents an algorithm for evaluating one +** term of a join. Every term of the FROM clause will have at least +** one corresponding WhereLoop object (unless INDEXED BY constraints +** prevent a query solution - which is an error) and many terms of the +** FROM clause will have multiple WhereLoop objects, each describing a +** potential way of implementing that FROM-clause term, together with +** dependencies and cost estimates for using the chosen algorithm. +** +** Query planning consists of building up a collection of these WhereLoop +** objects, then computing a particular sequence of WhereLoop objects, with +** one WhereLoop object per FROM clause term, that satisfy all dependencies +** and that minimize the overall cost. +*/ +struct WhereLoop { + Bitmask prereq; /* Bitmask of other loops that must run first */ + Bitmask maskSelf; /* Bitmask identifying table iTab */ +#ifdef SQLITE_DEBUG + char cId; /* Symbolic ID of this loop for debugging use */ +#endif + u8 iTab; /* Position in FROM clause of table for this loop */ + u8 iSortIdx; /* Sorting index number. 0==None */ + LogEst rSetup; /* One-time setup cost (ex: create transient index) */ + LogEst rRun; /* Cost of running each loop */ + LogEst nOut; /* Estimated number of output rows */ + union { + struct { /* Information for internal btree tables */ + u16 nEq; /* Number of equality constraints */ + u16 nSkip; /* Number of initial index columns to skip */ + Index *pIndex; /* Index used, or NULL */ + } btree; + struct { /* Information for virtual tables */ + int idxNum; /* Index number */ + u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + i8 isOrdered; /* True if satisfies ORDER BY */ + u16 omitMask; /* Terms that may be omitted */ + char *idxStr; /* Index identifier string */ + } vtab; + } u; + u32 wsFlags; /* WHERE_* flags describing the plan */ + u16 nLTerm; /* Number of entries in aLTerm[] */ + /**** whereLoopXfer() copies fields above ***********************/ +# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) + u16 nLSlot; /* Number of slots allocated for aLTerm[] */ + WhereTerm **aLTerm; /* WhereTerms used */ + WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ + WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ +}; + +/* This object holds the prerequisites and the cost of running a +** subquery on one operand of an OR operator in the WHERE clause. +** See WhereOrSet for additional information +*/ +struct WhereOrCost { + Bitmask prereq; /* Prerequisites */ + LogEst rRun; /* Cost of running this subquery */ + LogEst nOut; /* Number of outputs for this subquery */ +}; + +/* The WhereOrSet object holds a set of possible WhereOrCosts that +** correspond to the subquery(s) of OR-clause processing. Only the +** best N_OR_COST elements are retained. +*/ +#define N_OR_COST 3 +struct WhereOrSet { + u16 n; /* Number of valid a[] entries */ + WhereOrCost a[N_OR_COST]; /* Set of best costs */ +}; + + +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + +/* +** Each instance of this object holds a sequence of WhereLoop objects +** that implement some or all of a query plan. +** +** Think of each WhereLoop object as a node in a graph with arcs +** showing dependencies and costs for travelling between nodes. (That is +** not a completely accurate description because WhereLoop costs are a +** vector, not a scalar, and because dependencies are many-to-one, not +** one-to-one as are graph nodes. But it is a useful visualization aid.) +** Then a WherePath object is a path through the graph that visits some +** or all of the WhereLoop objects once. +** +** The "solver" works by creating the N best WherePath objects of length +** 1. Then using those as a basis to compute the N best WherePath objects +** of length 2. And so forth until the length of WherePaths equals the +** number of nodes in the FROM clause. The best (lowest cost) WherePath +** at the end is the chosen query plan. +*/ +struct WherePath { + Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ + Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ + LogEst nRow; /* Estimated number of rows generated by this path */ + LogEst rCost; /* Total cost of this path */ + LogEst rUnsorted; /* Total cost of this path ignoring sorting costs */ + i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ + WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ +}; /* ** The query generator uses an array of instances of this structure to @@ -98393,9 +113387,9 @@ typedef struct WhereCost WhereCost; ** ** (t1.X ) OR (t1.Y ) OR .... ** -** In this second case, wtFlag as the TERM_ORINFO set and eOperator==WO_OR +** In this second case, wtFlag has the TERM_ORINFO bit set and eOperator==WO_OR ** and the WhereTerm.u.pOrInfo field points to auxiliary information that -** is collected about the +** is collected about the OR clause. ** ** If a term in the WHERE clause does not match either of the two previous ** categories, then eOperator==0. The WhereTerm.pExpr field is still set @@ -98418,16 +113412,16 @@ typedef struct WhereCost WhereCost; ** in prereqRight and prereqAll. The default is 64 bits, hence SQLite ** is only able to process joins with 64 or fewer tables. */ -typedef struct WhereTerm WhereTerm; struct WhereTerm { Expr *pExpr; /* Pointer to the subexpression that is this term */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X " */ union { int leftColumn; /* Column number of X in "X " */ - WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ - WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; + LogEst truthProb; /* Probability of truth for this expression */ u16 eOperator; /* A WO_xx value describing */ u8 wtFlags; /* TERM_xxx bit flags. See below */ u8 nChild; /* Number of children that must disable us */ @@ -98446,20 +113440,43 @@ struct WhereTerm { #define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ #define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ #define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 # define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ #else -# define TERM_VNULL 0x00 /* Disabled if not using stat2 */ +# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ #endif +/* +** An instance of the WhereScan object is used as an iterator for locating +** terms in the WHERE clause that are useful to the query planner. +*/ +struct WhereScan { + WhereClause *pOrigWC; /* Original, innermost WhereClause */ + WhereClause *pWC; /* WhereClause currently being scanned */ + char *zCollName; /* Required collating sequence, if not NULL */ + char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char nEquiv; /* Number of entries in aEquiv[] */ + unsigned char iEquiv; /* Next unused slot in aEquiv[] */ + u32 opMask; /* Acceptable operators */ + int k; /* Resume scanning at this->pWC->a[this->k] */ + int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ +}; + /* ** An instance of the following structure holds all information about a ** WHERE clause. Mostly this is a container for one or more WhereTerms. +** +** Explanation of pOuter: For a WHERE clause of the form +** +** a AND ((b AND c) OR (d AND e)) AND f +** +** There are separate WhereClause objects for the whole clause and for +** the subclauses "(b AND c)" and "(d AND e)". The pOuter field of the +** subclauses points to the WhereClause object for the whole clause. */ struct WhereClause { - Parse *pParse; /* The parser context */ - WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ - Bitmask vmask; /* Bitmask identifying virtual table cursors */ + WhereInfo *pWInfo; /* WHERE clause processing context */ + WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ @@ -98520,19 +113537,61 @@ struct WhereMaskSet { }; /* -** A WhereCost object records a lookup strategy and the estimated -** cost of pursuing that strategy. +** This object is a convenience wrapper holding all information needed +** to construct WhereLoop objects for a particular query. */ -struct WhereCost { - WherePlan plan; /* The lookup strategy */ - double rCost; /* Overall cost of pursuing this search strategy */ - Bitmask used; /* Bitmask of cursors used by this plan */ +struct WhereLoopBuilder { + WhereInfo *pWInfo; /* Information about this WHERE */ + WhereClause *pWC; /* WHERE clause terms */ + ExprList *pOrderBy; /* ORDER BY clause */ + WhereLoop *pNew; /* Template WhereLoop */ + WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + UnpackedRecord *pRec; /* Probe for stat4 (if required) */ + int nRecValid; /* Number of valid fields currently in pRec */ +#endif }; /* -** Bitmasks for the operators that indices are able to exploit. An +** The WHERE clause processing routine has two halves. The +** first part does the start of the WHERE loop and the second +** half does the tail of the WHERE loop. An instance of +** this structure is returned by the first half and passed +** into the second half to give some continuity. +** +** An instance of this object holds the complete state of the query +** planner. +*/ +struct WhereInfo { + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pResultSet; /* Result set. DISTINCT operates on these */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + LogEst nRowOut; /* Estimated number of output rows */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u8 sorted; /* True if really sorted (not just grouped) */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + u8 nLevel; /* Number of nested loop */ + int iTop; /* The very beginning of the WHERE loop */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ +}; + +/* +** Bitmasks for the operators on WhereTerm objects. These are all +** operators that are of interest to the query planner. An ** OR-ed combination of these values can be used when searching for -** terms in the where clause. +** particular WhereTerms within a WhereClause. */ #define WO_IN 0x001 #define WO_EQ 0x002 @@ -98544,58 +113603,159 @@ struct WhereCost { #define WO_ISNULL 0x080 #define WO_OR 0x100 /* Two or more OR-connected terms */ #define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ #define WO_NOOP 0x800 /* This term does not restrict search space */ #define WO_ALL 0xfff /* Mask of all possible WO_* values */ #define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ /* -** Value for wsFlags returned by bestIndex() and stored in -** WhereLevel.wsFlags. These flags determine which search -** strategies are appropriate. -** -** The least significant 12 bits is reserved as a mask for WO_ values above. -** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL. -** But if the table is the right table of a left join, WhereLevel.wsFlags -** is set to WO_IN|WO_EQ. The WhereLevel.wsFlags field can then be used as -** the "op" parameter to findTerm when we are resolving equality constraints. -** ISNULL constraints will then not be used on the right table of a left -** join. Tickets #2177 and #2189. +** These are definitions of bits in the WhereLoop.wsFlags field. +** The particular combination of bits in each WhereLoop help to +** determine the algorithm that WhereLoop represents. */ -#define WHERE_ROWID_EQ 0x00001000 /* rowid=EXPR or rowid IN (...) */ -#define WHERE_ROWID_RANGE 0x00002000 /* rowidEXPR */ -#define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ -#define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ -#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ -#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ -#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ -#define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ -#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xEXPR */ +#define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ +#define WHERE_CONSTRAINT 0x0000000f /* Any of the WHERE_COLUMN_xxx values */ +#define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and xnRowOut); +} + +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} + +/* +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. +*/ +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat; +} + +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return TRUE if an UPDATE or DELETE statement can operate directly on +** the rowids returned by a WHERE clause. Return FALSE if doing an +** UPDATE or DELETE might change subsequent WHERE clause results. +** +** If the ONEPASS optimization is used (if this routine returns true) +** then also write the indices of open cursors used by ONEPASS +** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data +** table and iaCur[1] gets the cursor used by an auxiliary index. +** Either value may be -1, indicating that cursor is not used. +** Any cursors returned will have been opened for writing. +** +** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is +** unable to use the ONEPASS optimization. +*/ +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ + memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); + return pWInfo->okOnePass; +} + +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); +} + +/* +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. +*/ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ +){ + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; + } + if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; + } + } + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; + } + if( p->rRun<=rRun ) return 0; + } +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; +} /* ** Initialize a preallocated WhereClause structure. */ static void whereClauseInit( WhereClause *pWC, /* The WhereClause to be initialized */ - Parse *pParse, /* The parsing context */ - WhereMaskSet *pMaskSet /* Mapping from table cursor numbers to bitmasks */ + WhereInfo *pWInfo /* The WHERE processing context */ ){ - pWC->pParse = pParse; - pWC->pMaskSet = pMaskSet; + pWC->pWInfo = pWInfo; + pWC->pOuter = 0; pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; - pWC->vmask = 0; } /* Forward reference */ @@ -98624,7 +113784,7 @@ static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ static void whereClauseClear(WhereClause *pWC){ int i; WhereTerm *a; - sqlite3 *db = pWC->pParse->db; + sqlite3 *db = pWC->pWInfo->pParse->db; for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ if( a->wtFlags & TERM_DYNAMIC ){ sqlite3ExprDelete(db, a->pExpr); @@ -98662,10 +113822,10 @@ static void whereClauseClear(WhereClause *pWC){ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ WhereTerm *pTerm; int idx; - testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ + testcase( wtFlags & TERM_VIRTUAL ); if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; - sqlite3 *db = pWC->pParse->db; + sqlite3 *db = pWC->pWInfo->pParse->db; pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ @@ -98681,7 +113841,12 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); } pTerm = &pWC->a[idx = pWC->nTerm++]; - pTerm->pExpr = p; + if( p && ExprHasProperty(p, EP_Unlikely) ){ + pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + }else{ + pTerm->truthProb = 1; + } + pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; @@ -98705,8 +113870,8 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ ** the WhereClause.a[] array. The slot[] array grows as needed to contain ** all terms of the WHERE clause. */ -static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ - pWC->op = (u8)op; +static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + pWC->op = op; if( pExpr==0 ) return; if( pExpr->op!=op ){ whereClauseInsert(pWC, pExpr, 0); @@ -98717,9 +113882,9 @@ static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ } /* -** Initialize an expression mask set (a WhereMaskSet object) +** Initialize a WhereMaskSet object */ -#define initMaskSet(P) memset(P, 0, sizeof(*P)) +#define initMaskSet(P) (P)->n=0 /* ** Return the bitmask for the given cursor number. Return 0 if @@ -98730,7 +113895,7 @@ static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); for(i=0; in; i++){ if( pMaskSet->ix[i]==iCursor ){ - return ((Bitmask)1)<pSrc; mask |= exprListTableUsage(pMaskSet, pS->pEList); mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); mask |= exprTableUsage(pMaskSet, pS->pWhere); mask |= exprTableUsage(pMaskSet, pS->pHaving); + if( ALWAYS(pSrc!=0) ){ + int i; + for(i=0; inSrc; i++){ + mask |= exprSelectTableUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= exprTableUsage(pMaskSet, pSrc->a[i].pOn); + } + } pS = pS->pPrior; } return mask; @@ -98807,14 +113971,7 @@ static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ /* ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", and "IN". -** -** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be -** of one of the following forms: column = expression column > expression -** column >= expression column < expression column <= expression -** expression = column expression > column expression >= column -** expression < column expression <= column column IN -** (expression-list) column IN (subquery) column IS NULL +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" */ static int allowedOp(int op){ assert( TK_GT>TK_EQ && TK_GT=TK_EQ && op<=TK_GE) || op==TK_ISNULL; } -/* -** Swap two objects of type TYPE. -*/ -#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} - /* ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". ** -** If a collation sequence is associated with either the left or right -** side of the comparison, it remains associated with the same side after -** the commutation. So "Y collate NOCASE op X" becomes -** "X collate NOCASE op Y". This is because any collation sequence on +** If left/right precedence rules come into play when determining the +** collating sequence, then COLLATE operators are adjusted to ensure +** that the collating sequence does not change. For example: +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on ** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_ExpCollate flag +** attached to the right. For the same reason the EP_Collate flag ** is not commuted. */ static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_ExpCollate); - u16 expLeft = (pExpr->pLeft->flags & EP_ExpCollate); + u16 expRight = (pExpr->pRight->flags & EP_Collate); + u16 expLeft = (pExpr->pLeft->flags & EP_Collate); assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - pExpr->pRight->pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); - pExpr->pLeft->pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl); - pExpr->pRight->flags = (pExpr->pRight->flags & ~EP_ExpCollate) | expLeft; - pExpr->pLeft->flags = (pExpr->pLeft->flags & ~EP_ExpCollate) | expRight; + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_Collate flag from Y. */ + pExpr->pRight->flags &= ~EP_Collate; + }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_Collate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_Collate; + } + } SWAP(Expr*,pExpr->pRight,pExpr->pLeft); if( pExpr->op>=TK_GT ){ assert( TK_LT==TK_GT+2 ); @@ -98885,11 +114045,155 @@ static u16 operatorMask(int op){ return c; } +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + int iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ + + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aEquiv[pScan->iEquiv-2]; + iColumn = pScan->aEquiv[pScan->iEquiv-1]; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaEquiv) + ){ + int j; + pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; jnEquiv; j+=2){ + if( pScan->aEquiv[j]==pX->iTable + && pScan->aEquiv[j+1]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aEquiv[j] = pX->iTable; + pScan->aEquiv[j+1] = pX->iColumn; + pScan->nEquiv += 2; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & WO_EQ)!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aEquiv[0] + && pX->iColumn==pScan->aEquiv[1] + ){ + continue; + } + pScan->k = k+1; + return pTerm; + } + } + } + pScan->pWC = pScan->pWC->pOuter; + k = 0; + } + pScan->pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv += 2; + } + return 0; +} + +/* +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. The must be one of the operators described by opMask. +** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. +*/ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ +){ + int j; + + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + if( pIdx && iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ + if( NEVER(j>pIdx->nColumn) ) return 0; + } + pScan->zCollName = pIdx->azColl[j]; + }else{ + pScan->idxaff = 0; + pScan->zCollName = 0; + } + pScan->opMask = opMask; + pScan->k = 0; + pScan->aEquiv[0] = iCur; + pScan->aEquiv[1] = iColumn; + pScan->nEquiv = 2; + pScan->iEquiv = 2; + return whereScanNext(pScan); +} + /* ** Search for a term in the WHERE clause that is of the form "X " ** where X is a reference to the iColumn of table iCur and is one of ** the WO_xx operator codes specified by the op parameter. ** Return a pointer to the term. Return 0 if not found. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aEquiv[] array holds X and all its equivalents, with each SQL variable +** taking up two slots in aEquiv[]. The first slot is for the cursor number +** and the second is for the column number. There are 22 slots in aEquiv[] +** so that means we can look for X plus up to 10 other equivalent values. +** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 +** and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. */ static WhereTerm *findTerm( WhereClause *pWC, /* The WHERE clause to be searched */ @@ -98899,43 +114203,21 @@ static WhereTerm *findTerm( u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - WhereTerm *pTerm; - int k; - assert( iCur>=0 ); - op &= WO_ALL; - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( pIdx && pTerm->eOperator!=WO_ISNULL ){ - Expr *pX = pTerm->pExpr; - CollSeq *pColl; - char idxaff; - int j; - Parse *pParse = pWC->pParse; + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; - idxaff = pIdx->pTable->aCol[iColumn].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - assert(pColl || pParse->nErr); - - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; - } - if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ + return p; } - return pTerm; + if( pResult==0 ) pResult = p; } + p = whereScanNext(&scan); } - return 0; + return pResult; } /* Forward reference */ @@ -98943,8 +114225,6 @@ static void exprAnalyze(SrcList*, WhereClause*, int); /* ** Call exprAnalyze on all terms in a WHERE clause. -** -** */ static void exprAnalyzeAll( SrcList *pTabList, /* the FROM clause */ @@ -98990,26 +114270,26 @@ static int isLikeOrGlob( #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) + ){ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ return 0; } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - pRight = pList->a[0].pExpr; + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); op = pRight->op; - if( op==TK_REGISTER ){ - op = pRight->op2; - } if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; int iCol = pRight->iColumn; - pVal = sqlite3VdbeGetValue(pReprepare, iCol, SQLITE_AFF_NONE); + pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_NONE); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ z = (char *)sqlite3_value_text(pVal); } - sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */ + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ z = pRight->u.zToken; @@ -99027,13 +114307,13 @@ static int isLikeOrGlob( *ppPrefix = pPrefix; if( op==TK_VARIABLE ){ Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */ + sqlite3VdbeSetVarmask(v, pRight->iColumn); if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current ** value of the variable means there is no need to invoke the LIKE ** function, then no OP_Variable will be added to the program. ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To workaround them, add a dummy OP_Variable here. + ** API. To work around them, add a dummy OP_Variable here. */ int r1 = sqlite3GetTempReg(pParse); sqlite3ExprCodeTarget(pParse, pRight, r1); @@ -99087,8 +114367,10 @@ static int isMatchOfColumn( ** a join, then transfer the appropriate markings over to derived. */ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; + if( pDerived ){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; + } } #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) @@ -99118,7 +114400,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** ** CASE 1: ** -** If all subterms are of the form T.C=expr for some single column of C +** If all subterms are of the form T.C=expr for some single column of C and ** a single table T (as shown in example B above) then create a new virtual ** term that is an equivalent IN expression. In other words, if the term ** being analyzed is: @@ -99147,11 +114429,11 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** From another point of view, "indexable" means that the subterm could ** potentially be used with an index if an appropriate index exists. ** This analysis does not consider whether or not the index exists; that -** is something the bestIndex() routine will determine. This analysis -** only looks at whether subterms appropriate for indexing exist. +** is decided elsewhere. This analysis only looks at whether subterms +** appropriate for indexing exist. ** -** All examples A through E above all satisfy case 2. But if a term -** also statisfies case 1 (such as B) we know that the optimizer will +** All examples A through E above satisfy case 2. But if a term +** also satisfies case 1 (such as B) we know that the optimizer will ** always prefer case 1, so in that case we pretend that case 2 is not ** satisfied. ** @@ -99173,11 +114455,11 @@ static void exprAnalyzeOrTerm( WhereClause *pWC, /* the complete WHERE clause */ int idxTerm /* Index of the OR-term to be analyzed */ ){ - Parse *pParse = pWC->pParse; /* Parser context */ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ sqlite3 *db = pParse->db; /* Database connection */ WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ Expr *pExpr = pTerm->pExpr; /* The expression of the term */ - WhereMaskSet *pMaskSet = pWC->pMaskSet; /* Table use masks */ int i; /* Loop counters */ WhereClause *pOrWc; /* Breakup of pTerm into subterms */ WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ @@ -99196,7 +114478,7 @@ static void exprAnalyzeOrTerm( if( pOrInfo==0 ) return; pTerm->wtFlags |= TERM_ORINFO; pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWC->pParse, pMaskSet); + whereClauseInit(pOrWc, pWInfo); whereSplit(pOrWc, pExpr, TK_OR); exprAnalyzeAll(pSrc, pOrWc); if( db->mallocFailed ) return; @@ -99206,11 +114488,10 @@ static void exprAnalyzeOrTerm( ** Compute the set of tables that might satisfy cases 1 or 2. */ indexable = ~(Bitmask)0; - chngToIN = ~(pWC->vmask); + chngToIN = ~(Bitmask)0; for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ WhereAndInfo *pAndInfo; - assert( pOrTerm->eOperator==0 ); assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); @@ -99223,15 +114504,16 @@ static void exprAnalyzeOrTerm( pOrTerm->wtFlags |= TERM_ANDINFO; pOrTerm->eOperator = WO_AND; pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pParse, pMaskSet); + whereClauseInit(pAndWC, pWC->pWInfo); whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); exprAnalyzeAll(pSrc, pAndWC); + pAndWC->pOuter = pWC; testcase( db->mallocFailed ); if( !db->mallocFailed ){ for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ assert( pAndTerm->pExpr ); if( allowedOp(pAndTerm->pExpr->op) ){ - b |= getMask(pMaskSet, pAndTerm->leftCursor); + b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); } } } @@ -99242,13 +114524,13 @@ static void exprAnalyzeOrTerm( ** corresponding TERM_VIRTUAL term */ }else{ Bitmask b; - b = getMask(pMaskSet, pOrTerm->leftCursor); + b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); if( pOrTerm->wtFlags & TERM_VIRTUAL ){ WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= getMask(pMaskSet, pOther->leftCursor); + b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); } indexable &= b; - if( pOrTerm->eOperator!=WO_EQ ){ + if( (pOrTerm->eOperator & WO_EQ)==0 ){ chngToIN = 0; }else{ chngToIN &= b; @@ -99299,7 +114581,7 @@ static void exprAnalyzeOrTerm( for(j=0; j<2 && !okToChngToIN; j++){ pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); pOrTerm->wtFlags &= ~TERM_OR_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and @@ -99307,9 +114589,9 @@ static void exprAnalyzeOrTerm( assert( j==1 ); continue; } - if( (chngToIN & getMask(pMaskSet, pOrTerm->leftCursor))==0 ){ + if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceeded + ** chngToIN set but t1 is not. This term will be either preceded ** or follwed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); @@ -99325,8 +114607,8 @@ static void exprAnalyzeOrTerm( /* No candidate table+column was found. This can only occur ** on the second iteration */ assert( j==1 ); - assert( (chngToIN&(chngToIN-1))==0 ); - assert( chngToIN==getMask(pMaskSet, iCursor) ); + assert( IsPowerOfTwo(chngToIN) ); + assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); break; } testcase( j==1 ); @@ -99335,7 +114617,7 @@ static void exprAnalyzeOrTerm( ** table and column is common to every term in the OR clause */ okToChngToIN = 1; for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); if( pOrTerm->leftCursor!=iCursor ){ pOrTerm->wtFlags &= ~TERM_OR_OK; }else if( pOrTerm->u.leftColumn!=iColumn ){ @@ -99360,8 +114642,6 @@ static void exprAnalyzeOrTerm( /* At this point, okToChngToIN is true if original pTerm satisfies ** case 1. In that case, construct a new virtual term that is ** pTerm converted into an IN operator. - ** - ** EV: R-00211-15100 */ if( okToChngToIN ){ Expr *pDup; /* A transient duplicate expression */ @@ -99371,11 +114651,11 @@ static void exprAnalyzeOrTerm( for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); assert( pOrTerm->leftCursor==iCursor ); assert( pOrTerm->u.leftColumn==iColumn ); pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); - pList = sqlite3ExprListAppend(pWC->pParse, pList, pDup); + pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); pLeft = pOrTerm->pExpr->pLeft; } assert( pLeft!=0 ); @@ -99401,7 +114681,6 @@ static void exprAnalyzeOrTerm( } #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ - /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -99425,6 +114704,7 @@ static void exprAnalyze( WhereClause *pWC, /* the WHERE clause */ int idxTerm /* Index of the term to be analyzed */ ){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ WhereTerm *pTerm; /* The term to be analyzed */ WhereMaskSet *pMaskSet; /* Set of table index masks */ Expr *pExpr; /* The expression to be analyzed */ @@ -99435,15 +114715,16 @@ static void exprAnalyze( int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ int noCase = 0; /* LIKE/GLOB distinguishes case */ int op; /* Top-level operator. pExpr->op */ - Parse *pParse = pWC->pParse; /* Parsing context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ if( db->mallocFailed ){ return; } pTerm = &pWC->a[idxTerm]; - pMaskSet = pWC->pMaskSet; + pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); op = pExpr->op; if( op==TK_IN ){ @@ -99469,17 +114750,19 @@ static void exprAnalyze( pTerm->leftCursor = -1; pTerm->iParent = -1; pTerm->eOperator = 0; - if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){ - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pRight; + if( allowedOp(op) ){ + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; if( pLeft->op==TK_COLUMN ){ pTerm->leftCursor = pLeft->iTable; pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op); + pTerm->eOperator = operatorMask(op) & opMask; } if( pRight && pRight->op==TK_COLUMN ){ WhereTerm *pNew; Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite3ExprDup(db, pExpr, 0); @@ -99494,18 +114777,25 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; + if( pExpr->op==TK_EQ + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE_Transitive) + ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ pDup = pExpr; pNew = pTerm; } exprCommute(pParse, pDup); - pLeft = pDup->pLeft; + pLeft = sqlite3ExprSkipCollate(pDup->pLeft); pNew->leftCursor = pLeft->iTable; pNew->u.leftColumn = pLeft->iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; - pNew->eOperator = operatorMask(pDup->op); + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } } @@ -99537,6 +114827,7 @@ static void exprAnalyze( pNewExpr = sqlite3PExpr(pParse, ops[i], sqlite3ExprDup(db, pExpr->pLeft, 0), sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + transferJoinMarkings(pNewExpr, pExpr); idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); @@ -99578,7 +114869,7 @@ static void exprAnalyze( Expr *pNewExpr2; int idxNew1; int idxNew2; - CollSeq *pColl; /* Collating sequence to use */ + Token sCollSeqName; /* Name of collating sequence */ pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); @@ -99593,23 +114884,26 @@ static void exprAnalyze( ** inequality. To avoid this, make sure to also run the full ** LIKE on all candidate expressions by clearing the isComplete flag */ - if( c=='A'-1 ) isComplete = 0; /* EV: R-64339-08207 */ - - + if( c=='A'-1 ) isComplete = 0; c = sqlite3UpperToLower[c]; } *pC = c + 1; } - pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); + sCollSeqName.z = noCase ? "NOCASE" : "BINARY"; + sCollSeqName.n = 6; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr1, 0); + sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName), + pStr1, 0); + transferJoinMarkings(pNewExpr1, pExpr); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr2, 0); + sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName), + pStr2, 0); + transferJoinMarkings(pNewExpr2, pExpr); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); @@ -99659,8 +114953,8 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_STAT2 - /* When sqlite_stat2 histogram data is available an operator of the +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* When sqlite_stat3 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a ** virtual term of that form. @@ -99673,6 +114967,7 @@ static void exprAnalyze( if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 + && OptimizationEnabled(db, SQLITE_Stat3) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -99698,7 +114993,7 @@ static void exprAnalyze( pNewTerm->prereqAll = pTerm->prereqAll; } } -#endif /* SQLITE_ENABLE_STAT2 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. @@ -99707,185 +115002,111 @@ static void exprAnalyze( } /* -** Return TRUE if any of the expressions in pList->a[iFirst...] contain -** a reference to any table other than the iBase table. +** This function searches pList for an entry that matches the iCol-th column +** of index pIdx. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. */ -static int referencesOtherTables( - ExprList *pList, /* Search expressions in ths list */ - WhereMaskSet *pMaskSet, /* Mapping from tables to bitmaps */ - int iFirst, /* Be searching with the iFirst-th expression */ - int iBase /* Ignore references to this table */ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ ){ - Bitmask allowed = ~getMask(pMaskSet, iBase); - while( iFirstnExpr ){ - if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){ + int i; + const char *zColl = pIdx->azColl[iCol]; + + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); + if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; + } + } + } + + return -1; +} + +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. +** +** A DISTINCT list is redundant if the database contains some subset of +** columns that are unique and non-null. +*/ +static int isDistinctRedundant( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !IsUniqueIndex(pIdx) ) continue; + for(i=0; inKeyCol; i++){ + i16 iCol = pIdx->aiColumn[i]; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); + if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ + break; + } + } + } + if( i==pIdx->nKeyCol ){ + /* This index implies that the DISTINCT qualifier is redundant. */ return 1; } } + return 0; } /* -** This routine decides if pIdx can be used to satisfy the ORDER BY -** clause. If it can, it returns 1. If pIdx cannot satisfy the -** ORDER BY clause, this routine returns 0. -** -** pOrderBy is an ORDER BY clause from a SELECT statement. pTab is the -** left-most table in the FROM clause of that same SELECT statement and -** the table has a cursor number of "base". pIdx is an index on pTab. -** -** nEqCol is the number of columns of pIdx that are used as equality -** constraints. Any of these columns may be missing from the ORDER BY -** clause and the match can still be a success. -** -** All terms of the ORDER BY that match against the index must be either -** ASC or DESC. (Terms of the ORDER BY clause past the end of a UNIQUE -** index do not need to satisfy this constraint.) The *pbRev value is -** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if -** the ORDER BY clause is all ASC. +** Estimate the logarithm of the input value to base 2. */ -static int isSortingIndex( - Parse *pParse, /* Parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */ - Index *pIdx, /* The index we are testing */ - int base, /* Cursor number for the table to be sorted */ - ExprList *pOrderBy, /* The ORDER BY clause */ - int nEqCol, /* Number of index columns with == constraints */ - int wsFlags, /* Index usages flags */ - int *pbRev /* Set to 1 if ORDER BY is DESC */ -){ - int i, j; /* Loop counters */ - int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ - int nTerm; /* Number of ORDER BY terms */ - struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ - sqlite3 *db = pParse->db; - - assert( pOrderBy!=0 ); - nTerm = pOrderBy->nExpr; - assert( nTerm>0 ); - - /* Argument pIdx must either point to a 'real' named index structure, - ** or an index structure allocated on the stack by bestBtreeIndex() to - ** represent the rowid index that is part of every table. */ - assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); - - /* Match terms of the ORDER BY clause against columns of - ** the index. - ** - ** Note that indices have pIdx->nColumn regular columns plus - ** one additional column containing the rowid. The rowid column - ** of the index is also allowed to match against the ORDER BY - ** clause. - */ - for(i=j=0, pTerm=pOrderBy->a; jnColumn; i++){ - Expr *pExpr; /* The expression of the ORDER BY pTerm */ - CollSeq *pColl; /* The collating sequence of pExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - const char *zColl; /* Name of the collating sequence for i-th index term */ - - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ - /* Can not use an index sort on anything that is not a column in the - ** left-most table of the FROM clause */ - break; - } - pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } - if( pIdx->zName && inColumn ){ - iColumn = pIdx->aiColumn[i]; - if( iColumn==pIdx->pTable->iPKey ){ - iColumn = -1; - } - iSortOrder = pIdx->aSortOrder[i]; - zColl = pIdx->azColl[i]; - }else{ - iColumn = -1; - iSortOrder = 0; - zColl = pColl->zName; - } - if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ - /* Term j of the ORDER BY clause does not match column i of the index */ - if( inColumn ){ - /* Index column i is the rowid. All other terms match. */ - break; - }else{ - /* If an index column fails to match and is not constrained by == - ** then the index cannot satisfy the ORDER BY constraint. - */ - return 0; - } - } - assert( pIdx->aSortOrder!=0 || iColumn==-1 ); - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqCol ){ - if( termSortOrder!=sortOrder ){ - /* Indices can only be used if all ORDER BY terms past the - ** equality constraints are all either DESC or ASC. */ - return 0; - } - }else{ - sortOrder = termSortOrder; - } - j++; - pTerm++; - if( iColumn<0 && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* If the indexed column is the primary key and everything matches - ** so far and none of the ORDER BY terms to the right reference other - ** tables in the join, then we are assured that the index can be used - ** to sort because the primary key is unique and so none of the other - ** columns will make any difference - */ - j = nTerm; - } - } - - *pbRev = sortOrder!=0; - if( j>=nTerm ){ - /* All terms of the ORDER BY clause are covered by this index so - ** this index can be used for sorting. */ - return 1; - } - if( pIdx->onError!=OE_None && i==pIdx->nColumn - && (wsFlags & WHERE_COLUMN_NULL)==0 - && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* All terms of this index match some prefix of the ORDER BY clause - ** and the index is UNIQUE and no terms on the tail of the ORDER BY - ** clause reference other tables in a join. If this is all true then - ** the order by clause is superfluous. Not that if the matching - ** condition is IS NULL then the result is not necessarily unique - ** even on a UNIQUE index, so disallow those cases. */ - return 1; - } - return 0; -} - -/* -** Prepare a crude estimate of the logarithm of the input value. -** The results need not be exact. This is only used for estimating -** the total cost of performing operations with O(logN) or O(NlogN) -** complexity. Because N is just a guess, it is no great tragedy if -** logN is a little off. -*/ -static double estLog(double N){ - double logN = 1; - double x = 10; - while( N>x ){ - logN += 1; - x *= 10; - } - return logN; +static LogEst estLog(LogEst N){ + return N<=10 ? 0 : sqlite3LogEst(N) - 33; } /* @@ -99894,7 +115115,7 @@ static double estLog(double N){ ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines ** are no-ops. */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_DEBUG) +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ int i; if( !sqlite3WhereTrace ) return; @@ -99926,110 +115147,13 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); + sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else #define TRACE_IDX_INPUTS(A) #define TRACE_IDX_OUTPUTS(A) #endif -/* -** Required because bestIndex() is called by bestOrClauseIndex() -*/ -static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, - Bitmask, Bitmask, ExprList*, WhereCost*); - -/* -** This routine attempts to find an scanning strategy that can be used -** to optimize an 'OR' expression that is part of a WHERE clause. -** -** The table associated with FROM clause term pSrc may be either a -** regular B-Tree table or a virtual table. -*/ -static void bestOrClauseIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ - const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ - WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - - /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses - ** are used */ - if( pSrc->notIndexed || pSrc->pIndex!=0 ){ - return; - } - - /* Search the WHERE clause terms for a usable WO_OR term. */ - for(pTerm=pWC->a; pTermeOperator==WO_OR - && ((pTerm->prereqAll & ~maskSrc) & notReady)==0 - && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 - ){ - WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; - WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; - WhereTerm *pOrTerm; - int flags = WHERE_MULTI_OR; - double rTotal = 0; - double nRow = 0; - Bitmask used = 0; - - for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) - )); - if( pOrTerm->eOperator==WO_AND ){ - WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); - }else if( pOrTerm->leftCursor==iCur ){ - WhereClause tempWC; - tempWC.pParse = pWC->pParse; - tempWC.pMaskSet = pWC->pMaskSet; - tempWC.op = TK_AND; - tempWC.a = pOrTerm; - tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); - }else{ - continue; - } - rTotal += sTermCost.rCost; - nRow += sTermCost.plan.nRow; - used |= sTermCost.used; - if( rTotal>=pCost->rCost ) break; - } - - /* If there is an ORDER BY clause, increase the scan cost to account - ** for the cost of the sort. */ - if( pOrderBy!=0 ){ - WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", - rTotal, rTotal+nRow*estLog(nRow))); - rTotal += nRow*estLog(nRow); - } - - /* If the cost of scanning using this OR term for optimization is - ** less than the current cost stored in pCost, replace the contents - ** of pCost. */ - WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); - if( rTotalrCost ){ - pCost->rCost = rTotal; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = flags; - pCost->plan.u.pTerm = pTerm; - } - } - } -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ -} - #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it @@ -100043,79 +115167,15 @@ static int termCanDriveIndex( ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( pTerm->eOperator!=WO_EQ ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; return 1; } #endif -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX -/* -** If the query plan for pSrc specified in pCost is a full table scan -** and indexing is allows (if there is no NOT INDEXED clause) and it -** possible to construct a transient index that would perform better -** than a full table scan even when the cost of constructing the index -** is taken into account, then alter the query plan to use the -** transient index. -*/ -static void bestAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereCost *pCost /* Lowest cost query plan */ -){ - double nTableRow; /* Rows in the input table */ - double logN; /* log(nTableRow) */ - double costTempIdx; /* per-query cost of the transient index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - Table *pTable; /* Table tht might be indexed */ - - if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){ - /* Automatic indices are disabled at run-time */ - return; - } - if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ - /* We already have some kind of index in use for this query. */ - return; - } - if( pSrc->notIndexed ){ - /* The NOT INDEXED clause appears in the SQL. */ - return; - } - - assert( pParse->nQueryLoop >= (double)1 ); - pTable = pSrc->pTab; - nTableRow = pTable->nRowEst; - logN = estLog(nTableRow); - costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); - if( costTempIdx>=pCost->rCost ){ - /* The cost of creating the transient table would be greater than - ** doing the full table scan */ - return; - } - - /* Search for any equality comparison term */ - pWCEnd = &pWC->a[pWC->nTerm]; - for(pTerm=pWC->a; pTermrCost, costTempIdx)); - pCost->rCost = costTempIdx; - pCost->plan.nRow = logN + 1; - pCost->plan.wsFlags = WHERE_TEMP_INDEX; - pCost->used = pTerm->prereqRight; - break; - } - } -} -#else -# define bestAutomaticIndex(A,B,C,D,E) /* no-op */ -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* @@ -100130,53 +115190,61 @@ static void constructAutomaticIndex( Bitmask notReady, /* Mask of cursors that are not available */ WhereLevel *pLevel /* Write new index here */ ){ - int nColumn; /* Number of columns in the constructed index */ + int nKeyCol; /* Number of columns in the constructed index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ WhereTerm *pWCEnd; /* End of pWC->a[] */ - int nByte; /* Byte of memory needed for pIdx */ Index *pIdx; /* Object describing the transient index */ Vdbe *v; /* Prepared statement under construction */ - int regIsInit; /* Register set by initialization */ int addrInit; /* Address of the initialization bypass jump */ Table *pTable; /* The table being indexed */ - KeyInfo *pKeyinfo; /* Key information for the index */ int addrTop; /* Top of the index fill loop */ int regRecord; /* Register holding an index record */ int n; /* Column counter */ int i; /* Loop counter */ int mxBitCol; /* Maximum column in pSrc->colUsed */ CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ + char *zNotUsed; /* Extra space on the end of pIdx */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ + u8 sentWarning = 0; /* True if a warnning has been issued */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ v = pParse->pVdbe; assert( v!=0 ); - regIsInit = ++pParse->nMem; - addrInit = sqlite3VdbeAddOp1(v, OP_If, regIsInit); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regIsInit); + addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ - nColumn = 0; + nKeyCol = 0; pTable = pSrc->pTab; pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); testcase( iCol==BMS ); testcase( iCol==BMS-1 ); + if( !sentWarning ){ + sqlite3_log(SQLITE_WARNING_AUTOINDEX, + "automatic index on %s(%s)", pTable->zName, + pTable->aCol[iCol].zName); + sentWarning = 1; + } if( (idxCols & cMask)==0 ){ - nColumn++; + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + pLoop->aLTerm[nKeyCol++] = pTerm; idxCols |= cMask; } } } - assert( nColumn>0 ); - pLevel->plan.nEq = nColumn; + assert( nKeyCol>0 ); + pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_AUTO_INDEX; /* Count the number of additional columns needed to create a ** covering index. A "covering index" is an index that contains all @@ -100186,38 +115254,32 @@ static void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | (((Bitmask)1)<<(BMS-1))); + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); for(i=0; icolUsed & (((Bitmask)1)<<(BMS-1)) ){ - nColumn += pTable->nCol - BMS + 1; + if( pSrc->colUsed & MASKBIT(BMS-1) ){ + nKeyCol += pTable->nCol - BMS + 1; } - pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WO_EQ; + pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ - nByte = sizeof(Index); - nByte += nColumn*sizeof(int); /* Index.aiColumn */ - nByte += nColumn*sizeof(char*); /* Index.azColl */ - nByte += nColumn; /* Index.aSortOrder */ - pIdx = sqlite3DbMallocZero(pParse->db, nByte); + pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); if( pIdx==0 ) return; - pLevel->plan.u.pIdx = pIdx; - pIdx->azColl = (char**)&pIdx[1]; - pIdx->aiColumn = (int*)&pIdx->azColl[nColumn]; - pIdx->aSortOrder = (u8*)&pIdx->aiColumn[nColumn]; + pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; - pIdx->nColumn = nColumn; pIdx->pTable = pTable; n = 0; idxCols = 0; for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); if( (idxCols & cMask)==0 ){ Expr *pX = pTerm->pExpr; idxCols |= cMask; @@ -100228,40 +115290,42 @@ static void constructAutomaticIndex( } } } - assert( (u32)n==pLevel->plan.nEq ); + assert( (u32)n==pLoop->u.btree.nEq ); /* Add additional columns needed to make the automatic index into ** a covering index */ for(i=0; iaiColumn[n] = i; pIdx->azColl[n] = "BINARY"; n++; } } - if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){ + if( pSrc->colUsed & MASKBIT(BMS-1) ){ for(i=BMS-1; inCol; i++){ pIdx->aiColumn[n] = i; pIdx->azColl[n] = "BINARY"; n++; } } - assert( n==nColumn ); + assert( n==nKeyCol ); + pIdx->aiColumn[n] = -1; + pIdx->azColl[n] = "BINARY"; /* Create the automatic index */ - pKeyinfo = sqlite3IndexKeyinfo(pParse, pIdx); assert( pLevel->iIdxCur>=0 ); - sqlite3VdbeAddOp4(v, OP_OpenAutoindex, pLevel->iIdxCur, nColumn+1, 0, - (char*)pKeyinfo, P4_KEYINFO_HANDOFF); + pLevel->iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 1); + sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); @@ -100278,7 +115342,7 @@ static void constructAutomaticIndex( ** by passing the pointer returned by this function to sqlite3_free(). */ static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, + Parse *pParse, WhereClause *pWC, struct SrcList_item *pSrc, ExprList *pOrderBy @@ -100292,16 +115356,16 @@ static sqlite3_index_info *allocateIndexInfo( int nOrderBy; sqlite3_index_info *pIdxInfo; - WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName)); - /* Count the number of possible WHERE clause constraints referring ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; } @@ -100311,12 +115375,13 @@ static sqlite3_index_info *allocateIndexInfo( */ nOrderBy = 0; if( pOrderBy ){ - for(i=0; inExpr; i++){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; } - if( i==pOrderBy->nExpr ){ - nOrderBy = pOrderBy->nExpr; + if( i==n){ + nOrderBy = n; } } @@ -100327,7 +115392,6 @@ static sqlite3_index_info *allocateIndexInfo( + sizeof(*pIdxOrderBy)*nOrderBy ); if( pIdxInfo==0 ){ sqlite3ErrorMsg(pParse, "out of memory"); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return 0; } @@ -100347,14 +115411,19 @@ static sqlite3_index_info *allocateIndexInfo( pUsage; for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; - pIdxCons[j].op = (u8)pTerm->eOperator; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The ** following asserts verify this fact. */ @@ -100364,7 +115433,7 @@ static sqlite3_index_info *allocateIndexInfo( assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); j++; } for(i=0; izName)); TRACE_IDX_INPUTS(p); rc = pVtab->pModule->xBestIndex(pVtab, p); TRACE_IDX_OUTPUTS(p); @@ -100421,310 +115489,235 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ return pParse->nErr; } +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* -** Compute the best index for a virtual table. +** Estimate the location of a particular key among all keys in an +** index. Store the results in aStat as follows: ** -** The best index is computed by the xBestIndex method of the virtual -** table module. This routine is really just a wrapper that sets up -** the sqlite3_index_info structure that is used to communicate with -** xBestIndex. +** aStat[0] Est. number of rows less than pVal +** aStat[1] Est. number of rows equal to pVal ** -** In a join, this routine might be called multiple times for the -** same virtual table. The sqlite3_index_info structure is created -** and initialized on the first invocation and reused on all subsequent -** invocations. The sqlite3_index_info structure is also used when -** code is generated to access the virtual table. The whereInfoDelete() -** routine takes care of freeing the sqlite3_index_info structure after -** everybody has finished with it. +** Return SQLITE_OK on success. */ -static void bestVirtualIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for index */ - Bitmask notValid, /* Cursors not valid for any purpose */ - ExprList *pOrderBy, /* The order by clause */ - WhereCost *pCost, /* Lowest cost query plan */ - sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ -){ - Table *pTab = pSrc->pTab; - sqlite3_index_info *pIdxInfo; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int i, j; - int nOrderBy; - double rCost; - - /* Make sure wsFlags is initialized to some sane value. Otherwise, if the - ** malloc in allocateIndexInfo() fails and this function returns leaving - ** wsFlags in an uninitialized state, the caller may behave unpredictably. - */ - memset(pCost, 0, sizeof(*pCost)); - pCost->plan.wsFlags = WHERE_VIRTUALTABLE; - - /* If the sqlite3_index_info structure has not been previously - ** allocated and initialized, then allocate and initialize it now. - */ - pIdxInfo = *ppIdxInfo; - if( pIdxInfo==0 ){ - *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy); - } - if( pIdxInfo==0 ){ - return; - } - - /* At this point, the sqlite3_index_info structure that pIdxInfo points - ** to will have been initialized, either during the current invocation or - ** during some prior invocation. Now we just have to customize the - ** details of pIdxInfo for the current invocation and pass it to - ** xBestIndex. - */ - - /* The module name must be defined. Also, by this point there must - ** be a pointer to an sqlite3_vtab structure. Otherwise - ** sqlite3ViewGetColumnNames() would have picked up the error. - */ - assert( pTab->azModuleArg && pTab->azModuleArg[0] ); - assert( sqlite3GetVTable(pParse->db, pTab) ); - - /* Set the aConstraint[].usable fields and initialize all - ** output variables to zero. - ** - ** aConstraint[].usable is true for constraints where the right-hand - ** side contains only references to tables to the left of the current - ** table. In other words, if the constraint is of the form: - ** - ** column = expr - ** - ** and we are evaluating a join, then the constraint on column is - ** only valid if all tables referenced in expr occur to the left - ** of the table containing column. - ** - ** The aConstraints[] array contains entries for all constraints - ** on the current table. That way we only have to compute it once - ** even though we might try to pick the best index multiple times. - ** For each attempt at picking an index, the order of tables in the - ** join might be different so we have to recompute the usable flag - ** each time. - */ - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pUsage = pIdxInfo->aConstraintUsage; - for(i=0; inConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - pIdxCons->usable = (pTerm->prereqRight¬Ready) ? 0 : 1; - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - } - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); - nOrderBy = pIdxInfo->nOrderBy; - if( !pOrderBy ){ - pIdxInfo->nOrderBy = 0; - } - - if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ - return; - } - - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; inConstraint; i++){ - if( pUsage[i].argvIndex>0 ){ - pCost->used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; - } - } - - /* If there is an ORDER BY clause, and the selected virtual table index - ** does not satisfy it, increase the cost of the scan accordingly. This - ** matches the processing for non-virtual tables in bestBtreeIndex(). - */ - rCost = pIdxInfo->estimatedCost; - if( pOrderBy && pIdxInfo->orderByConsumed==0 ){ - rCost += estLog(rCost)*rCost; - } - - /* The cost is not allowed to be larger than SQLITE_BIG_DBL (the - ** inital value of lowestCost in this loop. If it is, then the - ** (costrCost = (SQLITE_BIG_DBL/((double)2)); - }else{ - pCost->rCost = rCost; - } - pCost->plan.u.pVtabIdx = pIdxInfo; - if( pIdxInfo->orderByConsumed ){ - pCost->plan.wsFlags |= WHERE_ORDERBY; - } - pCost->plan.nEq = 0; - pIdxInfo->nOrderBy = nOrderBy; - - /* Try to find a more efficient access pattern by using multiple indexes - ** to optimize an OR expression within the WHERE clause. - */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/* -** Argument pIdx is a pointer to an index structure that has an array of -** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column -** stored in Index.aSample. These samples divide the domain of values stored -** the index into (SQLITE_INDEX_SAMPLES+1) regions. -** Region 0 contains all values less than the first sample value. Region -** 1 contains values between the first and second samples. Region 2 contains -** values between samples 2 and 3. And so on. Region SQLITE_INDEX_SAMPLES -** contains values larger than the last sample. -** -** If the index contains many duplicates of a single value, then it is -** possible that two or more adjacent samples can hold the same value. -** When that is the case, the smallest possible region code is returned -** when roundUp is false and the largest possible region code is returned -** when roundUp is true. -** -** If successful, this function determines which of the regions value -** pVal lies in, sets *piRegion to the region index (a value between 0 -** and SQLITE_INDEX_SAMPLES+1, inclusive) and returns SQLITE_OK. -** Or, if an OOM occurs while converting text values between encodings, -** SQLITE_NOMEM is returned and *piRegion is undefined. -*/ -#ifdef SQLITE_ENABLE_STAT2 -static int whereRangeRegion( +static void whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ - sqlite3_value *pVal, /* Value to consider */ - int roundUp, /* Return largest valid region if true */ - int *piRegion /* OUT: Region of domain in which value lies */ + UnpackedRecord *pRec, /* Vector of values to consider */ + int roundUp, /* Round up if true. Round down if false */ + tRowcnt *aStat /* OUT: stats written here */ ){ - assert( roundUp==0 || roundUp==1 ); - if( ALWAYS(pVal) ){ - IndexSample *aSample = pIdx->aSample; - int i = 0; - int eType = sqlite3_value_type(pVal); + IndexSample *aSample = pIdx->aSample; + int iCol; /* Index of required stats in anEq[] etc. */ + int iMin = 0; /* Smallest sample not yet tested */ + int i = pIdx->nSample; /* Smallest sample larger than or equal to pRec */ + int iTest; /* Next sample to test */ + int res; /* Result of comparison operation */ - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - double r = sqlite3_value_double(pVal); - for(i=0; i=SQLITE_TEXT ) break; - if( roundUp ){ - if( aSample[i].u.r>r ) break; - }else{ - if( aSample[i].u.r>=r ) break; - } - } - }else if( eType==SQLITE_NULL ){ - i = 0; - if( roundUp ){ - while( idb; - CollSeq *pColl; - const u8 *z; - int n; - - /* pVal comes from sqlite3ValueFromExpr() so the type cannot be NULL */ - assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); - - if( eType==SQLITE_BLOB ){ - z = (const u8 *)sqlite3_value_blob(pVal); - pColl = db->pDfltColl; - assert( pColl->enc==SQLITE_UTF8 ); - }else{ - pColl = sqlite3GetCollSeq(db, SQLITE_UTF8, 0, *pIdx->azColl); - if( pColl==0 ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", - *pIdx->azColl); - return SQLITE_ERROR; - } - z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); - if( !z ){ - return SQLITE_NOMEM; - } - assert( z && pColl && pColl->xCmp ); - } - n = sqlite3ValueBytes(pVal, pColl->enc); - - for(i=0; ienc!=SQLITE_UTF8 ){ - int nSample; - char *zSample = sqlite3Utf8to16( - db, pColl->enc, aSample[i].u.z, aSample[i].nByte, &nSample - ); - if( !zSample ){ - assert( db->mallocFailed ); - return SQLITE_NOMEM; - } - c = pColl->xCmp(pColl->pUser, nSample, zSample, n, z); - sqlite3DbFree(db, zSample); - }else +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( pParse ); #endif - { - c = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z); - } - if( c-roundUp>=0 ) break; - } + assert( pRec!=0 ); + iCol = pRec->nField - 1; + assert( pIdx->nSample>0 ); + assert( pRec->nField>0 && iColnSampleCol ); + do{ + iTest = (iMin+i)/2; + res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); + if( res<0 ){ + iMin = iTest+1; + }else{ + i = iTest; } + }while( res && iMin=0 && i<=SQLITE_INDEX_SAMPLES ); - *piRegion = i; +#ifdef SQLITE_DEBUG + /* The following assert statements check that the binary search code + ** above found the right answer. This block serves no purpose other + ** than to invoke the asserts. */ + if( res==0 ){ + /* If (res==0) is true, then sample $i must be equal to pRec */ + assert( inSample ); + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed ); + }else{ + /* Otherwise, pRec must be smaller than sample $i and larger than + ** sample ($i-1). */ + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || pParse->db->mallocFailed ); + assert( i==0 + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || pParse->db->mallocFailed ); + } +#endif /* ifdef SQLITE_DEBUG */ + + /* At this point, aSample[i] is the first sample that is greater than + ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less + ** than pVal. If aSample[i]==pVal, then res==0. + */ + if( res==0 ){ + aStat[0] = aSample[i].anLt[iCol]; + aStat[1] = aSample[i].anEq[iCol]; + }else{ + tRowcnt iLower, iUpper, iGap; + if( i==0 ){ + iLower = 0; + iUpper = aSample[0].anLt[iCol]; + }else{ + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; + iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; + } + aStat[1] = pIdx->aAvgEq[iCol]; + if( iLower>=iUpper ){ + iGap = 0; + }else{ + iGap = iUpper - iLower; + } + if( roundUp ){ + iGap = (iGap*2)/3; + }else{ + iGap = iGap/3; + } + aStat[0] = iLower + iGap; } - return SQLITE_OK; } -#endif /* #ifdef SQLITE_ENABLE_STAT2 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* -** If expression pExpr represents a literal value, set *pp to point to -** an sqlite3_value structure containing the same value, with affinity -** aff applied to it, before returning. It is the responsibility of the -** caller to eventually release this structure by passing it to -** sqlite3ValueFree(). +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. ** -** If the current parse is a recompile (sqlite3Reprepare()) and pExpr -** is an SQL variable that currently has a non-NULL value bound to it, -** create an sqlite3_value structure containing this value, again with -** affinity aff applied to it, instead. -** -** If neither of the above apply, set *pp to NULL. -** -** If an error occurs, return an error code. Otherwise, SQLITE_OK. +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. */ -#ifdef SQLITE_ENABLE_STAT2 -static int valueFromExpr( - Parse *pParse, - Expr *pExpr, - u8 aff, - sqlite3_value **pp -){ - if( pExpr->op==TK_VARIABLE - || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) - ){ - int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */ - *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff); - return SQLITE_OK; +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } } - return sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, aff, pp); + return nRet; } -#endif + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: +** +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; +** +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. +** +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + int iCol = p->aiColumn[nEq]; + u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } + } + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* ** This function is used to estimate the number of rows that will be visited @@ -100741,116 +115734,188 @@ static int valueFromExpr( ** If either of the upper or lower bound is not present, then NULL is passed in ** place of the corresponding WhereTerm. ** -** The nEq parameter is passed the index of the index column subject to the -** range constraint. Or, equivalently, the number of equality constraints -** optimized by the proposed index scan. For example, assuming index p is -** on t1(a, b), and the SQL query is: +** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index +** column subject to the range constraint. Or, equivalently, the number of +** equality constraints optimized by the proposed index scan. For example, +** assuming index p is on t1(a, b), and the SQL query is: ** ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... ** -** then nEq should be passed the value 1 (as the range restricted column, -** b, is the second left-most column of the index). Or, if the query is: +** then nEq is set to 1 (as the range restricted column, b, is the second +** left-most column of the index). Or, if the query is: ** ** ... FROM t1 WHERE a > ? AND a < ? ... ** -** then nEq should be passed 0. +** then nEq is set to 0. ** -** The returned value is an integer between 1 and 100, inclusive. A return -** value of 1 indicates that the proposed range scan is expected to visit -** approximately 1/100th (1%) of the rows selected by the nEq equality -** constraints (if any). A return value of 100 indicates that it is expected -** that the range scan will visit every row (100%) selected by the equality -** constraints. -** -** In the absence of sqlite_stat2 ANALYZE data, each range inequality -** reduces the search space by 3/4ths. Hence a single constraint (x>?) -** results in a return of 25 and a range constraint (x>? AND x? AND xaCol[] of the range-compared column */ + WhereLoopBuilder *pBuilder, WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - int *piEst /* OUT: Return value */ + WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ ){ int rc = SQLITE_OK; + int nOut = pLoop->nOut; + LogEst nNew; -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; - if( nEq==0 && p->aSample ){ - sqlite3_value *pLowerVal = 0; - sqlite3_value *pUpperVal = 0; - int iEst; - int iLower = 0; - int iUpper = SQLITE_INDEX_SAMPLES; - int roundUpUpper = 0; - int roundUpLower = 0; - u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity; + if( p->nSample>0 + && nEqnSampleCol + && OptimizationEnabled(pParse->db, SQLITE_Stat3) + ){ + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; - if( pLower ){ - Expr *pExpr = pLower->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pLowerVal); - assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); - roundUpLower = (pLower->eOperator==WO_GT) ?1:0; - } - if( rc==SQLITE_OK && pUpper ){ - Expr *pExpr = pUpper->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pUpperVal); - assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); - roundUpUpper = (pUpper->eOperator==WO_LE) ?1:0; - } + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + */ + tRowcnt iLower; + tRowcnt iUpper; - if( rc!=SQLITE_OK || (pLowerVal==0 && pUpperVal==0) ){ - sqlite3ValueFree(pLowerVal); - sqlite3ValueFree(pUpperVal); - goto range_est_fallback; - }else if( pLowerVal==0 ){ - rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper); - if( pLower ) iLower = iUpper/2; - }else if( pUpperVal==0 ){ - rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower); - if( pUpper ) iUpper = (iLower + SQLITE_INDEX_SAMPLES + 1)/2; - }else{ - rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper); - if( rc==SQLITE_OK ){ - rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower); + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; + } + if( nEq==p->nKeyCol ){ + aff = SQLITE_AFF_INTEGER; + }else{ + aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; + } + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; } - } - WHERETRACE(("range scan regions: %d..%d\n", iLower, iUpper)); - iEst = iUpper - iLower; - testcase( iEst==SQLITE_INDEX_SAMPLES ); - assert( iEst<=SQLITE_INDEX_SAMPLES ); - if( iEst<1 ){ - *piEst = 50/SQLITE_INDEX_SAMPLES; + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); + } + + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + pLower = 0; + } + } + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewwtFlags & TERM_VNULL)==0 ) *piEst /= 4; - if( pUpper ) *piEst /= 4; +#endif + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; + + nOut -= (pLower!=0) + (pUpper!=0); + if( nNew<10 ) nNew = 10; + if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); + } +#endif + pLoop->nOut = (LogEst)nOut; return rc; } -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in ** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat2 histogram data is available +** column of an index and sqlite_stat3 histogram data is available ** for that index. When pExpr==NULL that means the constraint is ** "x IS NULL" instead of "x=VALUE". ** @@ -100865,45 +115930,53 @@ range_est_fallback: */ static int whereEqualScanEst( Parse *pParse, /* Parsing & code generating context */ - Index *p, /* The index whose left-most column is pTerm */ + WhereLoopBuilder *pBuilder, Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ - double *pnRow /* Write the revised row estimate here */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ - sqlite3_value *pRhs = 0; /* VALUE on right-hand side of pTerm */ - int iLower, iUpper; /* Range of histogram regions containing pRhs */ + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; + UnpackedRecord *pRec = pBuilder->pRec; u8 aff; /* Column affinity */ int rc; /* Subfunction return code */ - double nRowEst; /* New estimate of the number of rows */ + tRowcnt a[2]; /* Statistics */ + int bOk; + assert( nEq>=1 ); + assert( nEq<=p->nColumn ); assert( p->aSample!=0 ); - aff = p->pTable->aCol[p->aiColumn[0]].affinity; - if( pExpr ){ - rc = valueFromExpr(pParse, pExpr, aff, &pRhs); - if( rc ) goto whereEqualScanEst_cancel; - }else{ - pRhs = sqlite3ValueNew(pParse->db); - } - if( pRhs==0 ) return SQLITE_NOTFOUND; - rc = whereRangeRegion(pParse, p, pRhs, 0, &iLower); - if( rc ) goto whereEqualScanEst_cancel; - rc = whereRangeRegion(pParse, p, pRhs, 1, &iUpper); - if( rc ) goto whereEqualScanEst_cancel; - WHERETRACE(("equality scan regions: %d..%d\n", iLower, iUpper)); - if( iLower>=iUpper ){ - nRowEst = p->aiRowEst[0]/(SQLITE_INDEX_SAMPLES*2); - if( nRowEst<*pnRow ) *pnRow = nRowEst; - }else{ - nRowEst = (iUpper-iLower)*p->aiRowEst[0]/SQLITE_INDEX_SAMPLES; - *pnRow = nRowEst; + assert( p->nSample>0 ); + assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ + return SQLITE_NOTFOUND; } -whereEqualScanEst_cancel: - sqlite3ValueFree(pRhs); + /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() + ** below would return the same value. */ + if( nEq>=p->nColumn ){ + *pnRow = 1; + return SQLITE_OK; + } + + aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + pBuilder->pRec = pRec; + if( rc!=SQLITE_OK ) return rc; + if( bOk==0 ) return SQLITE_NOTFOUND; + pBuilder->nRecValid = nEq; + + whereKeyStats(pParse, p, pRec, 0, a); + WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); + *pnRow = a[1]; + return rc; } -#endif /* defined(SQLITE_ENABLE_STAT2) */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -100922,580 +115995,35 @@ whereEqualScanEst_cancel: */ static int whereInScanEst( Parse *pParse, /* Parsing & code generating context */ - Index *p, /* The index whose left-most column is pTerm */ + WhereLoopBuilder *pBuilder, ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ - double *pnRow /* Write the revised row estimate here */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ - sqlite3_value *pVal = 0; /* One value from list */ - int iLower, iUpper; /* Range of histogram regions containing pRhs */ - u8 aff; /* Column affinity */ - int rc = SQLITE_OK; /* Subfunction return code */ - double nRowEst; /* New estimate of the number of rows */ - int nSpan = 0; /* Number of histogram regions spanned */ - int nSingle = 0; /* Histogram regions hit by a single value */ - int nNotFound = 0; /* Count of values that are not constants */ - int i; /* Loop counter */ - u8 aSpan[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions that are spanned */ - u8 aSingle[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions hit once */ + Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); + int nRecValid = pBuilder->nRecValid; + int rc = SQLITE_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ assert( p->aSample!=0 ); - aff = p->pTable->aCol[p->aiColumn[0]].affinity; - memset(aSpan, 0, sizeof(aSpan)); - memset(aSingle, 0, sizeof(aSingle)); - for(i=0; inExpr; i++){ - sqlite3ValueFree(pVal); - rc = valueFromExpr(pParse, pList->a[i].pExpr, aff, &pVal); - if( rc ) break; - if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ - nNotFound++; - continue; - } - rc = whereRangeRegion(pParse, p, pVal, 0, &iLower); - if( rc ) break; - rc = whereRangeRegion(pParse, p, pVal, 1, &iUpper); - if( rc ) break; - if( iLower>=iUpper ){ - aSingle[iLower] = 1; - }else{ - assert( iLower>=0 && iUpper<=SQLITE_INDEX_SAMPLES ); - while( iLowernExpr; i++){ + nEst = nRow0; + rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); + nRowEst += nEst; + pBuilder->nRecValid = nRecValid; } + if( rc==SQLITE_OK ){ - for(i=nSpan=0; i<=SQLITE_INDEX_SAMPLES; i++){ - if( aSpan[i] ){ - nSpan++; - }else if( aSingle[i] ){ - nSingle++; - } - } - nRowEst = (nSpan*2+nSingle)*p->aiRowEst[0]/(2*SQLITE_INDEX_SAMPLES) - + nNotFound*p->aiRowEst[1]; - if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(("IN row estimate: nSpan=%d, nSingle=%d, nNotFound=%d, est=%g\n", - nSpan, nSingle, nNotFound, nRowEst)); + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); } - sqlite3ValueFree(pVal); + assert( pBuilder->nRecValid==nRecValid ); return rc; } -#endif /* defined(SQLITE_ENABLE_STAT2) */ - - -/* -** Find the best query plan for accessing a particular table. Write the -** best query plan and its cost into the WhereCost object supplied as the -** last parameter. -** -** The lowest cost plan wins. The cost is an estimate of the amount of -** CPU and disk I/O needed to process the requested result. -** Factors that influence cost include: -** -** * The estimated number of rows that will be retrieved. (The -** fewer the better.) -** -** * Whether or not sorting must occur. -** -** * Whether or not there must be separate lookups in the -** index and in the main table. -** -** If there was an INDEXED BY clause (pSrc->pIndex) attached to the table in -** the SQL statement, then this function only considers plans using the -** named index. If no such plan is found, then the returned cost is -** SQLITE_BIG_DBL. If a plan is found that uses the named index, -** then the cost is calculated in the usual way. -** -** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table -** in the SELECT statement, then no indexes are considered. However, the -** selected plan may still take advantage of the built-in rowid primary key -** index. -*/ -static void bestBtreeIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ - int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ - Index *pProbe; /* An index we are evaluating */ - Index *pIdx; /* Copy of pProbe, or zero for IPK index */ - int eqTermMask; /* Current mask of valid equality operators */ - int idxEqTermMask; /* Index mask of valid equality operators */ - Index sPk; /* A fake index object for the primary key */ - unsigned int aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ - int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ - int wsFlagMask; /* Allowed flags in pCost->plan.wsFlag */ - - /* Initialize the cost to a worst-case value */ - memset(pCost, 0, sizeof(*pCost)); - pCost->rCost = SQLITE_BIG_DBL; - - /* If the pSrc table is the right table of a LEFT JOIN then we may not - ** use an index to satisfy IS NULL constraints on that table. This is - ** because columns might end up being NULL if the table does not match - - ** a circumstance which the index cannot help us discover. Ticket #2177. - */ - if( pSrc->jointype & JT_LEFT ){ - idxEqTermMask = WO_EQ|WO_IN; - }else{ - idxEqTermMask = WO_EQ|WO_IN|WO_ISNULL; - } - - if( pSrc->pIndex ){ - /* An INDEXED BY clause specifies a particular index to use */ - pIdx = pProbe = pSrc->pIndex; - wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); - eqTermMask = idxEqTermMask; - }else{ - /* There is no INDEXED BY clause. Create a fake Index object in local - ** variable sPk to represent the rowid primary key index. Make this - ** fake index the first in a chain of Index objects with all of the real - ** indices to follow */ - Index *pFirst; /* First of real indices on the table */ - memset(&sPk, 0, sizeof(Index)); - sPk.nColumn = 1; - sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; - sPk.onError = OE_Replace; - sPk.pTable = pSrc->pTab; - aiRowEstPk[0] = pSrc->pTab->nRowEst; - aiRowEstPk[1] = 1; - pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ - /* The real indices of the table are only considered if the - ** NOT INDEXED qualifier is omitted from the FROM clause */ - sPk.pNext = pFirst; - } - pProbe = &sPk; - wsFlagMask = ~( - WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE - ); - eqTermMask = WO_EQ|WO_IN; - pIdx = 0; - } - - /* Loop over all indices looking for the best one to use - */ - for(; pProbe; pIdx=pProbe=pProbe->pNext){ - const unsigned int * const aiRowEst = pProbe->aiRowEst; - double cost; /* Cost of using pProbe */ - double nRow; /* Estimated number of rows in result set */ - double log10N; /* base-10 logarithm of nRow (inexact) */ - int rev; /* True to scan in reverse order */ - int wsFlags = 0; - Bitmask used = 0; - - /* The following variables are populated based on the properties of - ** index being evaluated. They are then used to determine the expected - ** cost and number of rows returned. - ** - ** nEq: - ** Number of equality terms that can be implemented using the index. - ** In other words, the number of initial fields in the index that - ** are used in == or IN or NOT NULL constraints of the WHERE clause. - ** - ** nInMul: - ** The "in-multiplier". This is an estimate of how many seek operations - ** SQLite must perform on the index in question. For example, if the - ** WHERE clause is: - ** - ** WHERE a IN (1, 2, 3) AND b IN (4, 5, 6) - ** - ** SQLite must perform 9 lookups on an index on (a, b), so nInMul is - ** set to 9. Given the same schema and either of the following WHERE - ** clauses: - ** - ** WHERE a = 1 - ** WHERE a >= 2 - ** - ** nInMul is set to 1. - ** - ** If there exists a WHERE term of the form "x IN (SELECT ...)", then - ** the sub-select is assumed to return 25 rows for the purposes of - ** determining nInMul. - ** - ** bInEst: - ** Set to true if there was at least one "x IN (SELECT ...)" term used - ** in determining the value of nInMul. Note that the RHS of the - ** IN operator must be a SELECT, not a value list, for this variable - ** to be true. - ** - ** estBound: - ** An estimate on the amount of the table that must be searched. A - ** value of 100 means the entire table is searched. Range constraints - ** might reduce this to a value less than 100 to indicate that only - ** a fraction of the table needs searching. In the absence of - ** sqlite_stat2 ANALYZE data, a single inequality reduces the search - ** space to 1/4rd its original size. So an x>? constraint reduces - ** estBound to 25. Two constraints (x>? AND xnColumn; nEq++){ - int j = pProbe->aiColumn[nEq]; - pTerm = findTerm(pWC, iCur, j, notReady, eqTermMask, pIdx); - if( pTerm==0 ) break; - wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); - if( pTerm->eOperator & WO_IN ){ - Expr *pExpr = pTerm->pExpr; - wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nInMul *= 25; - bInEst = 1; - }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ - /* "x IN (value, value, ...)" */ - nInMul *= pExpr->x.pList->nExpr; - } - }else if( pTerm->eOperator & WO_ISNULL ){ - wsFlags |= WHERE_COLUMN_NULL; - } -#ifdef SQLITE_ENABLE_STAT2 - if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; -#endif - used |= pTerm->prereqRight; - } - - /* Determine the value of estBound. */ - if( nEqnColumn && pProbe->bUnordered==0 ){ - int j = pProbe->aiColumn[nEq]; - if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ - WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); - WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); - whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &estBound); - if( pTop ){ - nBound = 1; - wsFlags |= WHERE_TOP_LIMIT; - used |= pTop->prereqRight; - } - if( pBtm ){ - nBound++; - wsFlags |= WHERE_BTM_LIMIT; - used |= pBtm->prereqRight; - } - wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); - } - }else if( pProbe->onError!=OE_None ){ - testcase( wsFlags & WHERE_COLUMN_IN ); - testcase( wsFlags & WHERE_COLUMN_NULL ); - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - wsFlags |= WHERE_UNIQUE; - } - } - - /* If there is an ORDER BY clause and the index being considered will - ** naturally scan rows in the required order, set the appropriate flags - ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index - ** will scan rows in a different order, set the bSort variable. */ - if( pOrderBy ){ - if( (wsFlags & WHERE_COLUMN_IN)==0 - && pProbe->bUnordered==0 - && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, - nEq, wsFlags, &rev) - ){ - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); - }else{ - bSort = 1; - } - } - - /* If currently calculating the cost of using an index (not the IPK - ** index), determine if all required column data may be obtained without - ** using the main table (i.e. if the index is a covering - ** index for this query). If it is, set the WHERE_IDX_ONLY flag in - ** wsFlags. Otherwise, set the bLookup variable to true. */ - if( pIdx && wsFlags ){ - Bitmask m = pSrc->colUsed; - int j; - for(j=0; jnColumn; j++){ - int x = pIdx->aiColumn[j]; - if( xaiRowEst[0] ){ - nRow = aiRowEst[0]/2; - nInMul = (int)(nRow / aiRowEst[nEq]); - } - -#ifdef SQLITE_ENABLE_STAT2 - /* If the constraint is of the form x=VALUE and histogram - ** data is available for column x, then it might be possible - ** to get a better estimate on the number of rows based on - ** VALUE and how common that value is according to the histogram. - */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){ - if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - testcase( pFirstTerm->eOperator==WO_EQ ); - testcase( pFirstTerm->eOperator==WO_ISNULL ); - whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); - }else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){ - whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow); - } - } -#endif /* SQLITE_ENABLE_STAT2 */ - - /* Adjust the number of output rows and downward to reflect rows - ** that are excluded by range constraints. - */ - nRow = (nRow * (double)estBound) / (double)100; - if( nRow<1 ) nRow = 1; - - /* Experiments run on real SQLite databases show that the time needed - ** to do a binary search to locate a row in a table or index is roughly - ** log10(N) times the time to move from one row to the next row within - ** a table or index. The actual times can vary, with the size of - ** records being an important factor. Both moves and searches are - ** slower with larger records, presumably because fewer records fit - ** on one page and hence more pages have to be fetched. - ** - ** The ANALYZE command and the sqlite_stat1 and sqlite_stat2 tables do - ** not give us data on the relative sizes of table and index records. - ** So this computation assumes table records are about twice as big - ** as index records - */ - if( (wsFlags & WHERE_NOT_FULLSCAN)==0 ){ - /* The cost of a full table scan is a number of move operations equal - ** to the number of rows in the table. - ** - ** We add an additional 4x penalty to full table scans. This causes - ** the cost function to err on the side of choosing an index over - ** choosing a full scan. This 4x full-scan penalty is an arguable - ** decision and one which we expect to revisit in the future. But - ** it seems to be working well enough at the moment. - */ - cost = aiRowEst[0]*4; - }else{ - log10N = estLog(aiRowEst[0]); - cost = nRow; - if( pIdx ){ - if( bLookup ){ - /* For an index lookup followed by a table lookup: - ** nInMul index searches to find the start of each index range - ** + nRow steps through the index - ** + nRow table searches to lookup the table entry using the rowid - */ - cost += (nInMul + nRow)*log10N; - }else{ - /* For a covering index: - ** nInMul index searches to find the initial entry - ** + nRow steps through the index - */ - cost += nInMul*log10N; - } - }else{ - /* For a rowid primary key lookup: - ** nInMult table searches to find the initial entry for each range - ** + nRow steps through the table - */ - cost += nInMul*log10N; - } - } - - /* Add in the estimated cost of sorting the result. Actual experimental - ** measurements of sorting performance in SQLite show that sorting time - ** adds C*N*log10(N) to the cost, where N is the number of rows to be - ** sorted and C is a factor between 1.95 and 4.3. We will split the - ** difference and select C of 3.0. - */ - if( bSort ){ - cost += nRow*estLog(nRow)*3; - } - - /**** Cost of using this index has now been computed ****/ - - /* If there are additional constraints on this table that cannot - ** be used with the current index, but which might lower the number - ** of output rows, adjust the nRow value accordingly. This only - ** matters if the current index is the least costly, so do not bother - ** with this step if we already know this index will not be chosen. - ** Also, never reduce the output row count below 2 using this step. - ** - ** It is critical that the notValid mask be used here instead of - ** the notReady mask. When computing an "optimal" index, the notReady - ** mask will only have one bit set - the bit for the current table. - ** The notValid mask, on the other hand, always has all bits set for - ** tables that are not in outer loops. If notReady is used here instead - ** of notValid, then a optimal index that depends on inner joins loops - ** might be selected even when there exists an optimal index that has - ** no such dependency. - */ - if( nRow>2 && cost<=pCost->rCost ){ - int k; /* Loop counter */ - int nSkipEq = nEq; /* Number of == constraints to skip */ - int nSkipRange = nBound; /* Number of < constraints to skip */ - Bitmask thisTab; /* Bitmap for pSrc */ - - thisTab = getMask(pWC->pMaskSet, iCur); - for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ - if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notValid)!=thisTab ) continue; - if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ - if( nSkipEq ){ - /* Ignore the first nEq equality matches since the index - ** has already accounted for these */ - nSkipEq--; - }else{ - /* Assume each additional equality match reduces the result - ** set size by a factor of 10 */ - nRow /= 10; - } - }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ - if( nSkipRange ){ - /* Ignore the first nSkipRange range constraints since the index - ** has already accounted for these */ - nSkipRange--; - }else{ - /* Assume each additional range constraint reduces the result - ** set size by a factor of 3. Indexed range constraints reduce - ** the search space by a larger factor: 4. We make indexed range - ** more selective intentionally because of the subjective - ** observation that indexed range constraints really are more - ** selective in practice, on average. */ - nRow /= 3; - } - }else if( pTerm->eOperator!=WO_NOOP ){ - /* Any other expression lowers the output row count by half */ - nRow /= 2; - } - } - if( nRow<2 ) nRow = 2; - } - - - WHERETRACE(( - "%s(%s): nEq=%d nInMul=%d estBound=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", - pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), - nEq, nInMul, estBound, bSort, bLookup, wsFlags, - notReady, log10N, nRow, cost, used - )); - - /* If this index is the best we have seen so far, then record this - ** index and its cost in the pCost structure. - */ - if( (!pIdx || wsFlags) - && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) - ){ - pCost->rCost = cost; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = (wsFlags&wsFlagMask); - pCost->plan.nEq = nEq; - pCost->plan.u.pIdx = pIdx; - } - - /* If there was an INDEXED BY clause, then only that one index is - ** considered. */ - if( pSrc->pIndex ) break; - - /* Reset masks for the next index in the loop */ - wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); - eqTermMask = idxEqTermMask; - } - - /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag - ** is set, then reverse the order that the index will be scanned - ** in. This is used for application testing, to help find cases - ** where application behaviour depends on the (undefined) order that - ** SQLite outputs rows in in the absence of an ORDER BY clause. */ - if( !pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ - pCost->plan.wsFlags |= WHERE_REVERSE; - } - - assert( pOrderBy || (pCost->plan.wsFlags&WHERE_ORDERBY)==0 ); - assert( pCost->plan.u.pIdx==0 || (pCost->plan.wsFlags&WHERE_ROWID_EQ)==0 ); - assert( pSrc->pIndex==0 - || pCost->plan.u.pIdx==0 - || pCost->plan.u.pIdx==pSrc->pIndex - ); - - WHERETRACE(("best index is: %s\n", - ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : - pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") - )); - - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); - bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); - pCost->plan.wsFlags |= eqTermMask; -} - -/* -** Find the query plan for accessing table pSrc->pTab. Write the -** best query plan and its cost into the WhereCost object supplied -** as the last parameter. This function may calculate the cost of -** both real and virtual table scans. -*/ -static void bestIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pSrc->pTab) ){ - sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); - if( p->needToFreeIdxStr ){ - sqlite3_free(p->idxStr); - } - sqlite3DbFree(pParse->db, p); - }else -#endif - { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); - } -} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* ** Disable a term in the WHERE clause. Except, do not disable the term @@ -101512,9 +116040,6 @@ static void bestIndex( ** in the ON clause. The term is disabled in (3) because it is not part ** of a LEFT OUTER JOIN. In (1), the term is not disabled. ** -** IMPLEMENTATION-OF: R-24597-58655 No tests are done for terms that are -** completely satisfied by indices. -** ** Disabling a term causes that term to not be tested in the inner loop ** of the join. Disabling is an optimization. When terms are satisfied ** by indices, we disable them to prevent redundant tests in the inner @@ -101527,6 +116052,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ if( pTerm && (pTerm->wtFlags & TERM_CODED)==0 && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->notReady & pTerm->prereqAll)==0 ){ pTerm->wtFlags |= TERM_CODED; if( pTerm->iParent>=0 ){ @@ -101592,7 +116118,9 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ static int codeEqualityTerm( Parse *pParse, /* The parsing context */ WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* When level of the FROM clause we are working on */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; @@ -101610,13 +116138,29 @@ static int codeEqualityTerm( int eType; int iTab; struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } assert( pX->op==TK_IN ); iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, 0); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } iTab = pX->iTable; - sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); - assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; if( pLevel->u.in.nIn==0 ){ pLevel->addrNxt = sqlite3VdbeMakeLabel(v); } @@ -101633,7 +116177,8 @@ static int codeEqualityTerm( }else{ pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } - sqlite3VdbeAddOp1(v, OP_IsNull, iReg); + pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); }else{ pLevel->u.in.nIn = 0; } @@ -101645,7 +116190,7 @@ static int codeEqualityTerm( /* ** Generate code that will evaluate all == and IN constraints for an -** index. +** index scan. ** ** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). ** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 @@ -101660,9 +116205,15 @@ static int codeEqualityTerm( ** The only thing it does is allocate the pLevel->iMem memory cell and ** compute the affinity string. ** -** This routine always allocates at least one memory cell and returns -** the index of that memory cell. The code that -** calls this routine will use that memory cell to store the termination +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. ** key value of the loop. If one or more IN operators appear, then ** this routine allocates an additional nEq memory cells for internal ** use. @@ -101685,29 +116236,33 @@ static int codeEqualityTerm( static int codeAllEqualityTerms( Parse *pParse, /* Parsing context */ WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - WhereClause *pWC, /* The WHERE clause */ - Bitmask notReady, /* Which parts of FROM have not yet been coded */ + int bRev, /* Reverse the order of IN operators */ int nExtraReg, /* Number of extra registers to allocate */ char **pzAff /* OUT: Set to point to affinity string */ ){ - int nEq = pLevel->plan.nEq; /* The number of == or IN constraints to code */ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ Vdbe *v = pParse->pVdbe; /* The vm under construction */ Index *pIdx; /* The index being used for this loop */ - int iCur = pLevel->iTabCur; /* The cursor of the table */ WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ int j; /* Loop counter */ int regBase; /* Base register */ int nReg; /* Number of registers to allocate */ char *zAff; /* Affinity string to return */ /* This module is only called on query plans that use an index. */ - assert( pLevel->plan.wsFlags & WHERE_INDEXED ); - pIdx = pLevel->plan.u.pIdx; + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + nSkip = pLoop->u.btree.nSkip; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); /* Figure out how many memory cells we will need then allocate them. */ regBase = pParse->nMem + 1; - nReg = pLevel->plan.nEq + nExtraReg; + nReg = pLoop->u.btree.nEq + nExtraReg; pParse->nMem += nReg; zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); @@ -101715,19 +116270,37 @@ static int codeAllEqualityTerms( pParse->db->mallocFailed = 1; } + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), + iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + sqlite3VdbeJumpHere(v, j); + for(j=0; jaiColumn[j]>=0 ); + VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); + } + } + /* Evaluate the equality constraints */ - assert( pIdx->nColumn>=nEq ); - for(j=0; j=nEq ); + for(j=nSkip; jaiColumn[j]; - pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx); - if( NEVER(pTerm==0) ) break; - /* The following true for indices with redundant columns. + pTerm = pLoop->aLTerm[j]; + assert( pTerm!=0 ); + /* The following testcase is true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ sqlite3ReleaseTempReg(pParse, regBase); @@ -101740,7 +116313,10 @@ static int codeAllEqualityTerms( testcase( pTerm->eOperator & WO_IN ); if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ Expr *pRight = pTerm->pExpr->pRight; - sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk); + if( sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } if( zAff ){ if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ zAff[j] = SQLITE_AFF_NONE; @@ -101771,16 +116347,15 @@ static void explainAppendTerm( const char *zOp /* Name of the operator */ ){ if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3StrAccumAppend(pStr, zColumn, -1); + sqlite3StrAccumAppendAll(pStr, zColumn); sqlite3StrAccumAppend(pStr, zOp, 1); sqlite3StrAccumAppend(pStr, "?", 1); } /* ** Argument pLevel describes a strategy for scanning table pTab. This -** function returns a pointer to a string buffer containing a description -** of the subset of table rows scanned by the strategy in the form of an -** SQL expression. Or, if all rows are scanned, NULL is returned. +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. ** ** For example, if the query: ** @@ -101790,39 +116365,37 @@ static void explainAppendTerm( ** string similar to: ** ** "a=? AND b>?" -** -** The returned pointer points to memory obtained from sqlite3DbMalloc(). -** It is the responsibility of the caller to free the buffer when it is -** no longer required. */ -static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ - WherePlan *pPlan = &pLevel->plan; - Index *pIndex = pPlan->u.pIdx; - int nEq = pPlan->nEq; +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ + Index *pIndex = pLoop->u.btree.pIndex; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->u.btree.nSkip; int i, j; Column *aCol = pTab->aCol; - int *aiColumn = pIndex->aiColumn; - StrAccum txt; + i16 *aiColumn = pIndex->aiColumn; - if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ - return 0; - } - sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); - txt.db = db; - sqlite3StrAccumAppend(&txt, " (", 2); + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3StrAccumAppend(pStr, " (", 2); for(i=0; i=nSkip ){ + explainAppendTerm(pStr, i, z, "="); + }else{ + if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3XPrintf(pStr, 0, "ANY(%s)", z); + } } j = i; - if( pPlan->wsFlags&WHERE_BTM_LIMIT ){ - explainAppendTerm(&txt, i++, aCol[aiColumn[j]].zName, ">"); + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(pStr, i++, z, ">"); } - if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ - explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<"); + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(pStr, i, z, "<"); } - sqlite3StrAccumAppend(&txt, ")", 1); - return sqlite3StrAccumFinish(&txt); + sqlite3StrAccumAppend(pStr, ")", 1); } /* @@ -101839,69 +116412,93 @@ static void explainOneScan( int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ - if( pParse->explain==2 ){ - u32 flags = pLevel->plan.wsFlags; +#ifndef SQLITE_DEBUG + if( pParse->explain==2 ) +#endif + { struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ - char *zMsg; /* Text to add to EQP output */ - sqlite3_int64 nRow; /* Expected number of rows visited by scan */ int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; - isSearch = (pLevel->plan.nEq>0) - || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); + sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.db = db; + sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); + sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); }else{ - zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } if( pItem->zAlias ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); + sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); } - if( (flags & WHERE_INDEXED)!=0 ){ - char *zWhere = explainIndexRange(db, pLevel, pItem->pTab); - zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, - ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""), - ((flags & WHERE_IDX_ONLY)?"COVERING ":""), - ((flags & WHERE_TEMP_INDEX)?"":" "), - ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName), - zWhere - ); - sqlite3DbFree(db, zWhere); - }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; - if( flags&WHERE_ROWID_EQ ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); - }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); - }else if( flags&WHERE_TOP_LIMIT ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowidu.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( isSearch ){ + zFmt = "PRIMARY KEY"; + } + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "AUTOMATIC COVERING INDEX"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "COVERING INDEX %s"; + }else{ + zFmt = "INDEX %s"; } + if( zFmt ){ + sqlite3StrAccumAppend(&str, " USING ", 7); + sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop, pItem->pTab); + } + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + const char *zRange; + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zRange = "(rowid=?)"; + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zRange = "(rowid>? AND rowid?)"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRange = "(rowidplan.u.pVtabIdx; - zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, - pVtabIdx->idxNum, pVtabIdx->idxStr); + sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif - if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ - testcase( wctrlFlags & WHERE_ORDERBY_MIN ); - nRow = 1; +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); }else{ - nRow = (sqlite3_int64)pLevel->plan.nRow; + sqlite3StrAccumAppend(&str, " (~1 row)", 9); } - zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); +#endif + zMsg = sqlite3StrAccumFinish(&str); sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); } } @@ -101917,7 +116514,6 @@ static void explainOneScan( static Bitmask codeOneLoopStart( WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ - u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ Bitmask notReady /* Which tables are currently available */ ){ int j, k; /* Loop counters */ @@ -101926,9 +116522,11 @@ static Bitmask codeOneLoopStart( int omitTable; /* True if we use the index only */ int bRev; /* True if we need to scan in reverse order */ WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ WhereClause *pWC; /* Decomposition of the entire WHERE clause */ WhereTerm *pTerm; /* A WHERE clause term */ Parse *pParse; /* Parsing context */ + sqlite3 *db; /* Database connection */ Vdbe *v; /* The prepared stmt under constructions */ struct SrcList_item *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ @@ -101938,13 +116536,17 @@ static Bitmask codeOneLoopStart( pParse = pWInfo->pParse; v = pParse->pVdbe; - pWC = pWInfo->pWC; + pWC = &pWInfo->sWC; + db = pParse->db; pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; iCur = pTabItem->iCursor; - bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; - omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 - && (wctrlFlags & WHERE_FORCE_TABLE)==0; + pLevel->notReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -101969,72 +116571,88 @@ static Bitmask codeOneLoopStart( VdbeComment((v, "init LEFT JOIN no-match flag")); } + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + pLevel->op = OP_Goto; + }else + #ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 0: The table is a virtual-table. Use the VFilter and VNext + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext ** to access the data. */ int iReg; /* P3 Value for OP_VFilter */ - sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; - int nConstraint = pVtabIdx->nConstraint; - struct sqlite3_index_constraint_usage *aUsage = - pVtabIdx->aConstraintUsage; - const struct sqlite3_index_constraint *aConstraint = - pVtabIdx->aConstraint; + int addrNotFound; + int nConstraint = pLoop->nLTerm; sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); - for(j=1; j<=nConstraint; j++){ - for(k=0; ka[iTerm].pExpr->pRight, iReg+j+1); - break; - } - } - if( k==nConstraint ) break; - } - sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr, - pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); - pVtabIdx->needToFreeIdxStr = 0; + addrNotFound = pLevel->addrBrk; for(j=0; ja[iTerm]); + int iTarget = iReg+j+2; + pTerm = pLoop->aLTerm[j]; + if( pTerm==0 ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + VdbeCoverage(v); + pLoop->u.vtab.needFree = 0; + for(j=0; ju.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pLoop->aLTerm[j]); } } pLevel->op = OP_VNext; pLevel->p1 = iCur; pLevel->p2 = sqlite3VdbeCurrentAddr(v); sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ - if( pLevel->plan.wsFlags & WHERE_ROWID_EQ ){ - /* Case 1: We can directly reference a single row using an + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an ** equality comparison against the ROWID field. Or ** we reference multiple rows using a "rowid IN (...)" ** construct. */ - iReleaseReg = sqlite3GetTempReg(pParse); - pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); + assert( pLoop->u.btree.nEq==1 ); + pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); - assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); VdbeComment((v, "pk")); pLevel->op = OP_Noop; - }else if( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ){ - /* Case 2: We have an inequality comparison against the ROWID field. + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. */ int testOp = OP_Noop; int start; @@ -102042,8 +116660,11 @@ static Bitmask codeOneLoopStart( WhereTerm *pStart, *pEnd; assert( omitTable==0 ); - pStart = findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0); - pEnd = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); if( bRev ){ pTerm = pStart; pStart = pEnd; @@ -102057,34 +116678,42 @@ static Bitmask codeOneLoopStart( ** seek opcodes. It depends on a particular ordering of TK_xx */ const u8 aMoveOp[] = { - /* TK_GT */ OP_SeekGt, - /* TK_LE */ OP_SeekLe, - /* TK_LT */ OP_SeekLt, - /* TK_GE */ OP_SeekGe + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE }; assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ - testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); pX = pStart->pExpr; assert( pX!=0 ); - assert( pStart->leftCursor==iCur ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); sqlite3ExprCacheAffinityChange(pParse, r1, 1); sqlite3ReleaseTempReg(pParse, rTemp); disableTerm(pLevel, pStart); }else{ sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); } if( pEnd ){ Expr *pX; pX = pEnd->pExpr; assert( pX!=0 ); - assert( pEnd->leftCursor==iCur ); - testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); memEndValue = ++pParse->nMem; sqlite3ExprCode(pParse, pX->pRight, memEndValue); if( pX->op==TK_LT || pX->op==TK_GT ){ @@ -102098,20 +116727,20 @@ static Bitmask codeOneLoopStart( pLevel->op = bRev ? OP_Prev : OP_Next; pLevel->p1 = iCur; pLevel->p2 = start; - if( pStart==0 && pEnd==0 ){ - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - }else{ - assert( pLevel->p5==0 ); - } + assert( pLevel->p5==0 ); if( testOp!=OP_Noop ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } - }else if( pLevel->plan.wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ) ){ - /* Case 3: A scan using an index. + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. ** ** The WHERE clause may contain zero or more equality ** terms ("==" or "IN" operators) that refer to the N @@ -102147,20 +116776,19 @@ static Bitmask codeOneLoopStart( 0, OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ OP_Last, /* 3: (!start_constraints && startEq && bRev) */ - OP_SeekGt, /* 4: (start_constraints && !startEq && !bRev) */ - OP_SeekLt, /* 5: (start_constraints && !startEq && bRev) */ - OP_SeekGe, /* 6: (start_constraints && startEq && !bRev) */ - OP_SeekLe /* 7: (start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ }; static const u8 aEndOp[] = { - OP_Noop, /* 0: (!end_constraints) */ - OP_IdxGE, /* 1: (end_constraints && !bRev) */ - OP_IdxLT /* 2: (end_constraints && bRev) */ + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ }; - int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ - int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ int regBase; /* Base register holding constraint values */ - int r1; /* Temp register */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ int startEq; /* True if range start uses ==, >= or <= */ @@ -102172,11 +116800,13 @@ static Bitmask codeOneLoopStart( int nExtraReg = 0; /* Number of extra registers needed */ int op; /* Instruction opcode */ char *zStartAff; /* Affinity for start of range constraint */ - char *zEndAff; /* Affinity for end of range constraint */ + char cEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ - pIdx = pLevel->plan.u.pIdx; + pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; - k = pIdx->aiColumn[nEq]; /* Column for inequality constraints */ + assert( nEq>=pLoop->u.btree.nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -102186,50 +116816,62 @@ static Bitmask codeOneLoopStart( ** the first one after the nEq equality constraints in the index, ** this requires some special handling. */ - if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pLevel->plan.wsFlags&WHERE_ORDERBY) - && (pIdx->nColumn>nEq) + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && pWInfo->nOBSat>0 + && (pIdx->nKeyCol>nEq) ){ - /* assert( pOrderBy->nExpr==1 ); */ - /* assert( pOrderBy->a[0].pExpr->iColumn==pIdx->aiColumn[nEq] ); */ - isMinQuery = 1; + assert( pLoop->u.btree.nSkip==0 ); + bSeekPastNull = 1; nExtraReg = 1; } /* Find any inequality constraint terms for the start and end ** of the range. */ - if( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = findTerm(pWC, iCur, k, notReady, (WO_LT|WO_LE), pIdx); + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; nExtraReg = 1; } - if( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = findTerm(pWC, iCur, k, notReady, (WO_GT|WO_GE), pIdx); + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; nExtraReg = 1; + if( pRangeStart==0 + && (j = pIdx->aiColumn[nEq])>=0 + && pIdx->pTable->aCol[j].notNull==0 + ){ + bSeekPastNull = 1; + } } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); /* Generate code to evaluate all constraint terms using == or IN ** and store the values of those terms in an array of registers ** starting at regBase. */ - regBase = codeAllEqualityTerms( - pParse, pLevel, pWC, notReady, nExtraReg, &zStartAff - ); - zEndAff = sqlite3DbStrDup(pParse->db, zStartAff); + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff ) cEndAff = zStartAff[nEq]; addrNxt = pLevel->addrNxt; /* If we are doing a reverse order scan on an ascending index, or ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). */ - if( nEqnColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC) ){ + if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) + || (bRev && pIdx->nKeyCol==nEq) + ){ SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); } - testcase( pRangeStart && pRangeStart->eOperator & WO_LE ); - testcase( pRangeStart && pRangeStart->eOperator & WO_GE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_LE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_GE ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); start_constraints = pRangeStart || nEq>0; @@ -102239,8 +116881,11 @@ static Bitmask codeOneLoopStart( if( pRangeStart ){ Expr *pRight = pRangeStart->pExpr->pRight; sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeStart->wtFlags & TERM_VNULL)==0 ){ - sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); } if( zStartAff ){ if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ @@ -102254,23 +116899,24 @@ static Bitmask codeOneLoopStart( } } nConstraint++; - testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - }else if( isMinQuery ){ + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + }else if( bSeekPastNull ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); nConstraint++; startEq = 0; start_constraints = 1; } - codeApplyAffinity(pParse, regBase, nConstraint, zStartAff); + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); - testcase( op==OP_Rewind ); - testcase( op==OP_Last ); - testcase( op==OP_SeekGt ); - testcase( op==OP_SeekGe ); - testcase( op==OP_SeekLe ); - testcase( op==OP_SeekLt ); sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); /* Load the value for the inequality constraint at the end of the ** range (if any). @@ -102280,67 +116926,64 @@ static Bitmask codeOneLoopStart( Expr *pRight = pRangeEnd->pExpr->pRight; sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeEnd->wtFlags & TERM_VNULL)==0 ){ - sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_NONE + && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) + ){ + codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); } - if( zEndAff ){ - if( sqlite3CompareAffinity(pRight, zEndAff[nEq])==SQLITE_AFF_NONE){ - /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zEndAff[nEq] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zEndAff[nEq]) ){ - zEndAff[nEq] = SQLITE_AFF_NONE; - } - } - codeApplyAffinity(pParse, regBase, nEq+1, zEndAff); nConstraint++; - testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + }else if( bStopAtNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + nConstraint++; } - sqlite3DbFree(pParse->db, zStartAff); - sqlite3DbFree(pParse->db, zEndAff); + sqlite3DbFree(db, zStartAff); /* Top of the loop body */ pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ - op = aEndOp[(pRangeEnd || nEq) * (1 + bRev)]; - testcase( op==OP_Noop ); - testcase( op==OP_IdxGE ); - testcase( op==OP_IdxLT ); - if( op!=OP_Noop ){ + if( nConstraint ){ + op = aEndOp[bRev*2 + endEq]; sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } - /* If there are inequality constraints, check that the value - ** of the table column that the inequality contrains is not NULL. - ** If it is, jump to the next iteration of the loop. - */ - r1 = sqlite3GetTempReg(pParse); - testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); - testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); - if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); - sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); - } - sqlite3ReleaseTempReg(pParse, r1); - /* Seek the table cursor, if required */ disableTerm(pLevel, pRangeStart); disableTerm(pLevel, pRangeEnd); - if( !omitTable ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + if( omitTable ){ + /* pIdx is a covering index. No need to access the main table. */ + }else if( HasRowid(pIdx->pTable) ){ + iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ + }else if( iCur!=iIdxCur ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } /* Record the instruction used to terminate the loop. Disable ** WHERE clause terms made redundant by the index range scan. */ - if( pLevel->plan.wsFlags & WHERE_UNIQUE ){ + if( pLoop->wsFlags & WHERE_ONEROW ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -102348,11 +116991,17 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } }else #ifndef SQLITE_OMIT_OR_OPTIMIZATION - if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ - /* Case 4: Two or more separately indexed terms connected by OR + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR ** ** Example: ** @@ -102390,9 +117039,15 @@ static Bitmask codeOneLoopStart( ** ** B: ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephemeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** */ WhereClause *pOrWc; /* The OR-clause broken out into subterms */ SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ int regRowset = 0; /* Register for RowSet object */ @@ -102400,17 +117055,20 @@ static Bitmask codeOneLoopStart( int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ - int ii; + int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; - pTerm = pLevel->plan.u.pTerm; + pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); - assert( pTerm->eOperator==WO_OR ); + assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); pOrWc = &pTerm->u.pOrInfo->wc; pLevel->op = OP_Return; pLevel->p1 = regReturn; - /* Set up a new SrcList ni pOrTab containing the table being scanned + /* Set up a new SrcList in pOrTab containing the table being scanned ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). */ @@ -102418,10 +117076,10 @@ static Bitmask codeOneLoopStart( int nNotReady; /* The number of notReady tables */ struct SrcList_item *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(pParse->db, + pOrTab = sqlite3StackAllocRaw(db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (i16)(nNotReady + 1); + pOrTab->nAlloc = (u8)(nNotReady + 1); pOrTab->nSrc = pOrTab->nAlloc; memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); origSrc = pWInfo->pTabList->a; @@ -102433,7 +117091,8 @@ static Bitmask codeOneLoopStart( } /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. + ** equivalent to an empty rowset. Or, create an ephemeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. ** ** Also initialize regReturn to contain the address of the instruction ** immediately following the OP_Return at the bottom of the loop. This @@ -102443,35 +117102,136 @@ static Bitmask codeOneLoopStart( ** fall through to the next instruction, just as an OP_Next does if ** called on an uninitialized cursor. */ - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - regRowset = ++pParse->nMem; + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } regRowid = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); } iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 + */ + if( pWC->nTerm>1 ){ + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); + if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + pExpr = sqlite3ExprDup(db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + } + if( pAndExpr ){ + pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); + } + } + + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE + | WHERE_FORCE_TABLE + | WHERE_ONETABLE_ONLY; for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ - WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + int j1 = 0; /* Address of jump operation */ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; + } /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, - WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + wctrlFlags, iCovCur); + assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ + WhereLoop *pSubLoop; explainOneScan( pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 ); - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ int r; - r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid); - sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, - sqlite3VdbeCurrentAddr(v)+2, r, iSet); + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0); + } + + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } } + + /* Invoke the main loop body as a subroutine */ sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( j1 ) sqlite3VdbeJumpHere(v, j1); + /* The pSubWInfo->untestedTerms flag means that this OR term ** contained one or more AND term from a notReady table. The ** terms from the notReady table could not be tested and will @@ -102479,48 +117239,81 @@ static Bitmask codeOneLoopStart( */ if( pSubWInfo->untestedTerms ) untestedTerms = 1; + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite3WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite3WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; + }else{ + pCov = 0; + } + /* Finish the loop through table entries that match term pOrTerm. */ sqlite3WhereEnd(pSubWInfo); } } } + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(db, pAndExpr); + } sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); - if( pWInfo->nLevel>1 ) sqlite3StackFree(pParse->db, pOrTab); + if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ { - /* Case 5: There is no usable index. We must do a complete + /* Case 6: There is no usable index. We must do a complete ** scan of the entire table. */ static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - assert( omitTable==0 ); - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + if( pTabItem->isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } } - notReady &= ~getMask(pWC->pMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. - ** - ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through - ** the use of indices become tests that are evaluated against each row of - ** the relevant input tables. */ for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ Expr *pE; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); pWInfo->untestedTerms = 1; @@ -102535,6 +117328,39 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE, *pEAlt; + WhereTerm *pAlt; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pLevel->iLeftJoin ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IN ); + VdbeModuleComment((v, "begin transitive constraint")); + pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); + if( pEAlt ){ + *pEAlt = *pAlt->pExpr; + pEAlt->pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); + sqlite3StackFree(db, pEAlt); + } + } + /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ @@ -102544,10 +117370,10 @@ static Bitmask codeOneLoopStart( VdbeComment((v, "record LEFT JOIN hit")); sqlite3ExprCacheClear(pParse); for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; } @@ -102556,52 +117382,2171 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } } - sqlite3ReleaseTempReg(pParse, iReleaseReg); - return notReady; + return pLevel->notReady; } -#if defined(SQLITE_TEST) +#ifdef WHERETRACE_ENABLED /* -** The following variable holds a text description of query plan generated -** by the most recent call to sqlite3WhereBegin(). Each call to WhereBegin -** overwrites the previous. This information is used for testing and -** analysis only. +** Print the content of a WhereTerm object */ -SQLITE_API char sqlite3_query_plan[BMS*2*40]; /* Text of the join */ -static int nQPlan = 0; /* Next free slow in _query_plan[] */ +static void whereTermPrint(WhereTerm *pTerm, int iTerm){ + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + }else{ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", + iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, + pTerm->eOperator); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + } +} +#endif -#endif /* SQLITE_TEST */ +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+7)/8; + struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ + int i = sqlite3Strlen30(zName) - 1; + while( zName[i]!='_' ) i--; + zName += i; + } + sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); + }else{ + sqlite3DebugPrintf("%20s",""); + } + }else{ + char *z; + if( p->u.vtab.idxStr ){ + z = sqlite3_mprintf("(%d,\"%s\",%x)", + p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); + }else{ + z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); + } + sqlite3DebugPrintf(" %-19s", z); + sqlite3_free(z); + } + if( p->wsFlags & WHERE_SKIPSCAN ){ + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip); + }else{ + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + } + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + int i; + for(i=0; inLTerm; i++){ + whereTermPrint(p->aLTerm[i], i); + } + } +} +#endif +/* +** Convert bulk memory into a valid WhereLoop that can be passed +** to whereLoopClear harmlessly. +*/ +static void whereLoopInit(WhereLoop *p){ + p->aLTerm = p->aLTermSpace; + p->nLTerm = 0; + p->nLSlot = ArraySize(p->aLTermSpace); + p->wsFlags = 0; +} + +/* +** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. +*/ +static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ + if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ + sqlite3_free(p->u.vtab.idxStr); + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, p->u.btree.pIndex->zColAff); + sqlite3KeyInfoUnref(p->u.btree.pIndex->pKeyInfo); + sqlite3DbFree(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; + } + } +} + +/* +** Deallocate internal memory used by a WhereLoop object +*/ +static void whereLoopClear(sqlite3 *db, WhereLoop *p){ + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + whereLoopClearUnion(db, p); + whereLoopInit(p); +} + +/* +** Increase the memory allocation for pLoop->aLTerm[] to be at least n. +*/ +static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ + WhereTerm **paNew; + if( p->nLSlot>=n ) return SQLITE_OK; + n = (n+7)&~7; + paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM; + memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + p->aLTerm = paNew; + p->nLSlot = n; + return SQLITE_OK; +} + +/* +** Transfer content from the second pLoop into the first. +*/ +static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ + whereLoopClearUnion(db, pTo); + if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + memset(&pTo->u, 0, sizeof(pTo->u)); + return SQLITE_NOMEM; + } + memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); + if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ + pFrom->u.vtab.needFree = 0; + }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + pFrom->u.btree.pIndex = 0; + } + return SQLITE_OK; +} + +/* +** Delete a WhereLoop object +*/ +static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + whereLoopClear(db, p); + sqlite3DbFree(db, p); +} /* ** Free a WhereInfo structure */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - sqlite3_index_info *pInfo = pWInfo->a[i].pIdxInfo; - if( pInfo ){ - /* assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed ); */ - if( pInfo->needToFreeIdxStr ){ - sqlite3_free(pInfo->idxStr); - } - sqlite3DbFree(db, pInfo); - } - if( pWInfo->a[i].plan.wsFlags & WHERE_TEMP_INDEX ){ - Index *pIdx = pWInfo->a[i].plan.u.pIdx; - if( pIdx ){ - sqlite3DbFree(db, pIdx->zColAff); - sqlite3DbFree(db, pIdx); - } - } + whereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); } - whereClauseClear(pWInfo->pWC); sqlite3DbFree(db, pWInfo); } } +/* +** Return TRUE if both of the following are true: +** +** (1) X has the same or lower cost that Y +** (2) X is a proper subset of Y +** +** By "proper subset" we mean that X uses fewer WHERE clause terms +** than Y and that every WHERE clause term used by X is also used +** by Y. +** +** If X is a proper subset of Y then Y is a better choice and ought +** to have a lower cost. This routine returns TRUE when that cost +** relationship is inverted and needs to be adjusted. +*/ +static int whereLoopCheaperProperSubset( + const WhereLoop *pX, /* First WhereLoop to compare */ + const WhereLoop *pY /* Compare against this WhereLoop */ +){ + int i, j; + if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->rRun >= pY->rRun ){ + if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ + if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ + } + for(i=pX->nLTerm-1; i>=0; i--){ + for(j=pY->nLTerm-1; j>=0; j--){ + if( pY->aLTerm[j]==pX->aLTerm[i] ) break; + } + if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + } + return 1; /* All conditions meet */ +} + +/* +** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so +** that: +** +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate +** +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. +** +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. +** +** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the +** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE +** clause terms covered, since some of the first nLTerm entries in aLTerm[] +** will be NULL (because they are skipped). That makes it more difficult +** to compare the loops. We could add extra code to do the comparison, and +** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this +** adjustment is sufficient minor, that it is very difficult to construct +** a test case where the extra code would improve the query plan. Better +** to avoid the added complexity and just omit cost adjustments to SKIPSCAN +** loops. +*/ +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; + if( whereLoopCheaperProperSubset(p, pTemplate) ){ + /* Adjust pTemplate cost downward so that it is cheaper than its + ** subset p */ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ + /* Adjust pTemplate cost upward so that it is costlier than p since + ** pTemplate is a proper subset of p */ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut + 1; + } + } +} + +/* +** Search the list of WhereLoops in *ppPrev looking for one that can be +** supplanted by pTemplate. +** +** Return NULL if the WhereLoop list contains an entry that can supplant +** pTemplate, in other words if pTemplate does not belong on the list. +** +** If pX is a WhereLoop that pTemplate can supplant, then return the +** link that points to pX. +** +** If pTemplate cannot supplant any existing element of the list but needs +** to be added to the list, then return a pointer to the tail of the list. +*/ +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; + } + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); + + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); + + /* Any loop using an appliation-defined index (or PRIMARY KEY or + ** UNIQUE constraint) with one or more == constraints is better + ** than an automatic index. */ + if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 + && (p->prereq & pTemplate->prereq)==pTemplate->prereq + ){ + break; + } + + /* If existing WhereLoop p is better than pTemplate, pTemplate can be + ** discarded. WhereLoop p is better if: + ** (1) p has no more dependencies than pTemplate, and + ** (2) p has an equal or lower cost than pTemplate + */ + if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ + && p->rSetup<=pTemplate->rSetup /* (2a) */ + && p->rRun<=pTemplate->rRun /* (2b) */ + && p->nOut<=pTemplate->nOut /* (2c) */ + ){ + return 0; /* Discard pTemplate */ + } + + /* If pTemplate is always better than p, then cause p to be overwritten + ** with pTemplate. pTemplate is better than p if: + ** (1) pTemplate has no more dependences than p, and + ** (2) pTemplate has an equal or lower cost than p. + */ + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ + && p->rRun>=pTemplate->rRun /* (2a) */ + && p->nOut>=pTemplate->nOut /* (2b) */ + ){ + assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ + break; /* Cause p to be overwritten by pTemplate */ + } + } + return ppPrev; +} + +/* +** Insert or replace a WhereLoop entry using the template supplied. +** +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based on the template. +** +** If pBuilder->pOrSet is not NULL then we care about only the +** prerequisites and rRun and nOut costs of the N best loops. That +** information is gathered in the pBuilder->pOrSet object. This special +** processing mode is used only for OR clause processing. +** +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we +** still might overwrite similar loops with the new template if the +** new template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop +*/ +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p; + WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite3 *db = pWInfo->pParse->db; + + /* If pBuilder->pOrSet is defined, then only keep track of the costs + ** and prereqs. + */ + if( pBuilder->pOrSet!=0 ){ +#if WHERETRACE_ENABLED + u16 n = pBuilder->pOrSet->n; + int x = +#endif + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + pTemplate->nOut); +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + } + + /* Look for an existing WhereLoop to replace with pTemplate + */ + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); + ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); + + if( ppPrev==0 ){ + /* There already exists a WhereLoop on the list that is better + ** than pTemplate, so just ignore pTemplate */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(" skip: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + }else{ + p = *ppPrev; + } + + /* If we reach this point it means that either p[] should be overwritten + ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new + ** WhereLoop and insert it. + */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + if( p!=0 ){ + sqlite3DebugPrintf("replace: "); + whereLoopPrint(p, pBuilder->pWC); + } + sqlite3DebugPrintf(" add: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + if( p==0 ){ + /* Allocate a new WhereLoop to add to the end of the list */ + *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE_NOMEM; + whereLoopInit(p); + p->pNextLoop = 0; + }else{ + /* We will be overwriting WhereLoop p[]. But before we do, first + ** go through the rest of the list and delete any other entries besides + ** p[] that are also supplated by pTemplate */ + WhereLoop **ppTail = &p->pNextLoop; + WhereLoop *pToDel; + while( *ppTail ){ + ppTail = whereLoopFindLesser(ppTail, pTemplate); + if( ppTail==0 ) break; + pToDel = *ppTail; + if( pToDel==0 ) break; + *ppTail = pToDel->pNextLoop; +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(" delete: "); + whereLoopPrint(pToDel, pBuilder->pWC); + } +#endif + whereLoopDelete(db, pToDel); + } + } + whereLoopXfer(db, p, pTemplate); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + Index *pIndex = p->u.btree.pIndex; + if( pIndex && pIndex->tnum==0 ){ + p->u.btree.pIndex = 0; + } + } + return SQLITE_OK; +} + +/* +** Adjust the WhereLoop.nOut value downward to account for terms of the +** WHERE clause that reference the loop but which are not used by an +** index. +** +** In the current implementation, the first extra WHERE clause term reduces +** the number of output rows by a factor of 10 and each additional term +** reduces the number of output rows by sqrt(2). +*/ +static void whereLoopOutputAdjust( + WhereClause *pWC, /* The WHERE clause */ + WhereLoop *pLoop, /* The loop to adjust downward */ + LogEst nRow /* Number of rows in the entire table */ +){ + WhereTerm *pTerm, *pX; + Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); + int i, j; + int nEq = 0; /* Number of = constraints not within likely()/unlikely() */ + + for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; + if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + if( (pTerm->prereqAll & notAllowed)!=0 ) continue; + for(j=pLoop->nLTerm-1; j>=0; j--){ + pX = pLoop->aLTerm[j]; + if( pX==0 ) continue; + if( pX==pTerm ) break; + if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; + } + if( j<0 ){ + if( pTerm->truthProb<=0 ){ + pLoop->nOut += pTerm->truthProb; + }else{ + pLoop->nOut--; + if( pTerm->eOperator&WO_EQ ) nEq++; + } + } + } + /* TUNING: If there is at least one equality constraint in the WHERE + ** clause that does not have a likelihood() explicitly assigned to it + ** then do not let the estimated number of output rows exceed half + ** the number of rows in the table. */ + if( nEq && pLoop->nOut>nRow-10 ){ + pLoop->nOut = nRow - 10; + } +} + +/* +** Adjust the cost C by the costMult facter T. This only occurs if +** compiled with -DSQLITE_ENABLE_COSTMULT +*/ +#ifdef SQLITE_ENABLE_COSTMULT +# define ApplyCostMultiplier(C,T) C += T +#else +# define ApplyCostMultiplier(C,T) +#endif + +/* +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. +** +** If pProbe->tnum==0, that means pIndex is a fake index used for the +** INTEGER PRIMARY KEY. +*/ +static int whereLoopAddBtreeIndex( + WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ + struct SrcList_item *pSrc, /* FROM clause term being analyzed */ + Index *pProbe, /* An index on pSrc */ + LogEst nInMul /* log(Number of iterations due to IN) */ +){ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection malloc context */ + WhereLoop *pNew; /* Template WhereLoop under construction */ + WhereTerm *pTerm; /* A WhereTerm under consideration */ + int opMask; /* Valid operators for constraints */ + WhereScan scan; /* Iterator for WHERE terms */ + Bitmask saved_prereq; /* Original value of pNew->prereq */ + u16 saved_nLTerm; /* Original value of pNew->nLTerm */ + u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u32 saved_wsFlags; /* Original value of pNew->wsFlags */ + LogEst saved_nOut; /* Original value of pNew->nOut */ + int iCol; /* Index of the column in the table */ + int rc = SQLITE_OK; /* Return code */ + LogEst rSize; /* Number of rows in the table */ + LogEst rLogSize; /* Logarithm of table size */ + WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ + + pNew = pBuilder->pNew; + if( db->mallocFailed ) return SQLITE_NOMEM; + + assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); + if( pNew->wsFlags & WHERE_BTM_LIMIT ){ + opMask = WO_LT|WO_LE; + }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; + }else{ + opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; + } + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + + assert( pNew->u.btree.nEqnColumn ); + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, + opMask, pProbe); + saved_nEq = pNew->u.btree.nEq; + saved_nSkip = pNew->u.btree.nSkip; + saved_nLTerm = pNew->nLTerm; + saved_wsFlags = pNew->wsFlags; + saved_prereq = pNew->prereq; + saved_nOut = pNew->nOut; + pNew->rSetup = 0; + rSize = pProbe->aiRowLogEst[0]; + rLogSize = estLog(rSize); + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->u.btree.nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + if( pTerm ){ + /* TUNING: When estimating skip-scan for a term that is also indexable, + ** multiply the cost of the skip-scan by 2.0, to make it a little less + ** desirable than the regular index lookup. */ + nIter += 10; assert( 10==sqlite3LogEst(2) ); + } + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nSkip = saved_nSkip; + } + for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ + LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ + int nIn = 0; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int nRecValid = pBuilder->nRecValid; +#endif + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + ){ + continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ + } + if( pTerm->prereqRight & pNew->maskSelf ) continue; + + pNew->wsFlags = saved_wsFlags; + pNew->u.btree.nEq = saved_nEq; + pNew->nLTerm = saved_nLTerm; + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTerm; + pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + pNew->wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + nIn = 46; assert( 46==sqlite3LogEst(25) ); + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = sqlite3LogEst(pExpr->x.pList->nExpr); + } + assert( nIn>0 ); /* RHS always has 2 or more terms... The parser + ** changes "x IN (?)" into "x=?". */ + + }else if( eOp & (WO_EQ) ){ + pNew->wsFlags |= WHERE_COLUMN_EQ; + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ + if( iCol>=0 && !IsUniqueIndex(pProbe) ){ + pNew->wsFlags |= WHERE_UNQ_WANTED; + }else{ + pNew->wsFlags |= WHERE_ONEROW; + } + } + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pBtm = pTerm; + pTop = 0; + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ + whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 && iCol>=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + }else{ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } + } + } + + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); + } + ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); + + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; + whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } + + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 + && pNew->u.btree.nEqnColumn + ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); + } + pNew->nOut = saved_nOut; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + pBuilder->nRecValid = nRecValid; +#endif + } + pNew->prereq = saved_prereq; + pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + pNew->nOut = saved_nOut; + pNew->nLTerm = saved_nLTerm; + return rc; +} + +/* +** Return True if it is possible that pIndex might be useful in +** implementing the ORDER BY clause in pBuilder. +** +** Return False if pBuilder does not contain an ORDER BY clause or +** if there is no way for pIndex to be useful in implementing that +** ORDER BY clause. +*/ +static int indexMightHelpWithOrderBy( + WhereLoopBuilder *pBuilder, + Index *pIndex, + int iCursor +){ + ExprList *pOB; + int ii, jj; + + if( pIndex->bUnordered ) return 0; + if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); + if( pExpr->op!=TK_COLUMN ) return 0; + if( pExpr->iTable==iCursor ){ + if( pExpr->iColumn<0 ) return 1; + for(jj=0; jjnKeyCol; jj++){ + if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; + } + } + } + return 0; +} + +/* +** Return a bitmask where 1s indicate that the corresponding column of +** the table is used by an index. Only the first 63 columns are considered. +*/ +static Bitmask columnsInIndex(Index *pIdx){ + Bitmask m = 0; + int j; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xa; inTerm; i++, pTerm++){ + if( sqlite3ExprImpliesExpr(pTerm->pExpr, pWhere, iTab) ) return 1; + } + return 0; +} + +/* +** Add all WhereLoop objects for a single table of the join where the table +** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be +** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. +** +** The estimated values (nRow, nVisit, nSeek) often contain a large amount +** of uncertainty. For this reason, scoring is designed to pick plans that +** "do the least harm" if the estimates are inaccurate. For example, a +** log(nRow) factor is omitted from a non-covering index scan in order to +** bias the scoring in favor of using an index, since the worst-case +** performance of using an index is far better than the worst-case performance +** of a full table scan. +*/ +static int whereLoopAddBtree( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mExtra /* Extra prerequesites for using this table */ +){ + WhereInfo *pWInfo; /* WHERE analysis context */ + Index *pProbe; /* An index we are evaluating */ + Index sPk; /* A fake index object for the primary key */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ + i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + SrcList *pTabList; /* The FROM clause */ + struct SrcList_item *pSrc; /* The FROM clause btree term to add */ + WhereLoop *pNew; /* Template WhereLoop object */ + int rc = SQLITE_OK; /* Return code */ + int iSortIdx = 1; /* Index number */ + int b; /* A boolean value */ + LogEst rSize; /* number of rows in the table */ + LogEst rLogSize; /* Logarithm of the number of rows in the table */ + WhereClause *pWC; /* The parsed WHERE clause */ + Table *pTab; /* Table being queried */ + + pNew = pBuilder->pNew; + pWInfo = pBuilder->pWInfo; + pTabList = pWInfo->pTabList; + pSrc = pTabList->a + pNew->iTab; + pTab = pSrc->pTab; + pWC = pBuilder->pWC; + assert( !IsVirtual(pSrc->pTab) ); + + if( pSrc->pIndex ){ + /* An INDEXED BY clause specifies a particular index to use */ + pProbe = pSrc->pIndex; + }else if( !HasRowid(pTab) ){ + pProbe = pTab->pIndex; + }else{ + /* There is no INDEXED BY clause. Create a fake Index object in local + ** variable sPk to represent the rowid primary key index. Make this + ** fake index the first in a chain of Index objects with all of the real + ** indices to follow */ + Index *pFirst; /* First of real indices on the table */ + memset(&sPk, 0, sizeof(Index)); + sPk.nKeyCol = 1; + sPk.nColumn = 1; + sPk.aiColumn = &aiColumnPk; + sPk.aiRowLogEst = aiRowEstPk; + sPk.onError = OE_Replace; + sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; + pFirst = pSrc->pTab->pIndex; + if( pSrc->notIndexed==0 ){ + /* The real indices of the table are only considered if the + ** NOT INDEXED qualifier is omitted from the FROM clause */ + sPk.pNext = pFirst; + } + pProbe = &sPk; + } + rSize = pTab->nRowLogEst; + rLogSize = estLog(rSize); + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + /* Automatic indexes */ + if( !pBuilder->pOrSet + && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 + && pSrc->pIndex==0 + && !pSrc->viaCoroutine + && !pSrc->notIndexed + && HasRowid(pTab) + && !pSrc->isCorrelated + && !pSrc->isRecursive + ){ + /* Generate auto-index WhereLoops */ + WhereTerm *pTerm; + WhereTerm *pWCEnd = pWC->a + pWC->nTerm; + for(pTerm=pWC->a; rc==SQLITE_OK && pTermprereqRight & pNew->maskSelf ) continue; + if( termCanDriveIndex(pTerm, pSrc, 0) ){ + pNew->u.btree.nEq = 1; + pNew->u.btree.nSkip = 0; + pNew->u.btree.pIndex = 0; + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + /* TUNING: One-time cost for computing the automatic index is + ** estimated to be X*N*log2(N) where N is the number of rows in + ** the table being indexed and where X is 7 (LogEst=28) for normal + ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** of X is smaller for views and subqueries so that the query planner + ** will be more aggressive about generating automatic indexes for + ** those objects, since there is no opportunity to add schema + ** indexes on subqueries and views. */ + pNew->rSetup = rLogSize + rSize + 4; + if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 24; + } + ApplyCostMultiplier(pNew->rSetup, pTab->costMult); + /* TUNING: Each index lookup yields 20 rows in the table. This + ** is more than the usual guess of 10 rows, since we have no way + ** of knowing how selective the index will ultimately be. It would + ** not be unreasonable to make this value much larger. */ + pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); + pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); + pNew->wsFlags = WHERE_AUTO_INDEX; + pNew->prereq = mExtra | pTerm->prereqRight; + rc = whereLoopInsert(pBuilder, pNew); + } + } + } +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + + /* Loop over all indices + */ + for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ + if( pProbe->pPartIdxWhere!=0 + && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ + testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ + continue; /* Partial index inappropriate for this query */ + } + rSize = pProbe->aiRowLogEst[0]; + pNew->u.btree.nEq = 0; + pNew->u.btree.nSkip = 0; + pNew->nLTerm = 0; + pNew->iSortIdx = 0; + pNew->rSetup = 0; + pNew->prereq = mExtra; + pNew->nOut = rSize; + pNew->u.btree.pIndex = pProbe; + b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); + /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ + assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); + if( pProbe->tnum<=0 ){ + /* Integer primary key index */ + pNew->wsFlags = WHERE_IPK; + + /* Full table scan */ + pNew->iSortIdx = b ? iSortIdx : 0; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + }else{ + Bitmask m; + if( pProbe->isCovering ){ + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; + m = 0; + }else{ + m = pSrc->colUsed & ~columnsInIndex(pProbe); + pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; + } + + /* Full scan via index */ + if( b + || !HasRowid(pTab) + || ( m==0 + && pProbe->bUnordered==0 + && (pProbe->szIdxRowszTabRow) + && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) + ) + ){ + pNew->iSortIdx = b ? iSortIdx : 0; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); + } + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + } + } + + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3Stat4ProbeFree(pBuilder->pRec); + pBuilder->nRecValid = 0; + pBuilder->pRec = 0; +#endif + + /* If there was an INDEXED BY clause, then only that one index is + ** considered. */ + if( pSrc->pIndex ) break; + } + return rc; +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Add all WhereLoop objects for a table of the join identified by +** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. +*/ +static int whereLoopAddVirtual( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mExtra +){ + WhereInfo *pWInfo; /* WHERE analysis context */ + Parse *pParse; /* The parsing context */ + WhereClause *pWC; /* The WHERE clause */ + struct SrcList_item *pSrc; /* The FROM clause term to search */ + Table *pTab; + sqlite3 *db; + sqlite3_index_info *pIdxInfo; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int i, j; + int iTerm, mxTerm; + int nConstraint; + int seenIn = 0; /* True if an IN operator is seen */ + int seenVar = 0; /* True if a non-constant constraint is seen */ + int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ + WhereLoop *pNew; + int rc = SQLITE_OK; + + pWInfo = pBuilder->pWInfo; + pParse = pWInfo->pParse; + db = pParse->db; + pWC = pBuilder->pWC; + pNew = pBuilder->pNew; + pSrc = &pWInfo->pTabList->a[pNew->iTab]; + pTab = pSrc->pTab; + assert( IsVirtual(pTab) ); + pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); + if( pIdxInfo==0 ) return SQLITE_NOMEM; + pNew->prereq = 0; + pNew->rSetup = 0; + pNew->wsFlags = WHERE_VIRTUALTABLE; + pNew->nLTerm = 0; + pNew->u.vtab.needFree = 0; + pUsage = pIdxInfo->aConstraintUsage; + nConstraint = pIdxInfo->nConstraint; + if( whereLoopResize(db, pNew, nConstraint) ){ + sqlite3DbFree(db, pIdxInfo); + return SQLITE_NOMEM; + } + + for(iPhase=0; iPhase<=3; iPhase++){ + if( !seenIn && (iPhase&1)!=0 ){ + iPhase++; + if( iPhase>3 ) break; + } + if( !seenVar && iPhase>1 ) break; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + switch( iPhase ){ + case 0: /* Constants without IN operator */ + pIdxCons->usable = 0; + if( (pTerm->eOperator & WO_IN)!=0 ){ + seenIn = 1; + } + if( pTerm->prereqRight!=0 ){ + seenVar = 1; + }else if( (pTerm->eOperator & WO_IN)==0 ){ + pIdxCons->usable = 1; + } + break; + case 1: /* Constants with IN operators */ + assert( seenIn ); + pIdxCons->usable = (pTerm->prereqRight==0); + break; + case 2: /* Variables without IN */ + assert( seenVar ); + pIdxCons->usable = (pTerm->eOperator & WO_IN)==0; + break; + default: /* Variables with IN */ + assert( seenVar && seenIn ); + pIdxCons->usable = 1; + break; + } + } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; + pIdxInfo->estimatedRows = 25; + rc = vtabBestIndex(pParse, pTab, pIdxInfo); + if( rc ) goto whereLoopAddVtab_exit; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pNew->prereq = mExtra; + mxTerm = -1; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; iaLTerm[i] = 0; + pNew->u.vtab.omitMask = 0; + for(i=0; i=0 ){ + j = pIdxCons->iTermOffset; + if( iTerm>=nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aLTerm[iTerm]!=0 + ){ + rc = SQLITE_ERROR; + sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); + goto whereLoopAddVtab_exit; + } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + assert( iTermnLSlot ); + pNew->aLTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); + if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + } + } + } + if( i>=nConstraint ){ + pNew->nLTerm = mxTerm+1; + assert( pNew->nLTerm<=pNew->nLSlot ); + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); + pNew->rSetup = 0; + pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); + pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + whereLoopInsert(pBuilder, pNew); + if( pNew->u.vtab.needFree ){ + sqlite3_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; + } + } + } + +whereLoopAddVtab_exit: + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + sqlite3DbFree(db, pIdxInfo); + return rc; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Add WhereLoop entries to handle OR terms. This works for either +** btrees or virtual tables. +*/ +static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ + WhereInfo *pWInfo = pBuilder->pWInfo; + WhereClause *pWC; + WhereLoop *pNew; + WhereTerm *pTerm, *pWCEnd; + int rc = SQLITE_OK; + int iCur; + WhereClause tempWC; + WhereLoopBuilder sSubBuild; + WhereOrSet sSum, sCur; + struct SrcList_item *pItem; + + pWC = pBuilder->pWC; + pWCEnd = pWC->a + pWC->nTerm; + pNew = pBuilder->pNew; + memset(&sSum, 0, sizeof(sSum)); + pItem = pWInfo->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; + + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 + && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + ){ + WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; + WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; + WhereTerm *pOrTerm; + int once = 1; + int i, j; + + sSubBuild = *pBuilder; + sSubBuild.pOrderBy = 0; + sSubBuild.pOrSet = &sCur; + + WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ + sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; + }else if( pOrTerm->leftCursor==iCur ){ + tempWC.pWInfo = pWC->pWInfo; + tempWC.pOuter = pWC; + tempWC.op = TK_AND; + tempWC.nTerm = 1; + tempWC.a = pOrTerm; + sSubBuild.pWC = &tempWC; + }else{ + continue; + } + sCur.n = 0; +#ifdef WHERETRACE_ENABLED + WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); + if( sqlite3WhereTrace & 0x400 ){ + for(i=0; inTerm; i++){ + whereTermPrint(&sSubBuild.pWC->a[i], i); + } + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(&sSubBuild, mExtra); + }else +#endif + { + rc = whereLoopAddBtree(&sSubBuild, mExtra); + } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(&sSubBuild, mExtra); + } + assert( rc==SQLITE_OK || sCur.n==0 ); + if( sCur.n==0 ){ + sSum.n = 0; + break; + }else if( once ){ + whereOrMove(&sSum, &sCur); + once = 0; + }else{ + WhereOrSet sPrev; + whereOrMove(&sPrev, &sSum); + sSum.n = 0; + for(i=0; inLTerm = 1; + pNew->aLTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = 0; + pNew->iSortIdx = 0; + memset(&pNew->u, 0, sizeof(pNew->u)); + for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 1; + pNew->nOut = sSum.a[i].nOut; + pNew->prereq = sSum.a[i].prereq; + rc = whereLoopInsert(pBuilder, pNew); + } + WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); + } + } + return rc; +} + +/* +** Add all WhereLoop objects for all tables +*/ +static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo = pBuilder->pWInfo; + Bitmask mExtra = 0; + Bitmask mPrior = 0; + int iTab; + SrcList *pTabList = pWInfo->pTabList; + struct SrcList_item *pItem; + sqlite3 *db = pWInfo->pParse->db; + int nTabList = pWInfo->nLevel; + int rc = SQLITE_OK; + u8 priorJoinType = 0; + WhereLoop *pNew; + + /* Loop over the tables in the join, from left to right */ + pNew = pBuilder->pNew; + whereLoopInit(pNew); + for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; + pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); + if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ + mExtra = mPrior; + } + priorJoinType = pItem->jointype; + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(pBuilder, mExtra); + }else{ + rc = whereLoopAddBtree(pBuilder, mExtra); + } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(pBuilder, mExtra); + } + mPrior |= pNew->maskSelf; + if( rc || db->mallocFailed ) break; + } + whereLoopClear(db, pNew); + return rc; +} + +/* +** Examine a WherePath (with the addition of the extra WhereLoop of the 5th +** parameters) to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate sort operation. Return N: +** +** N>0: N terms of the ORDER BY clause are satisfied +** N==0: No terms of the ORDER BY clause are satisfied +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. +** +** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as +** strict. With GROUP BY and DISTINCT the only requirement is that +** equivalent rows appear immediately adjacent to one another. GROUP BY +** and DISTINCT do not require rows to appear in any particular order as long +** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT +** the pOrderBy terms can be matched in any order. With ORDER BY, the +** pOrderBy terms must be matched in strict left-to-right order. +*/ +static i8 wherePathSatisfiesOrderBy( + WhereInfo *pWInfo, /* The WHERE clause */ + ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ + WherePath *pPath, /* The WherePath to check */ + u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 nLoop, /* Number of entries in pPath->aLoop[] */ + WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ + Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ +){ + u8 revSet; /* True if rev is known */ + u8 rev; /* Composite sort order */ + u8 revIdx; /* Index sort order */ + u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ + u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ + u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 nKeyCol; /* Number of key columns in pIndex */ + u16 nColumn; /* Total number of ordered columns in the index */ + u16 nOrderBy; /* Number terms in the ORDER BY clause */ + int iLoop; /* Index of WhereLoop in pPath being processed */ + int i, j; /* Loop counters */ + int iCur; /* Cursor number for current WhereLoop */ + int iColumn; /* A column number within table iCur */ + WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + Expr *pOBExpr; /* An expression from the ORDER BY clause */ + CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ + Index *pIndex; /* The index associated with pLoop */ + sqlite3 *db = pWInfo->pParse->db; /* Database connection */ + Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ + Bitmask obDone; /* Mask of all ORDER BY terms */ + Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ + Bitmask ready; /* Mask of inner loops */ + + /* + ** We say the WhereLoop is "one-row" if it generates no more than one + ** row of output. A WhereLoop is one-row if all of the following are true: + ** (a) All index columns match with WHERE_COLUMN_EQ. + ** (b) The index is unique + ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. + ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. + ** + ** We say the WhereLoop is "order-distinct" if the set of columns from + ** that WhereLoop that are in the ORDER BY clause are different for every + ** row of the WhereLoop. Every one-row WhereLoop is automatically + ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause + ** is not order-distinct. To be order-distinct is not quite the same as being + ** UNIQUE since a UNIQUE column or index can have multiple rows that + ** are NULL and NULL values are equivalent for the purpose of order-distinct. + ** To be order-distinct, the columns must be UNIQUE and NOT NULL. + ** + ** The rowid for a table is always UNIQUE and NOT NULL so whenever the + ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is + ** automatically order-distinct. + */ + + assert( pOrderBy!=0 ); + if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; + + nOrderBy = pOrderBy->nExpr; + testcase( nOrderBy==BMS-1 ); + if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ + isOrderDistinct = 1; + obDone = MASKBIT(nOrderBy)-1; + orderDistinctMask = 0; + ready = 0; + for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; + pLoop = iLoopaLoop[iLoop] : pLast; + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + if( pLoop->u.vtab.isOrdered ) obSat = obDone; + break; + } + iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; + + /* Mark off any ORDER BY term X that is a column in the table of + ** the current loop for which there is term in the WHERE + ** clause of the form X IS NULL or X=? that reference only outer + ** loops. + */ + for(i=0; ia[i].pExpr); + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, WO_EQ|WO_ISNULL, 0); + if( pTerm==0 ) continue; + if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ + const char *z1, *z2; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + z1 = pColl->zName; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr); + if( !pColl ) pColl = db->pDfltColl; + z2 = pColl->zName; + if( sqlite3StrICmp(z1, z2)!=0 ) continue; + } + obSat |= MASKBIT(i); + } + + if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ + if( pLoop->wsFlags & WHERE_IPK ){ + pIndex = 0; + nKeyCol = 0; + nColumn = 1; + }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ + return 0; + }else{ + nKeyCol = pIndex->nKeyCol; + nColumn = pIndex->nColumn; + assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); + assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + isOrderDistinct = IsUniqueIndex(pIndex); + } + + /* Loop through all columns of the index and deal with the ones + ** that are not constrained by == or IN. + */ + rev = revSet = 0; + distinctColumns = 0; + for(j=0; ju.btree.nEq + && pLoop->u.btree.nSkip==0 + && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + ){ + if( i & WO_ISNULL ){ + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } + continue; + } + + /* Get the column number in the table (iColumn) and sort order + ** (revIdx) for the j-th column of the index. + */ + if( pIndex ){ + iColumn = pIndex->aiColumn[j]; + revIdx = pIndex->aSortOrder[j]; + if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; + }else{ + iColumn = -1; + revIdx = 0; + } + + /* An unconstrained column that might be NULL means that this + ** WhereLoop is not well-ordered + */ + if( isOrderDistinct + && iColumn>=0 + && j>=pLoop->u.btree.nEq + && pIndex->pTable->aCol[iColumn].notNull==0 + ){ + isOrderDistinct = 0; + } + + /* Find the ORDER BY term that corresponds to the j-th column + ** of the index and mark that ORDER BY term off + */ + bOnce = 1; + isMatch = 0; + for(i=0; bOnce && ia[i].pExpr); + testcase( wctrlFlags & WHERE_GROUPBY ); + testcase( wctrlFlags & WHERE_DISTINCTBY ); + if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=0 ){ + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; + } + isMatch = 1; + break; + } + if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } + if( isMatch ){ + if( iColumn<0 ){ + testcase( distinctColumns==0 ); + distinctColumns = 1; + } + obSat |= MASKBIT(i); + }else{ + /* No match found */ + if( j==0 || jmaskSelf; + for(i=0; ia[i].pExpr; + mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( (mTerm&~orderDistinctMask)==0 ){ + obSat |= MASKBIT(i); + } + } + } + } /* End the loop over all WhereLoops from outer-most down to inner-most */ + if( obSat==obDone ) return (i8)nOrderBy; + if( !isOrderDistinct ){ + for(i=nOrderBy-1; i>0; i--){ + Bitmask m = MASKBIT(i) - 1; + if( (obSat&m)==m ) return i; + } + return 0; + } + return -1; +} + + +/* +** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), +** the planner assumes that the specified pOrderBy list is actually a GROUP +** BY clause - and so any order that groups rows as required satisfies the +** request. +** +** Normally, in this case it is not possible for the caller to determine +** whether or not the rows are really being delivered in sorted order, or +** just in some other order that provides the required grouping. However, +** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then +** this function may be called on the returned WhereInfo object. It returns +** true if the rows really will be sorted in the specified order, or false +** otherwise. +** +** For example, assuming: +** +** CREATE INDEX i1 ON t1(x, Y); +** +** then +** +** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 +** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 +*/ +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo *pWInfo){ + assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); + return pWInfo->sorted; +} + +#ifdef WHERETRACE_ENABLED +/* For debugging use only: */ +static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ + static char zName[65]; + int i; + for(i=0; iaLoop[i]->cId; } + if( pLast ) zName[i++] = pLast->cId; + zName[i] = 0; + return zName; +} +#endif + +/* +** Return the cost of sorting nRow rows, assuming that the keys have +** nOrderby columns and that the first nSorted columns are already in +** order. +*/ +static LogEst whereSortingCost( + WhereInfo *pWInfo, + LogEst nRow, + int nOrderBy, + int nSorted +){ + /* TUNING: Estimated cost of a full external sort, where N is + ** the number of rows to sort is: + ** + ** cost = (3.0 * N * log(N)). + ** + ** Or, if the order-by clause has X terms but only the last Y + ** terms are out of order, then block-sorting will reduce the + ** sorting cost to: + ** + ** cost = (3.0 * N * log(N)) * (Y/X) + ** + ** The (Y/X) term is implemented using stack variable rScale + ** below. */ + LogEst rScale, rSortCost; + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); + rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + rSortCost = nRow + estLog(nRow) + rScale + 16; + + /* TUNING: The cost of implementing DISTINCT using a B-TREE is + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; + } + + return rSortCost; +} + +/* +** Given the list of WhereLoop objects at pWInfo->pLoops, this routine +** attempts to find the lowest cost path that visits each WhereLoop +** once. This path is then loaded into the pWInfo->a[].pWLoop fields. +** +** Assume that the total number of output rows that will need to be sorted +** will be nRowEst (in the 10*log2 representation). Or, ignore sorting +** costs if nRowEst==0. +** +** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation +** error occurs. +*/ +static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ + int mxChoice; /* Maximum number of simultaneous paths tracked */ + int nLoop; /* Number of terms in the join */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* The database connection */ + int iLoop; /* Loop counter over the terms of the join */ + int ii, jj; /* Loop counters */ + int mxI = 0; /* Index of next entry to replace */ + int nOrderBy; /* Number of ORDER BY clause terms */ + LogEst mxCost = 0; /* Maximum cost of a set of paths */ + LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ + int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ + WherePath *aFrom; /* All nFrom paths at the previous level */ + WherePath *aTo; /* The nTo best paths at the current level */ + WherePath *pFrom; /* An element of aFrom[] that we are working on */ + WherePath *pTo; /* An element of aTo[] that we are working on */ + WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop **pX; /* Used to divy up the pSpace memory */ + LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ + char *pSpace; /* Temporary memory used by this routine */ + int nSpace; /* Bytes of space allocated at pSpace */ + + pParse = pWInfo->pParse; + db = pParse->db; + nLoop = pWInfo->nLevel; + /* TUNING: For simple queries, only the best path is tracked. + ** For 2-way joins, the 5 best paths are followed. + ** For joins of 3 or more tables, track the 10 best paths */ + mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); + assert( nLoop<=pWInfo->pTabList->nSrc ); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); + + /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this + ** case the purpose of this call is to estimate the number of rows returned + ** by the overall query. Once this estimate has been obtained, the caller + ** will invoke this function a second time, passing the estimate as the + ** nRowEst parameter. */ + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ + nOrderBy = 0; + }else{ + nOrderBy = pWInfo->pOrderBy->nExpr; + } + + /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ + nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + nSpace += sizeof(LogEst) * nOrderBy; + pSpace = sqlite3DbMallocRaw(db, nSpace); + if( pSpace==0 ) return SQLITE_NOMEM; + aTo = (WherePath*)pSpace; + aFrom = aTo+mxChoice; + memset(aFrom, 0, sizeof(aFrom[0])); + pX = (WhereLoop**)(aFrom+mxChoice); + for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ + pFrom->aLoop = pX; + } + if( nOrderBy ){ + /* If there is an ORDER BY clause and it is not being ignored, set up + ** space for the aSortCost[] array. Each element of the aSortCost array + ** is either zero - meaning it has not yet been initialized - or the + ** cost of sorting nRowEst rows of data where the first X terms of + ** the ORDER BY clause are already in order, where X is the array + ** index. */ + aSortCost = (LogEst*)pX; + memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); + } + assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); + assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); + + /* Seed the search with a single WherePath containing zero WhereLoops. + ** + ** TUNING: Do not let the number of iterations go above 25. If the cost + ** of computing an automatic index is not paid back within the first 25 + ** rows, then do not use the automatic index. */ + aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) ); + nFrom = 1; + assert( aFrom[0].isOrdered==0 ); + if( nOrderBy ){ + /* If nLoop is zero, then there are no FROM terms in the query. Since + ** in this case the query may return a maximum of one row, the results + ** are already in the requested order. Set isOrdered to nOrderBy to + ** indicate this. Or, if nLoop is greater than zero, set isOrdered to + ** -1, indicating that the result set may or may not be ordered, + ** depending on the loops added to the current plan. */ + aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; + } + + /* Compute successively longer WherePaths using the previous generation + ** of WherePaths as the basis for the next. Keep track of the mxChoice + ** best paths at each generation */ + for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ + LogEst rCost; /* Cost of path (pFrom+pWLoop) */ + LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ + i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + Bitmask maskNew; /* Mask of src visited by (..) */ + Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + /* At this point, pWLoop is a candidate to be the next loop. + ** Compute its cost */ + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); + nOut = pFrom->nRow + pWLoop->nOut; + maskNew = pFrom->maskLoop | pWLoop->maskSelf; + if( isOrdered<0 ){ + isOrdered = wherePathSatisfiesOrderBy(pWInfo, + pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, + iLoop, pWLoop, &revMask); + }else{ + revMask = pFrom->revLoop; + } + if( isOrdered>=0 && isOrderedisOrdered^isOrdered)&0x80)==0" is equivalent + ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range + ** of legal values for isOrdered, -1..64. + */ + for(jj=0, pTo=aTo; jjmaskLoop==maskNew + && ((pTo->isOrdered^isOrdered)&0x80)==0 + ){ + testcase( jj==nTo-1 ); + break; + } + } + if( jj>=nTo ){ + /* None of the existing best-so-far paths match the candidate. */ + if( nTo>=mxChoice + && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) + ){ + /* The current candidate is no better than any of the mxChoice + ** paths currently in the best-so-far buffer. So discard + ** this candidate as not viable. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + } +#endif + continue; + } + /* If we reach this points it means that the new candidate path + ** needs to be added to the set of best-so-far paths. */ + if( nTo=0 ? isOrdered+'0' : '?'); + } +#endif + }else{ + /* Control reaches here if best-so-far path pTo=aTo[jj] covers the + ** same set of loops and has the sam isOrdered setting as the + ** candidate path. Check to see if the candidate should replace + ** pTo or if the candidate should be skipped */ + if( pTo->rCostrCost==rCost && pTo->nRow<=nOut) ){ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Skip %s cost=%-3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + /* Discard the candidate path from further consideration */ + testcase( pTo->rCost==rCost ); + continue; + } + testcase( pTo->rCost==rCost+1 ); + /* Control reaches here if the candidate path is better than the + ** pTo path. Replace pTo with the candidate. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Update %s cost=%-3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + } + /* pWLoop is a winner. Add it to the set of best so far */ + pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->revLoop = revMask; + pTo->nRow = nOut; + pTo->rCost = rCost; + pTo->rUnsorted = rUnsorted; + pTo->isOrdered = isOrdered; + memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); + pTo->aLoop[iLoop] = pWLoop; + if( nTo>=mxChoice ){ + mxI = 0; + mxCost = aTo[0].rCost; + mxUnsorted = aTo[0].nRow; + for(jj=1, pTo=&aTo[1]; jjrCost>mxCost + || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + ){ + mxCost = pTo->rCost; + mxUnsorted = pTo->rUnsorted; + mxI = jj; + } + } + } + } + } + +#ifdef WHERETRACE_ENABLED /* >=2 */ + if( sqlite3WhereTrace>=2 ){ + sqlite3DebugPrintf("---- after round %d ----\n", iLoop); + for(ii=0, pTo=aTo; iirCost, pTo->nRow, + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } + } + } +#endif + + /* Swap the roles of aFrom and aTo for the next generation */ + pFrom = aTo; + aTo = aFrom; + aFrom = pFrom; + nFrom = nTo; + } + + if( nFrom==0 ){ + sqlite3ErrorMsg(pParse, "no query solution"); + sqlite3DbFree(db, pSpace); + return SQLITE_ERROR; + } + + /* Find the lowest cost path. pFrom will be left pointing to that path */ + pFrom = aFrom; + for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; + } + assert( pWInfo->nLevel==nLoop ); + /* Load the lowest cost path into pWInfo */ + for(iLoop=0; iLoopa + iLoop; + pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; + pLevel->iFrom = pWLoop->iTab; + pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; + } + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 + && pWInfo->eDistinct==WHERE_DISTINCT_NOOP + && nRowEst + ){ + Bitmask notUsed; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, + WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); + if( rc==pWInfo->pResultSet->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + } + if( pWInfo->pOrderBy ){ + if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ + if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + }else{ + pWInfo->nOBSat = pFrom->isOrdered; + if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; + pWInfo->revMask = pFrom->revLoop; + } + if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr + ){ + Bitmask revMask = 0; + int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask + ); + assert( pWInfo->sorted==0 ); + if( nOrder==pWInfo->pOrderBy->nExpr ){ + pWInfo->sorted = 1; + pWInfo->revMask = revMask; + } + } + } + + + pWInfo->nRowOut = pFrom->nRow; + + /* Free temporary memory and return success */ + sqlite3DbFree(db, pSpace); + return SQLITE_OK; +} + +/* +** Most queries use only a single table (they are not joins) and have +** simple == constraints against indexed fields. This routine attempts +** to plan those simple cases using much less ceremony than the +** general-purpose query planner, and thereby yield faster sqlite3_prepare() +** times for the common case. +** +** Return non-zero on success, if this query can be handled by this +** no-frills query planner. Return zero if this query needs the +** general-purpose query planner. +*/ +static int whereShortCut(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo; + struct SrcList_item *pItem; + WhereClause *pWC; + WhereTerm *pTerm; + WhereLoop *pLoop; + int iCur; + int j; + Table *pTab; + Index *pIdx; + + pWInfo = pBuilder->pWInfo; + if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; + assert( pWInfo->pTabList->nSrc>=1 ); + pItem = pWInfo->pTabList->a; + pTab = pItem->pTab; + if( IsVirtual(pTab) ) return 0; + if( pItem->zIndex ) return 0; + iCur = pItem->iCursor; + pWC = &pWInfo->sWC; + pLoop = pBuilder->pNew; + pLoop->wsFlags = 0; + pLoop->u.btree.nSkip = 0; + pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); + if( pTerm ){ + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; + pLoop->aLTerm[0] = pTerm; + pLoop->nLTerm = 1; + pLoop->u.btree.nEq = 1; + /* TUNING: Cost of a rowid lookup is 10 */ + pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ + }else{ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pLoop->aLTermSpace==pLoop->aLTerm ); + assert( ArraySize(pLoop->aLTermSpace)==4 ); + if( !IsUniqueIndex(pIdx) + || pIdx->pPartIdxWhere!=0 + || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) + ) continue; + for(j=0; jnKeyCol; j++){ + pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); + if( pTerm==0 ) break; + pLoop->aLTerm[j] = pTerm; + } + if( j!=pIdx->nKeyCol ) continue; + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; + if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + pLoop->wsFlags |= WHERE_IDX_ONLY; + } + pLoop->nLTerm = j; + pLoop->u.btree.nEq = j; + pLoop->u.btree.pIndex = pIdx; + /* TUNING: Cost of a unique index lookup is 15 */ + pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ + break; + } + } + if( pLoop->wsFlags ){ + pLoop->nOut = (LogEst)1; + pWInfo->a[0].pWLoop = pLoop; + pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pWInfo->a[0].iTabCur = iCur; + pWInfo->nRowOut = 1; + if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } +#ifdef SQLITE_DEBUG + pLoop->cId = '0'; +#endif + return 1; + } + return 0; +} /* ** Generate the beginning of the loop used for WHERE clause processing. @@ -102678,39 +119623,56 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** ORDER BY CLAUSE PROCESSING ** -** *ppOrderBy is a pointer to the ORDER BY clause of a SELECT statement, +** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause +** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement ** if there is one. If there is no ORDER BY clause or if this routine -** is called from an UPDATE or DELETE statement, then ppOrderBy is NULL. +** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. ** -** If an index can be used so that the natural output order of the table -** scan is correct for the ORDER BY clause, then that index is used and -** *ppOrderBy is set to NULL. This is an optimization that prevents an -** unnecessary sort of the result set if an index appropriate for the -** ORDER BY clause already exists. -** -** If the where clause loops cannot be arranged to provide the correct -** output order, then the *ppOrderBy is unchanged. +** The iIdxCur parameter is the cursor number of an index. If +** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index +** to use for OR clause processing. The WHERE clause should use this +** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is +** the first cursor in an array of cursors for all indices. iIdxCur should +** be used to compute the appropriate cursor depending on which index is +** used. */ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ - SrcList *pTabList, /* A list of all tables to be scanned */ + SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ - ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ - u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ + ExprList *pResultSet, /* Result set of the query */ + u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ + int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ ){ - int i; /* Loop counter */ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ WhereInfo *pWInfo; /* Will become the return value of this function */ Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ + WhereLoopBuilder sWLB; /* The WhereLoop builder */ WhereMaskSet *pMaskSet; /* The expression mask set */ - WhereClause *pWC; /* Decomposition of the WHERE clause */ - struct SrcList_item *pTabItem; /* A single entry from pTabList */ - WhereLevel *pLevel; /* A single level in the pWInfo list */ - int iFrom; /* First unused FROM clause element */ - int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ + int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ + int rc; /* Return code */ + + + /* Variable initialization */ + db = pParse->db; + memset(&sWLB, 0, sizeof(sWLB)); + + /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ + testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); + if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; + sWLB.pOrderBy = pOrderBy; + + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask @@ -102735,41 +119697,57 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - db = pParse->db; nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite3DbMallocZero(db, - nByteWInfo + - sizeof(WhereClause) + - sizeof(WhereMaskSet) - ); + pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); pWInfo = 0; goto whereBeginError; } + pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; - pWInfo->iBreak = sqlite3VdbeMakeLabel(v); - pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; + pWInfo->pOrderBy = pOrderBy; + pWInfo->pResultSet = pResultSet; + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&pWC[1]; + pMaskSet = &pWInfo->sMaskSet; + sWLB.pWInfo = pWInfo; + sWLB.pWC = &pWInfo->sWC; + sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); + assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) ); + whereLoopInit(sWLB.pNew); +#ifdef SQLITE_DEBUG + sWLB.pNew->cId = '*'; +#endif /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(pWC, pParse, pMaskSet); - sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + whereClauseInit(&pWInfo->sWC, pWInfo); + whereSplit(&pWInfo->sWC, pWhere, TK_AND); /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. */ - if( pWhere && (nTabList==0 || sqlite3ExprIsConstantNotJoin(pWhere)) ){ - sqlite3ExprIfFalse(pParse, pWhere, pWInfo->iBreak, SQLITE_JUMPIFNULL); - pWhere = 0; + for(ii=0; iinTerm; ii++){ + if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){ + sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak, + SQLITE_JUMPIFNULL); + sWLB.pWC->a[ii].wtFlags |= TERM_CODED; + } + } + + /* Special case: No FROM clause + */ + if( nTabList==0 ){ + if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } } /* Assign a bit from the bitmask to every term in the FROM clause. @@ -102783,30 +119761,19 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** bitmask for all tables to the left of the join. Knowing the bitmask ** for all tables to the left of a left join is important. Ticket #3015. ** - ** Configure the WhereClause.vmask variable so that bits that correspond - ** to virtual table cursors are set. This is used to selectively disable - ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful - ** with virtual tables. - ** ** Note that bitmasks are created for all pTabList->nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally ** equal to pTabList->nSrc but might be shortened to 1 if the ** WHERE_ONETABLE_ONLY flag is set. */ - assert( pWC->vmask==0 && pMaskSet->n==0 ); - for(i=0; inSrc; i++){ - createMask(pMaskSet, pTabList->a[i].iCursor); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ - pWC->vmask |= ((Bitmask)1 << i); - } -#endif + for(ii=0; iinSrc; ii++){ + createMask(pMaskSet, pTabList->a[ii].iCursor); } #ifndef NDEBUG { Bitmask toTheLeft = 0; - for(i=0; inSrc; i++){ - Bitmask m = getMask(pMaskSet, pTabList->a[i].iCursor); + for(ii=0; iinSrc; ii++){ + Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); assert( (m-1)==toTheLeft ); toTheLeft |= m; } @@ -102818,267 +119785,177 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** want to analyze these virtual terms, so start analyzing at the end ** and work forward so that the added virtual terms are never processed. */ - exprAnalyzeAll(pTabList, pWC); + exprAnalyzeAll(pTabList, &pWInfo->sWC); if( db->mallocFailed ){ goto whereBeginError; } - /* Chose the best index to use for each table in the FROM clause. - ** - ** This loop fills in the following fields: - ** - ** pWInfo->a[].pIdx The index to use for this level of the loop. - ** pWInfo->a[].wsFlags WHERE_xxx flags associated with pIdx - ** pWInfo->a[].nEq The number of == and IN constraints - ** pWInfo->a[].iFrom Which term of the FROM clause is being coded - ** pWInfo->a[].iTabCur The VDBE cursor for the database table - ** pWInfo->a[].iIdxCur The VDBE cursor for the index - ** pWInfo->a[].pTerm When wsFlags==WO_OR, the OR-clause term - ** - ** This loop also figures out the nesting order of tables in the FROM - ** clause. - */ - notReady = ~(Bitmask)0; - andFlags = ~0; - WHERETRACE(("*** Optimizer Start ***\n")); - for(i=iFrom=0, pLevel=pWInfo->a; i=0 && bestJ<0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ - for(j=iFrom, pTabItem=&pTabList->a[j]; jjointype & (JT_LEFT|JT_CROSS))!=0; - if( j!=iFrom && doNotReorder ) break; - m = getMask(pMaskSet, pTabItem->iCursor); - if( (m & notReady)==0 ){ - if( j==iFrom ) iFrom++; - continue; - } - mask = (isOptimal ? m : notReady); - pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); - if( pTabItem->pIndex==0 ) nUnconstrained++; - - WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", - j, isOptimal)); - assert( pTabItem->pTab ); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTabItem->pTab) ){ - sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost, pp); - }else -#endif - { - bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost); - } - assert( isOptimal || (sCost.used¬Ready)==0 ); - - /* If an INDEXED BY clause is present, then the plan must use that - ** index if it uses any index at all */ - assert( pTabItem->pIndex==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || sCost.plan.u.pIdx==pTabItem->pIndex ); - - if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ - notIndexed |= m; - } - - /* Conditions under which this table becomes the best so far: - ** - ** (1) The table must not depend on other tables that have not - ** yet run. - ** - ** (2) A full-table-scan plan cannot supercede indexed plan unless - ** the full-table-scan is an "optimal" plan as defined above. - ** - ** (3) All tables have an INDEXED BY clause or this table lacks an - ** INDEXED BY clause or this table uses the specific - ** index specified by its INDEXED BY clause. This rule ensures - ** that a best-so-far is always selected even if an impossible - ** combination of INDEXED BY clauses are given. The error - ** will be detected and relayed back to the application later. - ** The NEVER() comes about because rule (2) above prevents - ** An indexable full-table-scan from reaching rule (3). - ** - ** (4) The plan cost must be lower than prior plans or else the - ** cost must be the same and the number of rows must be lower. - */ - if( (sCost.used¬Ready)==0 /* (1) */ - && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ - || (bestPlan.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) - && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ - || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || sCost.rCost=0 ); - assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d" - " with cost=%g and nRow=%g\n", - bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); - if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ - *ppOrderBy = 0; - } - andFlags &= bestPlan.plan.wsFlags; - pLevel->plan = bestPlan.plan; - testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); - testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); - if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ - pLevel->iIdxCur = pParse->nTab++; - }else{ - pLevel->iIdxCur = -1; - } - notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); - pLevel->iFrom = (u8)bestJ; - if( bestPlan.plan.nRow>=(double)1 ){ - pParse->nQueryLoop *= bestPlan.plan.nRow; - } - - /* Check that if the table scanned by this loop iteration had an - ** INDEXED BY clause attached to it, that the named index is being - ** used for the scan. If not, then query compilation has failed. - ** Return an error. - */ - pIdx = pTabList->a[bestJ].pIndex; - if( pIdx ){ - if( (bestPlan.plan.wsFlags & WHERE_INDEXED)==0 ){ - sqlite3ErrorMsg(pParse, "cannot use index: %s", pIdx->zName); - goto whereBeginError; - }else{ - /* If an INDEXED BY clause is used, the bestIndex() function is - ** guaranteed to find the index specified in the INDEXED BY clause - ** if it find an index at all. */ - assert( bestPlan.plan.u.pIdx==pIdx ); - } + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + /* The DISTINCT marking is pointless. Ignore it. */ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + }else if( pOrderBy==0 ){ + /* Try to ORDER BY the result set to make distinct processing easier */ + pWInfo->wctrlFlags |= WHERE_DISTINCTBY; + pWInfo->pOrderBy = pResultSet; } } - WHERETRACE(("*** Optimizer Finished ***\n")); - if( pParse->nErr || db->mallocFailed ){ + + /* Construct the WhereLoop objects */ + WHERETRACE(0xffff,("*** Optimizer Start ***\n")); +#if defined(WHERETRACE_ENABLED) + /* Display all terms of the WHERE clause */ + if( sqlite3WhereTrace & 0x100 ){ + int i; + for(i=0; inTerm; i++){ + whereTermPrint(&sWLB.pWC->a[i], i); + } + } +#endif + + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + + /* Display all of the WhereLoop objects if wheretrace is enabled */ +#ifdef WHERETRACE_ENABLED /* !=0 */ + if( sqlite3WhereTrace ){ + WhereLoop *p; + int i; + static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%sizeof(zLabel)]; + whereLoopPrint(p, sWLB.pWC); + } + } +#endif + + wherePathSolver(pWInfo, 0); + if( db->mallocFailed ) goto whereBeginError; + if( pWInfo->pOrderBy ){ + wherePathSolver(pWInfo, pWInfo->nRowOut+1); + if( db->mallocFailed ) goto whereBeginError; + } + } + if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ + pWInfo->revMask = (Bitmask)(-1); + } + if( pParse->nErr || NEVER(db->mallocFailed) ){ goto whereBeginError; } - - /* If the total query only selects a single row, then the ORDER BY - ** clause is irrelevant. - */ - if( (andFlags & WHERE_UNIQUE)!=0 && ppOrderBy ){ - *ppOrderBy = 0; +#ifdef WHERETRACE_ENABLED /* !=0 */ + if( sqlite3WhereTrace ){ + int ii; + sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); + if( pWInfo->nOBSat>0 ){ + sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); + } + switch( pWInfo->eDistinct ){ + case WHERE_DISTINCT_UNIQUE: { + sqlite3DebugPrintf(" DISTINCT=unique"); + break; + } + case WHERE_DISTINCT_ORDERED: { + sqlite3DebugPrintf(" DISTINCT=ordered"); + break; + } + case WHERE_DISTINCT_UNORDERED: { + sqlite3DebugPrintf(" DISTINCT=unordered"); + break; + } + } + sqlite3DebugPrintf("\n"); + for(ii=0; iinLevel; ii++){ + whereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); + } } +#endif + /* Attempt to omit tables from the join that do not effect the result */ + if( pWInfo->nLevel>=2 + && pResultSet!=0 + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + ){ + Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); + if( sWLB.pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, sWLB.pOrderBy); + while( pWInfo->nLevel>=2 ){ + WhereTerm *pTerm, *pEnd; + pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; + if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 + ){ + break; + } + if( (tabUsed & pLoop->maskSelf)!=0 ) break; + pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; + for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + ){ + break; + } + } + if( pTerm drop loop %c not used\n", pLoop->cId)); + pWInfo->nLevel--; + nTabList--; + } + } + WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. - ** The one-pass algorithm only works if the WHERE clause constraints + ** The one-pass algorithm only works if the WHERE clause constrains ** the statement to update a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 && (andFlags & WHERE_UNIQUE)!=0 ){ + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ pWInfo->okOnePass = 1; - pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; + if( HasRowid(pTabList->a[0].pTab) ){ + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + } } /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ - sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; - pWInfo->nRowOut = (double)1; - for(i=0, pLevel=pWInfo->a; ia; iia[pLevel->iFrom]; pTab = pTabItem->pTab; - pLevel->iTabCur = pTabItem->iCursor; - pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ /* Do nothing */ }else #ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); int iCur = pTabItem->iCursor; sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else if( IsVirtual(pTab) ){ + /* noop */ }else #endif - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - && (wctrlFlags & WHERE_OMIT_OPEN)==0 ){ - int op = pWInfo->okOnePass ? OP_OpenWrite : OP_OpenRead; + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ + int op = OP_OpenRead; + if( pWInfo->okOnePass ){ + op = OP_OpenWrite; + pWInfo->aiCurOnePass[0] = pTabItem->iCursor; + }; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); - testcase( pTab->nCol==BMS-1 ); - testcase( pTab->nCol==BMS ); - if( !pWInfo->okOnePass && pTab->nColiCursor==pLevel->iTabCur ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); + if( !pWInfo->okOnePass && pTab->nColcolUsed; int n = 0; for(; b; b=b>>1, n++){} @@ -103089,23 +119966,46 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX - if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, pWC, pTabItem, notReady, pLevel); - }else -#endif - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - Index *pIx = pLevel->plan.u.pIdx; - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIx); - int iIdxCur = pLevel->iIdxCur; + if( pLoop->wsFlags & WHERE_INDEXED ){ + Index *pIx = pLoop->u.btree.pIndex; + int iIndexCur; + int op = OP_OpenRead; + /* iIdxCur is always set if to a positive value if ONEPASS is possible */ + assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) + && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 + ){ + /* This is one term of an OR-optimization using the PRIMARY KEY of a + ** WITHOUT ROWID table. No need for a separate index */ + iIndexCur = pLevel->iTabCur; + op = 0; + }else if( pWInfo->okOnePass ){ + Index *pJ = pTabItem->pTab->pIndex; + iIndexCur = iIdxCur; + assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); + while( ALWAYS(pJ) && pJ!=pIx ){ + iIndexCur++; + pJ = pJ->pNext; + } + op = OP_OpenWrite; + pWInfo->aiCurOnePass[1] = iIndexCur; + }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ + iIndexCur = iIdxCur; + if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; + }else{ + iIndexCur = pParse->nTab++; + } + pLevel->iIdxCur = iIndexCur; assert( pIx->pSchema==pTab->pSchema ); - assert( iIdxCur>=0 ); - sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIx->tnum, iDb, - (char*)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIx->zName)); + assert( iIndexCur>=0 ); + if( op ){ + sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIx); + VdbeComment((v, "%s", pIx->zName)); + } } - sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(pWC->pMaskSet, pTabItem->iCursor); + if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); + notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -103115,65 +120015,23 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** program. */ notReady = ~(Bitmask)0; - for(i=0; ia[i]; - explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); + for(ii=0; iia[ii]; +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + constructAutomaticIndex(pParse, &pWInfo->sWC, + &pTabList->a[pLevel->iFrom], notReady, pLevel); + if( db->mallocFailed ) goto whereBeginError; + } +#endif + explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + pLevel->addrBody = sqlite3VdbeCurrentAddr(v); + notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; } -#ifdef SQLITE_TEST /* For testing and debugging use only */ - /* Record in the query plan information about the current table - ** and the index used to access it (if any). If the table itself - ** is not used, its name is just '{}'. If no index is used - ** the index is listed as "{}". If the primary key is used the - ** index name is '*'. - */ - for(i=0; ia[i]; - pTabItem = &pTabList->a[pLevel->iFrom]; - z = pTabItem->zAlias; - if( z==0 ) z = pTabItem->pTab->zName; - n = sqlite3Strlen30(z); - if( n+nQPlan < sizeof(sqlite3_query_plan)-10 ){ - if( pLevel->plan.wsFlags & WHERE_IDX_ONLY ){ - memcpy(&sqlite3_query_plan[nQPlan], "{}", 2); - nQPlan += 2; - }else{ - memcpy(&sqlite3_query_plan[nQPlan], z, n); - nQPlan += n; - } - sqlite3_query_plan[nQPlan++] = ' '; - } - testcase( pLevel->plan.wsFlags & WHERE_ROWID_EQ ); - testcase( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ); - if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ - memcpy(&sqlite3_query_plan[nQPlan], "* ", 2); - nQPlan += 2; - }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - n = sqlite3Strlen30(pLevel->plan.u.pIdx->zName); - if( n+nQPlan < sizeof(sqlite3_query_plan)-2 ){ - memcpy(&sqlite3_query_plan[nQPlan], pLevel->plan.u.pIdx->zName, n); - nQPlan += n; - sqlite3_query_plan[nQPlan++] = ' '; - } - }else{ - memcpy(&sqlite3_query_plan[nQPlan], "{} ", 3); - nQPlan += 3; - } - } - while( nQPlan>0 && sqlite3_query_plan[nQPlan-1]==' ' ){ - sqlite3_query_plan[--nQPlan] = 0; - } - sqlite3_query_plan[nQPlan] = 0; - nQPlan = 0; -#endif /* SQLITE_TEST // Testing and debugging use only */ - - /* Record the continuation address in the WhereInfo structure. Then - ** clean up and return. - */ + /* Done. */ + VdbeModuleComment((v, "Begin WHERE-core")); return pWInfo; /* Jump here if malloc fails */ @@ -103194,40 +120052,56 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ Vdbe *v = pParse->pVdbe; int i; WhereLevel *pLevel; + WhereLoop *pLoop; SrcList *pTabList = pWInfo->pTabList; sqlite3 *db = pParse->db; /* Generate loop termination code. */ + VdbeModuleComment((v, "End WHERE-core")); sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ + int addr; pLevel = &pWInfo->a[i]; + pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ - sqlite3VdbeAddOp2(v, pLevel->op, pLevel->p1, pLevel->p2); + sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); + VdbeCoverage(v); + VdbeCoverageIf(v, pLevel->op==OP_Next); + VdbeCoverageIf(v, pLevel->op==OP_Prev); + VdbeCoverageIf(v, pLevel->op==OP_VNext); } - if( pLevel->plan.wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); + VdbeCoverage(v); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } sqlite3DbFree(db, pLevel->u.in.aInLoop); } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->addrSkip ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); + sqlite3VdbeJumpHere(v, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); + } if( pLevel->iLeftJoin ){ - int addr; - addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); - assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ); - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){ + addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); } - if( pLevel->iIdxCur>=0 ){ + if( pLoop->wsFlags & WHERE_INDEXED ){ sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); } if( pLevel->op==OP_Return ){ @@ -103237,6 +120111,8 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } sqlite3VdbeJumpHere(v, addr); } + VdbeModuleComment((v, "End WHERE-loop%d: %s", i, + pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); } /* The "break" point is here, just past the end of the outer loop. @@ -103244,32 +120120,65 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ sqlite3VdbeResolveLabel(v, pWInfo->iBreak); - /* Close all of the cursors that were opened by sqlite3WhereBegin. - */ - assert( pWInfo->nLevel==1 || pWInfo->nLevel==pTabList->nSrc ); + assert( pWInfo->nLevel<=pTabList->nSrc ); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ + int k, last; + VdbeOp *pOp; + Index *pIdx = 0; struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; Table *pTab = pTabItem->pTab; assert( pTab!=0 ); + pLoop = pLevel->pWLoop; + + /* For a co-routine, change all OP_Column references to the table of + ** the co-routine into OP_SCopy of result contained in a register. + ** OP_Rowid becomes OP_Null. + */ + if( pTabItem->viaCoroutine && !db->mallocFailed ){ + last = sqlite3VdbeCurrentAddr(v); + k = pLevel->addrBody; + pOp = sqlite3VdbeGetOp(v, k); + for(; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + pTabItem->regResult; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; + } + } + continue; + } + + /* Close all of the cursors that were opened by sqlite3WhereBegin. + ** Except, do not close cursors that will be reused by the OR optimization + ** (WHERE_OMIT_OPEN_CLOSE). And do not close the OP_OpenWrite cursors + ** created for the ONEPASS optimization. + */ if( (pTab->tabFlags & TF_Ephemeral)==0 && pTab->pSelect==0 - && (pWInfo->wctrlFlags & WHERE_OMIT_CLOSE)==0 + && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ - int ws = pLevel->plan.wsFlags; + int ws = pLoop->wsFlags; if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); } - if( (ws & WHERE_INDEXED)!=0 && (ws & WHERE_TEMP_INDEX)==0 ){ + if( (ws & WHERE_INDEXED)!=0 + && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 + && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] + ){ sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); } } - /* If this scan uses an index, make code substitutions to read data - ** from the index in preference to the table. Sometimes, this means - ** the table need never be read from. This is a performance boost, - ** as the vdbe level waits until the table is read before actually - ** seeking the table cursor to the record corresponding to the current - ** position in the index. + /* If this scan uses an index, make VDBE code substitutions to read data + ** from the index instead of from the table where possible. In some cases + ** this optimization prevents the table from ever being read, which can + ** yield a significant performance boost. ** ** Calls to the code generator in between sqlite3WhereBegin and ** sqlite3WhereEnd will have created code that references the table @@ -103277,26 +120186,30 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** that reference the table and converts them into opcodes that ** reference the index. */ - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 && !db->mallocFailed){ - int k, j, last; - VdbeOp *pOp; - Index *pIdx = pLevel->plan.u.pIdx; - - assert( pIdx!=0 ); - pOp = sqlite3VdbeGetOp(v, pWInfo->iTop); + if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ + pIdx = pLoop->u.btree.pIndex; + }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ + pIdx = pLevel->u.pCovidx; + } + if( pIdx && !db->mallocFailed ){ last = sqlite3VdbeCurrentAddr(v); - for(k=pWInfo->iTop; kaddrBody; + pOp = sqlite3VdbeGetOp(v, k); + for(; kp1!=pLevel->iTabCur ) continue; if( pOp->opcode==OP_Column ){ - for(j=0; jnColumn; j++){ - if( pOp->p2==pIdx->aiColumn[j] ){ - pOp->p2 = j; - pOp->p1 = pLevel->iIdxCur; - break; - } + int x = pOp->p2; + assert( pIdx->pTable==pTab ); + if( !HasRowid(pTab) ){ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + x = pPk->aiColumn[x]; } - assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - || jnColumn ); + x = sqlite3ColumnOfIndex(pIdx, x); + if( x>=0 ){ + pOp->p2 = x; + pOp->p1 = pLevel->iIdxCur; + } + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; @@ -103325,6 +120238,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ /* First off, code is included that follows the "include" declaration ** in the input grammar file. */ +/* #include */ /* @@ -103353,7 +120267,7 @@ struct LimitVal { */ struct LikeOp { Token eOperator; /* "like" or "glob" or "regexp" */ - int not; /* True if the NOT keyword is present */ + int bNot; /* True if the NOT keyword is present */ }; /* @@ -103425,7 +120339,7 @@ struct AttachKey { int type; Token key; }; ** unary TK_ISNULL or TK_NOTNULL expression. */ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ sqlite3 *db = pParse->db; - if( db->mallocFailed==0 && pY->op==TK_NULL ){ + if( pY && pA && pY->op==TK_NULL ){ pA->op = (u8)op; sqlite3ExprDelete(db, pA->pRight); pA->pRight = 0; @@ -103495,26 +120409,28 @@ struct AttachKey { int type; Token key; }; ** defined, then do no error processing. */ #define YYCODETYPE unsigned char -#define YYNOCODE 253 +#define YYNOCODE 254 #define YYACTIONTYPE unsigned short int -#define YYWILDCARD 67 +#define YYWILDCARD 70 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - int yy4; - struct TrigEvent yy90; - ExprSpan yy118; - TriggerStep* yy203; - u8 yy210; - struct {int value; int mask;} yy215; - SrcList* yy259; - struct LimitVal yy292; - Expr* yy314; - ExprList* yy322; - struct LikeOp yy342; - IdList* yy384; - Select* yy387; + Select* yy3; + ExprList* yy14; + With* yy59; + SrcList* yy65; + struct LikeOp yy96; + Expr* yy132; + u8 yy186; + int yy328; + ExprSpan yy346; + struct TrigEvent yy378; + u16 yy381; + IdList* yy408; + struct {int value; int mask;} yy429; + TriggerStep* yy473; + struct LimitVal yy476; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -103523,8 +120439,8 @@ typedef union { #define sqlite3ParserARG_PDECL ,Parse *pParse #define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse #define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 630 -#define YYNRULE 329 +#define YYNSTATE 642 +#define YYNRULE 327 #define YYFALLBACK 1 #define YY_NO_ACTION (YYNSTATE+YYNRULE+2) #define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) @@ -103594,473 +120510,463 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. */ -#define YY_ACTTAB_COUNT (1557) +#define YY_ACTTAB_COUNT (1497) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 313, 960, 186, 419, 2, 172, 627, 597, 55, 55, - /* 10 */ 55, 55, 48, 53, 53, 53, 53, 52, 52, 51, - /* 20 */ 51, 51, 50, 238, 302, 283, 623, 622, 516, 515, - /* 30 */ 590, 584, 55, 55, 55, 55, 282, 53, 53, 53, - /* 40 */ 53, 52, 52, 51, 51, 51, 50, 238, 6, 56, - /* 50 */ 57, 47, 582, 581, 583, 583, 54, 54, 55, 55, - /* 60 */ 55, 55, 608, 53, 53, 53, 53, 52, 52, 51, - /* 70 */ 51, 51, 50, 238, 313, 597, 409, 330, 579, 579, - /* 80 */ 32, 53, 53, 53, 53, 52, 52, 51, 51, 51, - /* 90 */ 50, 238, 330, 217, 620, 619, 166, 411, 624, 382, - /* 100 */ 379, 378, 7, 491, 590, 584, 200, 199, 198, 58, - /* 110 */ 377, 300, 414, 621, 481, 66, 623, 622, 621, 580, - /* 120 */ 254, 601, 94, 56, 57, 47, 582, 581, 583, 583, - /* 130 */ 54, 54, 55, 55, 55, 55, 671, 53, 53, 53, - /* 140 */ 53, 52, 52, 51, 51, 51, 50, 238, 313, 532, - /* 150 */ 226, 506, 507, 133, 177, 139, 284, 385, 279, 384, - /* 160 */ 169, 197, 342, 398, 251, 226, 253, 275, 388, 167, - /* 170 */ 139, 284, 385, 279, 384, 169, 570, 236, 590, 584, - /* 180 */ 672, 240, 275, 157, 620, 619, 554, 437, 51, 51, - /* 190 */ 51, 50, 238, 343, 439, 553, 438, 56, 57, 47, - /* 200 */ 582, 581, 583, 583, 54, 54, 55, 55, 55, 55, - /* 210 */ 465, 53, 53, 53, 53, 52, 52, 51, 51, 51, - /* 220 */ 50, 238, 313, 390, 52, 52, 51, 51, 51, 50, - /* 230 */ 238, 391, 166, 491, 566, 382, 379, 378, 409, 440, - /* 240 */ 579, 579, 252, 440, 607, 66, 377, 513, 621, 49, - /* 250 */ 46, 147, 590, 584, 621, 16, 466, 189, 621, 441, - /* 260 */ 442, 673, 526, 441, 340, 577, 595, 64, 194, 482, - /* 270 */ 434, 56, 57, 47, 582, 581, 583, 583, 54, 54, - /* 280 */ 55, 55, 55, 55, 30, 53, 53, 53, 53, 52, - /* 290 */ 52, 51, 51, 51, 50, 238, 313, 593, 593, 593, - /* 300 */ 387, 578, 606, 493, 259, 351, 258, 411, 1, 623, - /* 310 */ 622, 496, 623, 622, 65, 240, 623, 622, 597, 443, - /* 320 */ 237, 239, 414, 341, 237, 602, 590, 584, 18, 603, - /* 330 */ 166, 601, 87, 382, 379, 378, 67, 623, 622, 38, - /* 340 */ 623, 622, 176, 270, 377, 56, 57, 47, 582, 581, - /* 350 */ 583, 583, 54, 54, 55, 55, 55, 55, 175, 53, - /* 360 */ 53, 53, 53, 52, 52, 51, 51, 51, 50, 238, - /* 370 */ 313, 396, 233, 411, 531, 565, 317, 620, 619, 44, - /* 380 */ 620, 619, 240, 206, 620, 619, 597, 266, 414, 268, - /* 390 */ 409, 597, 579, 579, 352, 184, 505, 601, 73, 533, - /* 400 */ 590, 584, 466, 548, 190, 620, 619, 576, 620, 619, - /* 410 */ 547, 383, 551, 35, 332, 575, 574, 600, 504, 56, - /* 420 */ 57, 47, 582, 581, 583, 583, 54, 54, 55, 55, - /* 430 */ 55, 55, 567, 53, 53, 53, 53, 52, 52, 51, - /* 440 */ 51, 51, 50, 238, 313, 411, 561, 561, 528, 364, - /* 450 */ 259, 351, 258, 183, 361, 549, 524, 374, 411, 597, - /* 460 */ 414, 240, 560, 560, 409, 604, 579, 579, 328, 601, - /* 470 */ 93, 623, 622, 414, 590, 584, 237, 564, 559, 559, - /* 480 */ 520, 402, 601, 87, 409, 210, 579, 579, 168, 421, - /* 490 */ 950, 519, 950, 56, 57, 47, 582, 581, 583, 583, - /* 500 */ 54, 54, 55, 55, 55, 55, 192, 53, 53, 53, - /* 510 */ 53, 52, 52, 51, 51, 51, 50, 238, 313, 600, - /* 520 */ 293, 563, 511, 234, 357, 146, 475, 475, 367, 411, - /* 530 */ 562, 411, 358, 542, 425, 171, 411, 215, 144, 620, - /* 540 */ 619, 544, 318, 353, 414, 203, 414, 275, 590, 584, - /* 550 */ 549, 414, 174, 601, 94, 601, 79, 558, 471, 61, - /* 560 */ 601, 79, 421, 949, 350, 949, 34, 56, 57, 47, - /* 570 */ 582, 581, 583, 583, 54, 54, 55, 55, 55, 55, - /* 580 */ 535, 53, 53, 53, 53, 52, 52, 51, 51, 51, - /* 590 */ 50, 238, 313, 307, 424, 394, 272, 49, 46, 147, - /* 600 */ 349, 322, 4, 411, 491, 312, 321, 425, 568, 492, - /* 610 */ 216, 264, 407, 575, 574, 429, 66, 549, 414, 621, - /* 620 */ 540, 602, 590, 584, 13, 603, 621, 601, 72, 12, - /* 630 */ 618, 617, 616, 202, 210, 621, 546, 469, 422, 319, - /* 640 */ 148, 56, 57, 47, 582, 581, 583, 583, 54, 54, - /* 650 */ 55, 55, 55, 55, 338, 53, 53, 53, 53, 52, - /* 660 */ 52, 51, 51, 51, 50, 238, 313, 600, 600, 411, - /* 670 */ 39, 21, 37, 170, 237, 875, 411, 572, 572, 201, - /* 680 */ 144, 473, 538, 331, 414, 474, 143, 146, 630, 628, - /* 690 */ 334, 414, 353, 601, 68, 168, 590, 584, 132, 365, - /* 700 */ 601, 96, 307, 423, 530, 336, 49, 46, 147, 568, - /* 710 */ 406, 216, 549, 360, 529, 56, 57, 47, 582, 581, - /* 720 */ 583, 583, 54, 54, 55, 55, 55, 55, 411, 53, - /* 730 */ 53, 53, 53, 52, 52, 51, 51, 51, 50, 238, - /* 740 */ 313, 411, 605, 414, 484, 510, 172, 422, 597, 318, - /* 750 */ 496, 485, 601, 99, 411, 142, 414, 411, 231, 411, - /* 760 */ 540, 411, 359, 629, 2, 601, 97, 426, 308, 414, - /* 770 */ 590, 584, 414, 20, 414, 621, 414, 621, 601, 106, - /* 780 */ 503, 601, 105, 601, 108, 601, 109, 204, 28, 56, - /* 790 */ 57, 47, 582, 581, 583, 583, 54, 54, 55, 55, - /* 800 */ 55, 55, 411, 53, 53, 53, 53, 52, 52, 51, - /* 810 */ 51, 51, 50, 238, 313, 411, 597, 414, 411, 276, - /* 820 */ 214, 600, 411, 366, 213, 381, 601, 134, 274, 500, - /* 830 */ 414, 167, 130, 414, 621, 411, 354, 414, 376, 601, - /* 840 */ 135, 129, 601, 100, 590, 584, 601, 104, 522, 521, - /* 850 */ 414, 621, 224, 273, 600, 167, 327, 282, 600, 601, - /* 860 */ 103, 468, 521, 56, 57, 47, 582, 581, 583, 583, - /* 870 */ 54, 54, 55, 55, 55, 55, 411, 53, 53, 53, - /* 880 */ 53, 52, 52, 51, 51, 51, 50, 238, 313, 411, - /* 890 */ 27, 414, 411, 375, 276, 167, 359, 544, 50, 238, - /* 900 */ 601, 95, 128, 223, 414, 411, 165, 414, 411, 621, - /* 910 */ 411, 621, 612, 601, 102, 372, 601, 76, 590, 584, - /* 920 */ 414, 570, 236, 414, 470, 414, 167, 621, 188, 601, - /* 930 */ 98, 225, 601, 138, 601, 137, 232, 56, 45, 47, - /* 940 */ 582, 581, 583, 583, 54, 54, 55, 55, 55, 55, - /* 950 */ 411, 53, 53, 53, 53, 52, 52, 51, 51, 51, - /* 960 */ 50, 238, 313, 276, 276, 414, 411, 276, 544, 459, - /* 970 */ 359, 171, 209, 479, 601, 136, 628, 334, 621, 621, - /* 980 */ 125, 414, 621, 368, 411, 621, 257, 540, 589, 588, - /* 990 */ 601, 75, 590, 584, 458, 446, 23, 23, 124, 414, - /* 1000 */ 326, 325, 621, 427, 324, 309, 600, 288, 601, 92, - /* 1010 */ 586, 585, 57, 47, 582, 581, 583, 583, 54, 54, - /* 1020 */ 55, 55, 55, 55, 411, 53, 53, 53, 53, 52, - /* 1030 */ 52, 51, 51, 51, 50, 238, 313, 587, 411, 414, - /* 1040 */ 411, 207, 611, 476, 171, 472, 160, 123, 601, 91, - /* 1050 */ 323, 261, 15, 414, 464, 414, 411, 621, 411, 354, - /* 1060 */ 222, 411, 601, 74, 601, 90, 590, 584, 159, 264, - /* 1070 */ 158, 414, 461, 414, 621, 600, 414, 121, 120, 25, - /* 1080 */ 601, 89, 601, 101, 621, 601, 88, 47, 582, 581, - /* 1090 */ 583, 583, 54, 54, 55, 55, 55, 55, 544, 53, - /* 1100 */ 53, 53, 53, 52, 52, 51, 51, 51, 50, 238, - /* 1110 */ 43, 405, 263, 3, 610, 264, 140, 415, 622, 24, - /* 1120 */ 410, 11, 456, 594, 118, 155, 219, 452, 408, 621, - /* 1130 */ 621, 621, 156, 43, 405, 621, 3, 286, 621, 113, - /* 1140 */ 415, 622, 111, 445, 411, 400, 557, 403, 545, 10, - /* 1150 */ 411, 408, 264, 110, 205, 436, 541, 566, 453, 414, - /* 1160 */ 621, 621, 63, 621, 435, 414, 411, 621, 601, 94, - /* 1170 */ 403, 621, 411, 337, 601, 86, 150, 40, 41, 534, - /* 1180 */ 566, 414, 242, 264, 42, 413, 412, 414, 600, 595, - /* 1190 */ 601, 85, 191, 333, 107, 451, 601, 84, 621, 539, - /* 1200 */ 40, 41, 420, 230, 411, 149, 316, 42, 413, 412, - /* 1210 */ 398, 127, 595, 315, 621, 399, 278, 625, 181, 414, - /* 1220 */ 593, 593, 593, 592, 591, 14, 450, 411, 601, 71, - /* 1230 */ 240, 621, 43, 405, 264, 3, 615, 180, 264, 415, - /* 1240 */ 622, 614, 414, 593, 593, 593, 592, 591, 14, 621, - /* 1250 */ 408, 601, 70, 621, 417, 33, 405, 613, 3, 411, - /* 1260 */ 264, 411, 415, 622, 418, 626, 178, 509, 8, 403, - /* 1270 */ 241, 416, 126, 408, 414, 621, 414, 449, 208, 566, - /* 1280 */ 240, 221, 621, 601, 83, 601, 82, 599, 297, 277, - /* 1290 */ 296, 30, 403, 31, 395, 264, 295, 397, 489, 40, - /* 1300 */ 41, 411, 566, 220, 621, 294, 42, 413, 412, 271, - /* 1310 */ 621, 595, 600, 621, 59, 60, 414, 269, 267, 623, - /* 1320 */ 622, 36, 40, 41, 621, 601, 81, 598, 235, 42, - /* 1330 */ 413, 412, 621, 621, 595, 265, 344, 411, 248, 556, - /* 1340 */ 173, 185, 593, 593, 593, 592, 591, 14, 218, 29, - /* 1350 */ 621, 543, 414, 305, 304, 303, 179, 301, 411, 566, - /* 1360 */ 454, 601, 80, 289, 335, 593, 593, 593, 592, 591, - /* 1370 */ 14, 411, 287, 414, 151, 392, 246, 260, 411, 196, - /* 1380 */ 195, 523, 601, 69, 411, 245, 414, 526, 537, 285, - /* 1390 */ 389, 595, 621, 414, 536, 601, 17, 362, 153, 414, - /* 1400 */ 466, 463, 601, 78, 154, 414, 462, 152, 601, 77, - /* 1410 */ 355, 255, 621, 455, 601, 9, 621, 386, 444, 517, - /* 1420 */ 247, 621, 593, 593, 593, 621, 621, 244, 621, 243, - /* 1430 */ 430, 518, 292, 621, 329, 621, 145, 393, 280, 513, - /* 1440 */ 291, 131, 621, 514, 621, 621, 311, 621, 259, 346, - /* 1450 */ 249, 621, 621, 229, 314, 621, 228, 512, 227, 240, - /* 1460 */ 494, 488, 310, 164, 487, 486, 373, 480, 163, 262, - /* 1470 */ 369, 371, 162, 26, 212, 478, 477, 161, 141, 363, - /* 1480 */ 467, 122, 339, 187, 119, 348, 347, 117, 116, 115, - /* 1490 */ 114, 112, 182, 457, 320, 22, 433, 432, 448, 19, - /* 1500 */ 609, 431, 428, 62, 193, 596, 573, 298, 555, 552, - /* 1510 */ 571, 404, 290, 380, 498, 510, 495, 306, 281, 499, - /* 1520 */ 250, 5, 497, 460, 345, 447, 569, 550, 238, 299, - /* 1530 */ 527, 525, 508, 961, 502, 501, 961, 401, 961, 211, - /* 1540 */ 490, 356, 256, 961, 483, 961, 961, 961, 961, 961, - /* 1550 */ 961, 961, 961, 961, 961, 961, 370, + /* 0 */ 306, 212, 432, 955, 639, 191, 955, 295, 559, 88, + /* 10 */ 88, 88, 88, 81, 86, 86, 86, 86, 85, 85, + /* 20 */ 84, 84, 84, 83, 330, 185, 184, 183, 635, 635, + /* 30 */ 292, 606, 606, 88, 88, 88, 88, 683, 86, 86, + /* 40 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 16, + /* 50 */ 436, 597, 89, 90, 80, 600, 599, 601, 601, 87, + /* 60 */ 87, 88, 88, 88, 88, 684, 86, 86, 86, 86, + /* 70 */ 85, 85, 84, 84, 84, 83, 330, 306, 559, 84, + /* 80 */ 84, 84, 83, 330, 65, 86, 86, 86, 86, 85, + /* 90 */ 85, 84, 84, 84, 83, 330, 635, 635, 634, 633, + /* 100 */ 182, 682, 550, 379, 376, 375, 17, 322, 606, 606, + /* 110 */ 371, 198, 479, 91, 374, 82, 79, 165, 85, 85, + /* 120 */ 84, 84, 84, 83, 330, 598, 635, 635, 107, 89, + /* 130 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 140 */ 88, 88, 186, 86, 86, 86, 86, 85, 85, 84, + /* 150 */ 84, 84, 83, 330, 306, 594, 594, 142, 328, 327, + /* 160 */ 484, 249, 344, 238, 635, 635, 634, 633, 585, 448, + /* 170 */ 526, 525, 229, 388, 1, 394, 450, 584, 449, 635, + /* 180 */ 635, 635, 635, 319, 395, 606, 606, 199, 157, 273, + /* 190 */ 382, 268, 381, 187, 635, 635, 634, 633, 311, 555, + /* 200 */ 266, 593, 593, 266, 347, 588, 89, 90, 80, 600, + /* 210 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 478, + /* 220 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, + /* 230 */ 330, 306, 272, 536, 634, 633, 146, 610, 197, 310, + /* 240 */ 575, 182, 482, 271, 379, 376, 375, 506, 21, 634, + /* 250 */ 633, 634, 633, 635, 635, 374, 611, 574, 548, 440, + /* 260 */ 111, 563, 606, 606, 634, 633, 324, 479, 608, 608, + /* 270 */ 608, 300, 435, 573, 119, 407, 210, 162, 562, 883, + /* 280 */ 592, 592, 306, 89, 90, 80, 600, 599, 601, 601, + /* 290 */ 87, 87, 88, 88, 88, 88, 506, 86, 86, 86, + /* 300 */ 86, 85, 85, 84, 84, 84, 83, 330, 620, 111, + /* 310 */ 635, 635, 361, 606, 606, 358, 249, 349, 248, 433, + /* 320 */ 243, 479, 586, 634, 633, 195, 611, 93, 119, 221, + /* 330 */ 575, 497, 534, 534, 89, 90, 80, 600, 599, 601, + /* 340 */ 601, 87, 87, 88, 88, 88, 88, 574, 86, 86, + /* 350 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 306, + /* 360 */ 77, 429, 638, 573, 589, 530, 240, 230, 242, 105, + /* 370 */ 249, 349, 248, 515, 588, 208, 460, 529, 564, 173, + /* 380 */ 634, 633, 970, 144, 430, 2, 424, 228, 380, 557, + /* 390 */ 606, 606, 190, 153, 159, 158, 514, 51, 632, 631, + /* 400 */ 630, 71, 536, 432, 954, 196, 610, 954, 614, 45, + /* 410 */ 18, 89, 90, 80, 600, 599, 601, 601, 87, 87, + /* 420 */ 88, 88, 88, 88, 261, 86, 86, 86, 86, 85, + /* 430 */ 85, 84, 84, 84, 83, 330, 306, 608, 608, 608, + /* 440 */ 542, 424, 402, 385, 241, 506, 451, 320, 211, 543, + /* 450 */ 164, 436, 386, 293, 451, 587, 108, 496, 111, 334, + /* 460 */ 391, 591, 424, 614, 27, 452, 453, 606, 606, 72, + /* 470 */ 257, 70, 259, 452, 339, 342, 564, 582, 68, 415, + /* 480 */ 469, 328, 327, 62, 614, 45, 110, 393, 89, 90, + /* 490 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, + /* 500 */ 88, 152, 86, 86, 86, 86, 85, 85, 84, 84, + /* 510 */ 84, 83, 330, 306, 110, 499, 520, 538, 402, 389, + /* 520 */ 424, 110, 566, 500, 593, 593, 454, 82, 79, 165, + /* 530 */ 424, 591, 384, 564, 340, 615, 188, 162, 424, 350, + /* 540 */ 616, 424, 614, 44, 606, 606, 445, 582, 300, 434, + /* 550 */ 151, 19, 614, 9, 568, 580, 348, 615, 469, 567, + /* 560 */ 614, 26, 616, 614, 45, 89, 90, 80, 600, 599, + /* 570 */ 601, 601, 87, 87, 88, 88, 88, 88, 411, 86, + /* 580 */ 86, 86, 86, 85, 85, 84, 84, 84, 83, 330, + /* 590 */ 306, 579, 110, 578, 521, 282, 433, 398, 400, 255, + /* 600 */ 486, 82, 79, 165, 487, 164, 82, 79, 165, 488, + /* 610 */ 488, 364, 387, 424, 544, 544, 509, 350, 362, 155, + /* 620 */ 191, 606, 606, 559, 642, 640, 333, 82, 79, 165, + /* 630 */ 305, 564, 507, 312, 357, 614, 45, 329, 596, 595, + /* 640 */ 194, 337, 89, 90, 80, 600, 599, 601, 601, 87, + /* 650 */ 87, 88, 88, 88, 88, 424, 86, 86, 86, 86, + /* 660 */ 85, 85, 84, 84, 84, 83, 330, 306, 20, 323, + /* 670 */ 150, 263, 211, 543, 421, 596, 595, 614, 22, 424, + /* 680 */ 193, 424, 284, 424, 391, 424, 509, 424, 577, 424, + /* 690 */ 186, 335, 424, 559, 424, 313, 120, 546, 606, 606, + /* 700 */ 67, 614, 47, 614, 50, 614, 48, 614, 100, 614, + /* 710 */ 99, 614, 101, 576, 614, 102, 614, 109, 326, 89, + /* 720 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 730 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, + /* 740 */ 84, 84, 83, 330, 306, 424, 311, 424, 585, 54, + /* 750 */ 424, 516, 517, 590, 614, 112, 424, 584, 424, 572, + /* 760 */ 424, 195, 424, 571, 424, 67, 424, 614, 94, 614, + /* 770 */ 98, 424, 614, 97, 264, 606, 606, 195, 614, 46, + /* 780 */ 614, 96, 614, 30, 614, 49, 614, 115, 614, 114, + /* 790 */ 418, 229, 388, 614, 113, 306, 89, 90, 80, 600, + /* 800 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 424, + /* 810 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, + /* 820 */ 330, 119, 424, 590, 110, 372, 606, 606, 195, 53, + /* 830 */ 250, 614, 29, 195, 472, 438, 729, 190, 302, 498, + /* 840 */ 14, 523, 641, 2, 614, 43, 306, 89, 90, 80, + /* 850 */ 600, 599, 601, 601, 87, 87, 88, 88, 88, 88, + /* 860 */ 424, 86, 86, 86, 86, 85, 85, 84, 84, 84, + /* 870 */ 83, 330, 424, 613, 964, 964, 354, 606, 606, 420, + /* 880 */ 312, 64, 614, 42, 391, 355, 283, 437, 301, 255, + /* 890 */ 414, 410, 495, 492, 614, 28, 471, 306, 89, 90, + /* 900 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, + /* 910 */ 88, 424, 86, 86, 86, 86, 85, 85, 84, 84, + /* 920 */ 84, 83, 330, 424, 110, 110, 110, 110, 606, 606, + /* 930 */ 110, 254, 13, 614, 41, 532, 531, 283, 481, 531, + /* 940 */ 457, 284, 119, 561, 356, 614, 40, 284, 306, 89, + /* 950 */ 78, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 960 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, + /* 970 */ 84, 84, 83, 330, 110, 424, 341, 220, 555, 606, + /* 980 */ 606, 351, 555, 318, 614, 95, 413, 255, 83, 330, + /* 990 */ 284, 284, 255, 640, 333, 356, 255, 614, 39, 306, + /* 1000 */ 356, 90, 80, 600, 599, 601, 601, 87, 87, 88, + /* 1010 */ 88, 88, 88, 424, 86, 86, 86, 86, 85, 85, + /* 1020 */ 84, 84, 84, 83, 330, 424, 317, 316, 141, 465, + /* 1030 */ 606, 606, 219, 619, 463, 614, 10, 417, 462, 255, + /* 1040 */ 189, 510, 553, 351, 207, 363, 161, 614, 38, 315, + /* 1050 */ 218, 255, 255, 80, 600, 599, 601, 601, 87, 87, + /* 1060 */ 88, 88, 88, 88, 424, 86, 86, 86, 86, 85, + /* 1070 */ 85, 84, 84, 84, 83, 330, 76, 419, 255, 3, + /* 1080 */ 878, 461, 424, 247, 331, 331, 614, 37, 217, 76, + /* 1090 */ 419, 390, 3, 216, 215, 422, 4, 331, 331, 424, + /* 1100 */ 547, 12, 424, 545, 614, 36, 424, 541, 422, 424, + /* 1110 */ 540, 424, 214, 424, 408, 424, 539, 403, 605, 605, + /* 1120 */ 237, 614, 25, 119, 614, 24, 588, 408, 614, 45, + /* 1130 */ 118, 614, 35, 614, 34, 614, 33, 614, 23, 588, + /* 1140 */ 60, 223, 603, 602, 513, 378, 73, 74, 140, 139, + /* 1150 */ 424, 110, 265, 75, 426, 425, 59, 424, 610, 73, + /* 1160 */ 74, 549, 402, 404, 424, 373, 75, 426, 425, 604, + /* 1170 */ 138, 610, 614, 11, 392, 76, 419, 181, 3, 614, + /* 1180 */ 32, 271, 369, 331, 331, 493, 614, 31, 149, 608, + /* 1190 */ 608, 608, 607, 15, 422, 365, 614, 8, 137, 489, + /* 1200 */ 136, 190, 608, 608, 608, 607, 15, 485, 176, 135, + /* 1210 */ 7, 252, 477, 408, 174, 133, 175, 474, 57, 56, + /* 1220 */ 132, 130, 119, 76, 419, 588, 3, 468, 245, 464, + /* 1230 */ 171, 331, 331, 125, 123, 456, 447, 122, 446, 104, + /* 1240 */ 336, 231, 422, 166, 154, 73, 74, 332, 116, 431, + /* 1250 */ 121, 309, 75, 426, 425, 222, 106, 610, 308, 637, + /* 1260 */ 204, 408, 629, 627, 628, 6, 200, 428, 427, 290, + /* 1270 */ 203, 622, 201, 588, 62, 63, 289, 66, 419, 399, + /* 1280 */ 3, 401, 288, 92, 143, 331, 331, 287, 608, 608, + /* 1290 */ 608, 607, 15, 73, 74, 227, 422, 325, 69, 416, + /* 1300 */ 75, 426, 425, 612, 412, 610, 192, 61, 569, 209, + /* 1310 */ 396, 226, 278, 225, 383, 408, 527, 558, 276, 533, + /* 1320 */ 552, 528, 321, 523, 370, 508, 180, 588, 494, 179, + /* 1330 */ 366, 117, 253, 269, 522, 503, 608, 608, 608, 607, + /* 1340 */ 15, 551, 502, 58, 274, 524, 178, 73, 74, 304, + /* 1350 */ 501, 368, 303, 206, 75, 426, 425, 491, 360, 610, + /* 1360 */ 213, 177, 483, 131, 345, 298, 297, 296, 202, 294, + /* 1370 */ 480, 490, 466, 134, 172, 129, 444, 346, 470, 128, + /* 1380 */ 314, 459, 103, 127, 126, 148, 124, 167, 443, 235, + /* 1390 */ 608, 608, 608, 607, 15, 442, 439, 623, 234, 299, + /* 1400 */ 145, 583, 291, 377, 581, 160, 119, 156, 270, 636, + /* 1410 */ 971, 169, 279, 626, 520, 625, 473, 624, 170, 621, + /* 1420 */ 618, 119, 168, 55, 409, 423, 537, 609, 286, 285, + /* 1430 */ 405, 570, 560, 556, 5, 52, 458, 554, 147, 267, + /* 1440 */ 519, 504, 518, 406, 262, 239, 260, 512, 343, 511, + /* 1450 */ 258, 353, 565, 256, 224, 251, 359, 277, 275, 476, + /* 1460 */ 475, 246, 352, 244, 467, 455, 236, 233, 232, 307, + /* 1470 */ 441, 281, 205, 163, 397, 280, 535, 505, 330, 617, + /* 1480 */ 971, 971, 971, 971, 367, 971, 971, 971, 971, 971, + /* 1490 */ 971, 971, 971, 971, 971, 971, 338, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 142, 143, 144, 145, 24, 1, 26, 77, 78, - /* 10 */ 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, - /* 20 */ 89, 90, 91, 92, 15, 98, 26, 27, 7, 8, - /* 30 */ 49, 50, 77, 78, 79, 80, 109, 82, 83, 84, - /* 40 */ 85, 86, 87, 88, 89, 90, 91, 92, 22, 68, - /* 50 */ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - /* 60 */ 79, 80, 23, 82, 83, 84, 85, 86, 87, 88, - /* 70 */ 89, 90, 91, 92, 19, 94, 112, 19, 114, 115, - /* 80 */ 25, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 90 */ 91, 92, 19, 22, 94, 95, 96, 150, 150, 99, - /* 100 */ 100, 101, 76, 150, 49, 50, 105, 106, 107, 54, - /* 110 */ 110, 158, 165, 165, 161, 162, 26, 27, 165, 113, - /* 120 */ 16, 174, 175, 68, 69, 70, 71, 72, 73, 74, - /* 130 */ 75, 76, 77, 78, 79, 80, 118, 82, 83, 84, - /* 140 */ 85, 86, 87, 88, 89, 90, 91, 92, 19, 23, - /* 150 */ 92, 97, 98, 24, 96, 97, 98, 99, 100, 101, - /* 160 */ 102, 25, 97, 216, 60, 92, 62, 109, 221, 25, - /* 170 */ 97, 98, 99, 100, 101, 102, 86, 87, 49, 50, - /* 180 */ 118, 116, 109, 25, 94, 95, 32, 97, 88, 89, - /* 190 */ 90, 91, 92, 128, 104, 41, 106, 68, 69, 70, - /* 200 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, - /* 210 */ 11, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 220 */ 91, 92, 19, 19, 86, 87, 88, 89, 90, 91, - /* 230 */ 92, 27, 96, 150, 66, 99, 100, 101, 112, 150, - /* 240 */ 114, 115, 138, 150, 161, 162, 110, 103, 165, 222, - /* 250 */ 223, 224, 49, 50, 165, 22, 57, 24, 165, 170, - /* 260 */ 171, 118, 94, 170, 171, 23, 98, 25, 185, 186, - /* 270 */ 243, 68, 69, 70, 71, 72, 73, 74, 75, 76, - /* 280 */ 77, 78, 79, 80, 126, 82, 83, 84, 85, 86, - /* 290 */ 87, 88, 89, 90, 91, 92, 19, 129, 130, 131, - /* 300 */ 88, 23, 172, 173, 105, 106, 107, 150, 22, 26, - /* 310 */ 27, 181, 26, 27, 22, 116, 26, 27, 26, 230, - /* 320 */ 231, 197, 165, 230, 231, 113, 49, 50, 204, 117, - /* 330 */ 96, 174, 175, 99, 100, 101, 22, 26, 27, 136, - /* 340 */ 26, 27, 118, 16, 110, 68, 69, 70, 71, 72, - /* 350 */ 73, 74, 75, 76, 77, 78, 79, 80, 118, 82, - /* 360 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 370 */ 19, 214, 215, 150, 23, 23, 155, 94, 95, 22, - /* 380 */ 94, 95, 116, 160, 94, 95, 94, 60, 165, 62, - /* 390 */ 112, 26, 114, 115, 128, 23, 36, 174, 175, 88, - /* 400 */ 49, 50, 57, 120, 22, 94, 95, 23, 94, 95, - /* 410 */ 120, 51, 25, 136, 169, 170, 171, 194, 58, 68, - /* 420 */ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - /* 430 */ 79, 80, 23, 82, 83, 84, 85, 86, 87, 88, - /* 440 */ 89, 90, 91, 92, 19, 150, 12, 12, 23, 228, - /* 450 */ 105, 106, 107, 23, 233, 25, 165, 19, 150, 94, - /* 460 */ 165, 116, 28, 28, 112, 174, 114, 115, 108, 174, - /* 470 */ 175, 26, 27, 165, 49, 50, 231, 11, 44, 44, - /* 480 */ 46, 46, 174, 175, 112, 160, 114, 115, 50, 22, - /* 490 */ 23, 57, 25, 68, 69, 70, 71, 72, 73, 74, - /* 500 */ 75, 76, 77, 78, 79, 80, 119, 82, 83, 84, - /* 510 */ 85, 86, 87, 88, 89, 90, 91, 92, 19, 194, - /* 520 */ 225, 23, 23, 215, 19, 95, 105, 106, 107, 150, - /* 530 */ 23, 150, 27, 23, 67, 25, 150, 206, 207, 94, - /* 540 */ 95, 166, 104, 218, 165, 22, 165, 109, 49, 50, - /* 550 */ 120, 165, 25, 174, 175, 174, 175, 23, 21, 234, - /* 560 */ 174, 175, 22, 23, 239, 25, 25, 68, 69, 70, - /* 570 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, - /* 580 */ 205, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 590 */ 91, 92, 19, 22, 23, 216, 23, 222, 223, 224, - /* 600 */ 63, 220, 35, 150, 150, 163, 220, 67, 166, 167, - /* 610 */ 168, 150, 169, 170, 171, 161, 162, 25, 165, 165, - /* 620 */ 150, 113, 49, 50, 25, 117, 165, 174, 175, 35, - /* 630 */ 7, 8, 9, 160, 160, 165, 120, 100, 67, 247, - /* 640 */ 248, 68, 69, 70, 71, 72, 73, 74, 75, 76, - /* 650 */ 77, 78, 79, 80, 193, 82, 83, 84, 85, 86, - /* 660 */ 87, 88, 89, 90, 91, 92, 19, 194, 194, 150, - /* 670 */ 135, 24, 137, 35, 231, 138, 150, 129, 130, 206, - /* 680 */ 207, 30, 27, 213, 165, 34, 118, 95, 0, 1, - /* 690 */ 2, 165, 218, 174, 175, 50, 49, 50, 22, 48, - /* 700 */ 174, 175, 22, 23, 23, 244, 222, 223, 224, 166, - /* 710 */ 167, 168, 120, 239, 23, 68, 69, 70, 71, 72, - /* 720 */ 73, 74, 75, 76, 77, 78, 79, 80, 150, 82, - /* 730 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 740 */ 19, 150, 173, 165, 181, 182, 24, 67, 26, 104, - /* 750 */ 181, 188, 174, 175, 150, 39, 165, 150, 52, 150, - /* 760 */ 150, 150, 150, 144, 145, 174, 175, 249, 250, 165, - /* 770 */ 49, 50, 165, 52, 165, 165, 165, 165, 174, 175, - /* 780 */ 29, 174, 175, 174, 175, 174, 175, 160, 22, 68, - /* 790 */ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - /* 800 */ 79, 80, 150, 82, 83, 84, 85, 86, 87, 88, - /* 810 */ 89, 90, 91, 92, 19, 150, 94, 165, 150, 150, - /* 820 */ 160, 194, 150, 213, 160, 52, 174, 175, 23, 23, - /* 830 */ 165, 25, 22, 165, 165, 150, 150, 165, 52, 174, - /* 840 */ 175, 22, 174, 175, 49, 50, 174, 175, 190, 191, - /* 850 */ 165, 165, 240, 23, 194, 25, 187, 109, 194, 174, - /* 860 */ 175, 190, 191, 68, 69, 70, 71, 72, 73, 74, - /* 870 */ 75, 76, 77, 78, 79, 80, 150, 82, 83, 84, - /* 880 */ 85, 86, 87, 88, 89, 90, 91, 92, 19, 150, - /* 890 */ 22, 165, 150, 23, 150, 25, 150, 166, 91, 92, - /* 900 */ 174, 175, 22, 217, 165, 150, 102, 165, 150, 165, - /* 910 */ 150, 165, 150, 174, 175, 19, 174, 175, 49, 50, - /* 920 */ 165, 86, 87, 165, 23, 165, 25, 165, 24, 174, - /* 930 */ 175, 187, 174, 175, 174, 175, 205, 68, 69, 70, - /* 940 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, - /* 950 */ 150, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 960 */ 91, 92, 19, 150, 150, 165, 150, 150, 166, 23, - /* 970 */ 150, 25, 160, 20, 174, 175, 1, 2, 165, 165, - /* 980 */ 104, 165, 165, 43, 150, 165, 240, 150, 49, 50, - /* 990 */ 174, 175, 49, 50, 23, 23, 25, 25, 53, 165, - /* 1000 */ 187, 187, 165, 23, 187, 25, 194, 205, 174, 175, - /* 1010 */ 71, 72, 69, 70, 71, 72, 73, 74, 75, 76, - /* 1020 */ 77, 78, 79, 80, 150, 82, 83, 84, 85, 86, - /* 1030 */ 87, 88, 89, 90, 91, 92, 19, 98, 150, 165, - /* 1040 */ 150, 160, 150, 59, 25, 53, 104, 22, 174, 175, - /* 1050 */ 213, 138, 5, 165, 1, 165, 150, 165, 150, 150, - /* 1060 */ 240, 150, 174, 175, 174, 175, 49, 50, 118, 150, - /* 1070 */ 35, 165, 27, 165, 165, 194, 165, 108, 127, 76, - /* 1080 */ 174, 175, 174, 175, 165, 174, 175, 70, 71, 72, - /* 1090 */ 73, 74, 75, 76, 77, 78, 79, 80, 166, 82, - /* 1100 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 1110 */ 19, 20, 193, 22, 150, 150, 150, 26, 27, 76, - /* 1120 */ 150, 22, 1, 150, 119, 121, 217, 20, 37, 165, - /* 1130 */ 165, 165, 16, 19, 20, 165, 22, 205, 165, 119, - /* 1140 */ 26, 27, 108, 128, 150, 150, 150, 56, 150, 22, - /* 1150 */ 150, 37, 150, 127, 160, 23, 150, 66, 193, 165, - /* 1160 */ 165, 165, 16, 165, 23, 165, 150, 165, 174, 175, - /* 1170 */ 56, 165, 150, 65, 174, 175, 15, 86, 87, 88, - /* 1180 */ 66, 165, 140, 150, 93, 94, 95, 165, 194, 98, - /* 1190 */ 174, 175, 22, 3, 164, 193, 174, 175, 165, 150, - /* 1200 */ 86, 87, 4, 180, 150, 248, 251, 93, 94, 95, - /* 1210 */ 216, 180, 98, 251, 165, 221, 150, 149, 6, 165, - /* 1220 */ 129, 130, 131, 132, 133, 134, 193, 150, 174, 175, - /* 1230 */ 116, 165, 19, 20, 150, 22, 149, 151, 150, 26, - /* 1240 */ 27, 149, 165, 129, 130, 131, 132, 133, 134, 165, - /* 1250 */ 37, 174, 175, 165, 149, 19, 20, 13, 22, 150, - /* 1260 */ 150, 150, 26, 27, 146, 147, 151, 150, 25, 56, - /* 1270 */ 152, 159, 154, 37, 165, 165, 165, 193, 160, 66, - /* 1280 */ 116, 193, 165, 174, 175, 174, 175, 194, 199, 150, - /* 1290 */ 200, 126, 56, 124, 123, 150, 201, 122, 150, 86, - /* 1300 */ 87, 150, 66, 193, 165, 202, 93, 94, 95, 150, - /* 1310 */ 165, 98, 194, 165, 125, 22, 165, 150, 150, 26, - /* 1320 */ 27, 135, 86, 87, 165, 174, 175, 203, 226, 93, - /* 1330 */ 94, 95, 165, 165, 98, 150, 218, 150, 193, 157, - /* 1340 */ 118, 157, 129, 130, 131, 132, 133, 134, 5, 104, - /* 1350 */ 165, 211, 165, 10, 11, 12, 13, 14, 150, 66, - /* 1360 */ 17, 174, 175, 210, 246, 129, 130, 131, 132, 133, - /* 1370 */ 134, 150, 210, 165, 31, 121, 33, 150, 150, 86, - /* 1380 */ 87, 176, 174, 175, 150, 42, 165, 94, 211, 210, - /* 1390 */ 150, 98, 165, 165, 211, 174, 175, 150, 55, 165, - /* 1400 */ 57, 150, 174, 175, 61, 165, 150, 64, 174, 175, - /* 1410 */ 150, 150, 165, 150, 174, 175, 165, 104, 150, 184, - /* 1420 */ 150, 165, 129, 130, 131, 165, 165, 150, 165, 150, - /* 1430 */ 150, 176, 150, 165, 47, 165, 150, 150, 176, 103, - /* 1440 */ 150, 22, 165, 178, 165, 165, 179, 165, 105, 106, - /* 1450 */ 107, 165, 165, 229, 111, 165, 92, 176, 229, 116, - /* 1460 */ 184, 176, 179, 156, 176, 176, 18, 157, 156, 237, - /* 1470 */ 45, 157, 156, 135, 157, 157, 238, 156, 68, 157, - /* 1480 */ 189, 189, 139, 219, 22, 157, 18, 192, 192, 192, - /* 1490 */ 192, 189, 219, 199, 157, 242, 40, 157, 199, 242, - /* 1500 */ 153, 157, 38, 245, 196, 166, 232, 198, 177, 177, - /* 1510 */ 232, 227, 209, 178, 166, 182, 166, 148, 177, 177, - /* 1520 */ 209, 196, 177, 199, 209, 199, 166, 208, 92, 195, - /* 1530 */ 174, 174, 183, 252, 183, 183, 252, 191, 252, 235, - /* 1540 */ 186, 241, 241, 252, 186, 252, 252, 252, 252, 252, - /* 1550 */ 252, 252, 252, 252, 252, 252, 236, + /* 0 */ 19, 22, 22, 23, 1, 24, 26, 15, 27, 80, + /* 10 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + /* 20 */ 91, 92, 93, 94, 95, 108, 109, 110, 27, 28, + /* 30 */ 23, 50, 51, 80, 81, 82, 83, 122, 85, 86, + /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 22, + /* 50 */ 70, 23, 71, 72, 73, 74, 75, 76, 77, 78, + /* 60 */ 79, 80, 81, 82, 83, 122, 85, 86, 87, 88, + /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 91, + /* 80 */ 92, 93, 94, 95, 26, 85, 86, 87, 88, 89, + /* 90 */ 90, 91, 92, 93, 94, 95, 27, 28, 97, 98, + /* 100 */ 99, 122, 211, 102, 103, 104, 79, 19, 50, 51, + /* 110 */ 19, 122, 59, 55, 113, 224, 225, 226, 89, 90, + /* 120 */ 91, 92, 93, 94, 95, 23, 27, 28, 26, 71, + /* 130 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 140 */ 82, 83, 51, 85, 86, 87, 88, 89, 90, 91, + /* 150 */ 92, 93, 94, 95, 19, 132, 133, 58, 89, 90, + /* 160 */ 21, 108, 109, 110, 27, 28, 97, 98, 33, 100, + /* 170 */ 7, 8, 119, 120, 22, 19, 107, 42, 109, 27, + /* 180 */ 28, 27, 28, 95, 28, 50, 51, 99, 100, 101, + /* 190 */ 102, 103, 104, 105, 27, 28, 97, 98, 107, 152, + /* 200 */ 112, 132, 133, 112, 65, 69, 71, 72, 73, 74, + /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 11, + /* 220 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 230 */ 95, 19, 101, 97, 97, 98, 24, 101, 122, 157, + /* 240 */ 12, 99, 103, 112, 102, 103, 104, 152, 22, 97, + /* 250 */ 98, 97, 98, 27, 28, 113, 27, 29, 91, 164, + /* 260 */ 165, 124, 50, 51, 97, 98, 219, 59, 132, 133, + /* 270 */ 134, 22, 23, 45, 66, 47, 212, 213, 124, 140, + /* 280 */ 132, 133, 19, 71, 72, 73, 74, 75, 76, 77, + /* 290 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, + /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 164, 165, + /* 310 */ 27, 28, 230, 50, 51, 233, 108, 109, 110, 70, + /* 320 */ 16, 59, 23, 97, 98, 26, 97, 22, 66, 185, + /* 330 */ 12, 187, 27, 28, 71, 72, 73, 74, 75, 76, + /* 340 */ 77, 78, 79, 80, 81, 82, 83, 29, 85, 86, + /* 350 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 19, + /* 360 */ 22, 148, 149, 45, 23, 47, 62, 154, 64, 156, + /* 370 */ 108, 109, 110, 37, 69, 23, 163, 59, 26, 26, + /* 380 */ 97, 98, 144, 145, 146, 147, 152, 200, 52, 23, + /* 390 */ 50, 51, 26, 22, 89, 90, 60, 210, 7, 8, + /* 400 */ 9, 138, 97, 22, 23, 26, 101, 26, 174, 175, + /* 410 */ 197, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 420 */ 80, 81, 82, 83, 16, 85, 86, 87, 88, 89, + /* 430 */ 90, 91, 92, 93, 94, 95, 19, 132, 133, 134, + /* 440 */ 23, 152, 208, 209, 140, 152, 152, 111, 195, 196, + /* 450 */ 98, 70, 163, 160, 152, 23, 22, 164, 165, 246, + /* 460 */ 207, 27, 152, 174, 175, 171, 172, 50, 51, 137, + /* 470 */ 62, 139, 64, 171, 172, 222, 124, 27, 138, 24, + /* 480 */ 163, 89, 90, 130, 174, 175, 197, 163, 71, 72, + /* 490 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + /* 500 */ 83, 22, 85, 86, 87, 88, 89, 90, 91, 92, + /* 510 */ 93, 94, 95, 19, 197, 181, 182, 23, 208, 209, + /* 520 */ 152, 197, 26, 189, 132, 133, 232, 224, 225, 226, + /* 530 */ 152, 97, 91, 26, 232, 116, 212, 213, 152, 222, + /* 540 */ 121, 152, 174, 175, 50, 51, 243, 97, 22, 23, + /* 550 */ 22, 234, 174, 175, 177, 23, 239, 116, 163, 177, + /* 560 */ 174, 175, 121, 174, 175, 71, 72, 73, 74, 75, + /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 24, 85, + /* 580 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 590 */ 19, 23, 197, 11, 23, 227, 70, 208, 220, 152, + /* 600 */ 31, 224, 225, 226, 35, 98, 224, 225, 226, 108, + /* 610 */ 109, 110, 115, 152, 117, 118, 27, 222, 49, 123, + /* 620 */ 24, 50, 51, 27, 0, 1, 2, 224, 225, 226, + /* 630 */ 166, 124, 168, 169, 239, 174, 175, 170, 171, 172, + /* 640 */ 22, 194, 71, 72, 73, 74, 75, 76, 77, 78, + /* 650 */ 79, 80, 81, 82, 83, 152, 85, 86, 87, 88, + /* 660 */ 89, 90, 91, 92, 93, 94, 95, 19, 22, 208, + /* 670 */ 24, 23, 195, 196, 170, 171, 172, 174, 175, 152, + /* 680 */ 26, 152, 152, 152, 207, 152, 97, 152, 23, 152, + /* 690 */ 51, 244, 152, 97, 152, 247, 248, 23, 50, 51, + /* 700 */ 26, 174, 175, 174, 175, 174, 175, 174, 175, 174, + /* 710 */ 175, 174, 175, 23, 174, 175, 174, 175, 188, 71, + /* 720 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 730 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, + /* 740 */ 92, 93, 94, 95, 19, 152, 107, 152, 33, 24, + /* 750 */ 152, 100, 101, 27, 174, 175, 152, 42, 152, 23, + /* 760 */ 152, 26, 152, 23, 152, 26, 152, 174, 175, 174, + /* 770 */ 175, 152, 174, 175, 23, 50, 51, 26, 174, 175, + /* 780 */ 174, 175, 174, 175, 174, 175, 174, 175, 174, 175, + /* 790 */ 163, 119, 120, 174, 175, 19, 71, 72, 73, 74, + /* 800 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 152, + /* 810 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 820 */ 95, 66, 152, 97, 197, 23, 50, 51, 26, 53, + /* 830 */ 23, 174, 175, 26, 23, 23, 23, 26, 26, 26, + /* 840 */ 36, 106, 146, 147, 174, 175, 19, 71, 72, 73, + /* 850 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + /* 860 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 870 */ 94, 95, 152, 196, 119, 120, 19, 50, 51, 168, + /* 880 */ 169, 26, 174, 175, 207, 28, 152, 249, 250, 152, + /* 890 */ 163, 163, 163, 163, 174, 175, 163, 19, 71, 72, + /* 900 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + /* 910 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, + /* 920 */ 93, 94, 95, 152, 197, 197, 197, 197, 50, 51, + /* 930 */ 197, 194, 36, 174, 175, 191, 192, 152, 191, 192, + /* 940 */ 163, 152, 66, 124, 152, 174, 175, 152, 19, 71, + /* 950 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 960 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, + /* 970 */ 92, 93, 94, 95, 197, 152, 100, 188, 152, 50, + /* 980 */ 51, 152, 152, 188, 174, 175, 252, 152, 94, 95, + /* 990 */ 152, 152, 152, 1, 2, 152, 152, 174, 175, 19, + /* 1000 */ 152, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 1010 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, + /* 1020 */ 91, 92, 93, 94, 95, 152, 188, 188, 22, 194, + /* 1030 */ 50, 51, 240, 173, 194, 174, 175, 252, 194, 152, + /* 1040 */ 36, 181, 28, 152, 23, 219, 122, 174, 175, 219, + /* 1050 */ 221, 152, 152, 73, 74, 75, 76, 77, 78, 79, + /* 1060 */ 80, 81, 82, 83, 152, 85, 86, 87, 88, 89, + /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 152, 22, + /* 1080 */ 23, 194, 152, 240, 27, 28, 174, 175, 240, 19, + /* 1090 */ 20, 26, 22, 194, 194, 38, 22, 27, 28, 152, + /* 1100 */ 23, 22, 152, 116, 174, 175, 152, 23, 38, 152, + /* 1110 */ 23, 152, 221, 152, 57, 152, 23, 163, 50, 51, + /* 1120 */ 194, 174, 175, 66, 174, 175, 69, 57, 174, 175, + /* 1130 */ 40, 174, 175, 174, 175, 174, 175, 174, 175, 69, + /* 1140 */ 22, 53, 74, 75, 30, 53, 89, 90, 22, 22, + /* 1150 */ 152, 197, 23, 96, 97, 98, 22, 152, 101, 89, + /* 1160 */ 90, 91, 208, 209, 152, 53, 96, 97, 98, 101, + /* 1170 */ 22, 101, 174, 175, 152, 19, 20, 105, 22, 174, + /* 1180 */ 175, 112, 19, 27, 28, 20, 174, 175, 24, 132, + /* 1190 */ 133, 134, 135, 136, 38, 44, 174, 175, 107, 61, + /* 1200 */ 54, 26, 132, 133, 134, 135, 136, 54, 107, 22, + /* 1210 */ 5, 140, 1, 57, 36, 111, 122, 28, 79, 79, + /* 1220 */ 131, 123, 66, 19, 20, 69, 22, 1, 16, 20, + /* 1230 */ 125, 27, 28, 123, 111, 120, 23, 131, 23, 16, + /* 1240 */ 68, 142, 38, 15, 22, 89, 90, 3, 167, 4, + /* 1250 */ 248, 251, 96, 97, 98, 180, 180, 101, 251, 151, + /* 1260 */ 6, 57, 151, 13, 151, 26, 25, 151, 161, 202, + /* 1270 */ 153, 162, 153, 69, 130, 128, 203, 19, 20, 127, + /* 1280 */ 22, 126, 204, 129, 22, 27, 28, 205, 132, 133, + /* 1290 */ 134, 135, 136, 89, 90, 231, 38, 95, 137, 179, + /* 1300 */ 96, 97, 98, 206, 179, 101, 122, 107, 159, 159, + /* 1310 */ 125, 231, 216, 228, 107, 57, 184, 217, 216, 176, + /* 1320 */ 217, 176, 48, 106, 18, 184, 158, 69, 159, 158, + /* 1330 */ 46, 71, 237, 176, 176, 176, 132, 133, 134, 135, + /* 1340 */ 136, 217, 176, 137, 216, 178, 158, 89, 90, 179, + /* 1350 */ 176, 159, 179, 159, 96, 97, 98, 159, 159, 101, + /* 1360 */ 5, 158, 202, 22, 18, 10, 11, 12, 13, 14, + /* 1370 */ 190, 238, 17, 190, 158, 193, 41, 159, 202, 193, + /* 1380 */ 159, 202, 245, 193, 193, 223, 190, 32, 159, 34, + /* 1390 */ 132, 133, 134, 135, 136, 159, 39, 155, 43, 150, + /* 1400 */ 223, 177, 201, 178, 177, 186, 66, 199, 177, 152, + /* 1410 */ 253, 56, 215, 152, 182, 152, 202, 152, 63, 152, + /* 1420 */ 152, 66, 67, 242, 229, 152, 174, 152, 152, 152, + /* 1430 */ 152, 152, 152, 152, 199, 242, 202, 152, 198, 152, + /* 1440 */ 152, 152, 183, 192, 152, 215, 152, 183, 215, 183, + /* 1450 */ 152, 241, 214, 152, 211, 152, 152, 211, 211, 152, + /* 1460 */ 152, 241, 152, 152, 152, 152, 152, 152, 152, 114, + /* 1470 */ 152, 152, 235, 152, 152, 152, 174, 187, 95, 174, + /* 1480 */ 253, 253, 253, 253, 236, 253, 253, 253, 253, 253, + /* 1490 */ 253, 253, 253, 253, 253, 253, 141, }; -#define YY_SHIFT_USE_DFLT (-74) -#define YY_SHIFT_COUNT (418) -#define YY_SHIFT_MIN (-73) -#define YY_SHIFT_MAX (1468) +#define YY_SHIFT_USE_DFLT (-86) +#define YY_SHIFT_COUNT (429) +#define YY_SHIFT_MIN (-85) +#define YY_SHIFT_MAX (1383) static const short yy_shift_ofst[] = { - /* 0 */ 975, 1114, 1343, 1114, 1213, 1213, 90, 90, 0, -19, - /* 10 */ 1213, 1213, 1213, 1213, 1213, 345, 445, 721, 1091, 1213, - /* 20 */ 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, - /* 30 */ 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, - /* 40 */ 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1236, 1213, 1213, - /* 50 */ 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, 1213, - /* 60 */ 1213, 199, 445, 445, 835, 835, 365, 1164, 55, 647, - /* 70 */ 573, 499, 425, 351, 277, 203, 129, 795, 795, 795, - /* 80 */ 795, 795, 795, 795, 795, 795, 795, 795, 795, 795, - /* 90 */ 795, 795, 795, 795, 795, 869, 795, 943, 1017, 1017, - /* 100 */ -69, -45, -45, -45, -45, -45, -1, 58, 138, 100, - /* 110 */ 445, 445, 445, 445, 445, 445, 445, 445, 445, 445, - /* 120 */ 445, 445, 445, 445, 445, 445, 537, 438, 445, 445, - /* 130 */ 445, 445, 445, 365, 807, 1436, -74, -74, -74, 1293, - /* 140 */ 73, 434, 434, 311, 314, 290, 283, 286, 540, 467, - /* 150 */ 445, 445, 445, 445, 445, 445, 445, 445, 445, 445, - /* 160 */ 445, 445, 445, 445, 445, 445, 445, 445, 445, 445, - /* 170 */ 445, 445, 445, 445, 445, 445, 445, 445, 445, 445, - /* 180 */ 445, 445, 65, 722, 722, 722, 688, 266, 1164, 1164, - /* 190 */ 1164, -74, -74, -74, 136, 168, 168, 234, 360, 360, - /* 200 */ 360, 430, 372, 435, 352, 278, 126, -36, -36, -36, - /* 210 */ -36, 421, 651, -36, -36, 592, 292, 212, 623, 158, - /* 220 */ 204, 204, 505, 158, 505, 144, 365, 154, 365, 154, - /* 230 */ 645, 154, 204, 154, 154, 535, 548, 548, 365, 387, - /* 240 */ 508, 233, 1464, 1222, 1222, 1456, 1456, 1222, 1462, 1410, - /* 250 */ 1165, 1468, 1468, 1468, 1468, 1222, 1165, 1462, 1410, 1410, - /* 260 */ 1222, 1448, 1338, 1425, 1222, 1222, 1448, 1222, 1448, 1222, - /* 270 */ 1448, 1419, 1313, 1313, 1313, 1387, 1364, 1364, 1419, 1313, - /* 280 */ 1336, 1313, 1387, 1313, 1313, 1254, 1245, 1254, 1245, 1254, - /* 290 */ 1245, 1222, 1222, 1186, 1189, 1175, 1169, 1171, 1165, 1164, - /* 300 */ 1243, 1244, 1244, 1212, 1212, 1212, 1212, -74, -74, -74, - /* 310 */ -74, -74, -74, 939, 104, 680, 571, 327, 1, 980, - /* 320 */ 26, 972, 971, 946, 901, 870, 830, 806, 54, 21, - /* 330 */ -73, 510, 242, 1198, 1190, 1170, 1042, 1161, 1108, 1146, - /* 340 */ 1141, 1132, 1015, 1127, 1026, 1034, 1020, 1107, 1004, 1116, - /* 350 */ 1121, 1005, 1099, 951, 1043, 1003, 969, 1045, 1035, 950, - /* 360 */ 1053, 1047, 1025, 942, 913, 992, 1019, 945, 984, 940, - /* 370 */ 876, 904, 953, 896, 748, 804, 880, 786, 868, 819, - /* 380 */ 805, 810, 773, 751, 766, 706, 716, 691, 681, 568, - /* 390 */ 655, 638, 676, 516, 541, 594, 599, 567, 541, 534, - /* 400 */ 507, 527, 498, 523, 466, 382, 409, 384, 357, 6, - /* 410 */ 240, 224, 143, 62, 18, 71, 39, 9, 5, + /* 0 */ 992, 1057, 1355, 1156, 1204, 1204, 1, 262, -19, 135, + /* 10 */ 135, 776, 1204, 1204, 1204, 1204, 69, 69, 53, 208, + /* 20 */ 283, 755, 58, 725, 648, 571, 494, 417, 340, 263, + /* 30 */ 212, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 40 */ 827, 827, 827, 827, 827, 827, 878, 827, 929, 980, + /* 50 */ 980, 1070, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 60 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 70 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 80 */ 1258, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 90 */ 1204, 1204, 1204, 1204, -71, -47, -47, -47, -47, -47, + /* 100 */ 0, 29, -12, 283, 283, 139, 91, 392, 392, 894, + /* 110 */ 672, 726, 1383, -86, -86, -86, 88, 318, 318, 99, + /* 120 */ 381, -20, 283, 283, 283, 283, 283, 283, 283, 283, + /* 130 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + /* 140 */ 283, 283, 283, 283, 624, 876, 726, 672, 1340, 1340, + /* 150 */ 1340, 1340, 1340, 1340, -86, -86, -86, 305, 136, 136, + /* 160 */ 142, 167, 226, 154, 137, 152, 283, 283, 283, 283, + /* 170 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + /* 180 */ 283, 283, 283, 336, 336, 336, 283, 283, 352, 283, + /* 190 */ 283, 283, 283, 283, 228, 283, 283, 283, 283, 283, + /* 200 */ 283, 283, 283, 283, 283, 501, 569, 596, 596, 596, + /* 210 */ 507, 497, 441, 391, 353, 156, 156, 857, 353, 857, + /* 220 */ 735, 813, 639, 715, 156, 332, 715, 715, 496, 419, + /* 230 */ 646, 1357, 1184, 1184, 1335, 1335, 1184, 1341, 1260, 1144, + /* 240 */ 1346, 1346, 1346, 1346, 1184, 1306, 1144, 1341, 1260, 1260, + /* 250 */ 1144, 1184, 1306, 1206, 1284, 1184, 1184, 1306, 1184, 1306, + /* 260 */ 1184, 1306, 1262, 1207, 1207, 1207, 1274, 1262, 1207, 1217, + /* 270 */ 1207, 1274, 1207, 1207, 1185, 1200, 1185, 1200, 1185, 1200, + /* 280 */ 1184, 1184, 1161, 1262, 1202, 1202, 1262, 1154, 1155, 1147, + /* 290 */ 1152, 1144, 1241, 1239, 1250, 1250, 1254, 1254, 1254, 1254, + /* 300 */ -86, -86, -86, -86, -86, -86, 1068, 304, 526, 249, + /* 310 */ 408, -83, 434, 812, 27, 811, 807, 802, 751, 589, + /* 320 */ 651, 163, 131, 674, 366, 450, 299, 148, 23, 102, + /* 330 */ 229, -21, 1245, 1244, 1222, 1099, 1228, 1172, 1223, 1215, + /* 340 */ 1213, 1115, 1106, 1123, 1110, 1209, 1105, 1212, 1226, 1098, + /* 350 */ 1089, 1140, 1139, 1104, 1189, 1178, 1094, 1211, 1205, 1187, + /* 360 */ 1101, 1071, 1153, 1175, 1146, 1138, 1151, 1091, 1164, 1165, + /* 370 */ 1163, 1069, 1072, 1148, 1112, 1134, 1127, 1129, 1126, 1092, + /* 380 */ 1114, 1118, 1088, 1090, 1093, 1087, 1084, 987, 1079, 1077, + /* 390 */ 1074, 1065, 924, 1021, 1014, 1004, 1006, 819, 739, 896, + /* 400 */ 855, 804, 739, 740, 736, 690, 654, 665, 618, 582, + /* 410 */ 568, 528, 554, 379, 532, 479, 455, 379, 432, 371, + /* 420 */ 341, 28, 338, 116, -11, -57, -85, 7, -8, 3, }; -#define YY_REDUCE_USE_DFLT (-142) -#define YY_REDUCE_COUNT (312) -#define YY_REDUCE_MIN (-141) -#define YY_REDUCE_MAX (1369) +#define YY_REDUCE_USE_DFLT (-110) +#define YY_REDUCE_COUNT (305) +#define YY_REDUCE_MIN (-109) +#define YY_REDUCE_MAX (1323) static const short yy_reduce_ofst[] = { - /* 0 */ -141, 994, 1118, 223, 157, -53, 93, 89, 83, 375, - /* 10 */ 386, 381, 379, 308, 295, 325, -47, 27, 1240, 1234, - /* 20 */ 1228, 1221, 1208, 1187, 1151, 1111, 1109, 1077, 1054, 1022, - /* 30 */ 1016, 1000, 911, 908, 906, 890, 888, 874, 834, 816, - /* 40 */ 800, 760, 758, 755, 742, 739, 726, 685, 672, 668, - /* 50 */ 665, 652, 611, 609, 607, 604, 591, 578, 526, 519, - /* 60 */ 453, 474, 454, 461, 443, 245, 442, 473, 484, 484, - /* 70 */ 484, 484, 484, 484, 484, 484, 484, 484, 484, 484, - /* 80 */ 484, 484, 484, 484, 484, 484, 484, 484, 484, 484, - /* 90 */ 484, 484, 484, 484, 484, 484, 484, 484, 484, 484, - /* 100 */ 484, 484, 484, 484, 484, 484, 484, 130, 484, 484, - /* 110 */ 1145, 909, 1110, 1088, 1084, 1033, 1002, 965, 820, 837, - /* 120 */ 746, 686, 612, 817, 610, 919, 221, 563, 814, 813, - /* 130 */ 744, 669, 470, 543, 484, 484, 484, 484, 484, 291, - /* 140 */ 569, 671, 658, 970, 1290, 1287, 1286, 1282, 518, 518, - /* 150 */ 1280, 1279, 1277, 1270, 1268, 1263, 1261, 1260, 1256, 1251, - /* 160 */ 1247, 1227, 1185, 1168, 1167, 1159, 1148, 1139, 1117, 1066, - /* 170 */ 1049, 1006, 998, 996, 995, 973, 970, 966, 964, 892, - /* 180 */ 762, -52, 881, 932, 802, 731, 619, 812, 664, 660, - /* 190 */ 627, 392, 331, 124, 1358, 1357, 1356, 1354, 1352, 1351, - /* 200 */ 1349, 1319, 1334, 1346, 1334, 1334, 1334, 1334, 1334, 1334, - /* 210 */ 1334, 1320, 1304, 1334, 1334, 1319, 1360, 1325, 1369, 1326, - /* 220 */ 1315, 1311, 1301, 1324, 1300, 1335, 1350, 1345, 1348, 1342, - /* 230 */ 1333, 1341, 1303, 1332, 1331, 1284, 1278, 1274, 1339, 1309, - /* 240 */ 1308, 1347, 1258, 1344, 1340, 1257, 1253, 1337, 1273, 1302, - /* 250 */ 1299, 1298, 1297, 1296, 1295, 1328, 1294, 1264, 1292, 1291, - /* 260 */ 1322, 1321, 1238, 1232, 1318, 1317, 1316, 1314, 1312, 1310, - /* 270 */ 1307, 1283, 1289, 1288, 1285, 1276, 1229, 1224, 1267, 1281, - /* 280 */ 1265, 1262, 1235, 1255, 1205, 1183, 1179, 1177, 1162, 1140, - /* 290 */ 1153, 1184, 1182, 1102, 1124, 1103, 1095, 1090, 1089, 1093, - /* 300 */ 1112, 1115, 1086, 1105, 1092, 1087, 1068, 962, 955, 957, - /* 310 */ 1031, 1023, 1030, + /* 0 */ 238, 954, 213, 289, 310, 234, 144, 317, -109, 382, + /* 10 */ 377, 303, 461, 389, 378, 368, 302, 294, 253, 395, + /* 20 */ 293, 324, 403, 403, 403, 403, 403, 403, 403, 403, + /* 30 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, + /* 40 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, + /* 50 */ 403, 1022, 1012, 1005, 998, 963, 961, 959, 957, 950, + /* 60 */ 947, 930, 912, 873, 861, 823, 810, 771, 759, 720, + /* 70 */ 708, 670, 657, 619, 614, 612, 610, 608, 606, 604, + /* 80 */ 598, 595, 593, 580, 542, 540, 537, 535, 533, 531, + /* 90 */ 529, 527, 503, 386, 403, 403, 403, 403, 403, 403, + /* 100 */ 403, 403, 403, 95, 447, 82, 334, 504, 467, 403, + /* 110 */ 477, 464, 403, 403, 403, 403, 860, 747, 744, 785, + /* 120 */ 638, 638, 926, 891, 900, 899, 887, 844, 840, 835, + /* 130 */ 848, 830, 843, 829, 792, 839, 826, 737, 838, 795, + /* 140 */ 789, 47, 734, 530, 696, 777, 711, 677, 733, 730, + /* 150 */ 729, 728, 727, 627, 448, 64, 187, 1305, 1302, 1252, + /* 160 */ 1290, 1273, 1323, 1322, 1321, 1319, 1318, 1316, 1315, 1314, + /* 170 */ 1313, 1312, 1311, 1310, 1308, 1307, 1304, 1303, 1301, 1298, + /* 180 */ 1294, 1292, 1289, 1266, 1264, 1259, 1288, 1287, 1238, 1285, + /* 190 */ 1281, 1280, 1279, 1278, 1251, 1277, 1276, 1275, 1273, 1268, + /* 200 */ 1267, 1265, 1263, 1261, 1257, 1248, 1237, 1247, 1246, 1243, + /* 210 */ 1238, 1240, 1235, 1249, 1234, 1233, 1230, 1220, 1214, 1210, + /* 220 */ 1225, 1219, 1232, 1231, 1197, 1195, 1227, 1224, 1201, 1208, + /* 230 */ 1242, 1137, 1236, 1229, 1193, 1181, 1221, 1177, 1196, 1179, + /* 240 */ 1191, 1190, 1186, 1182, 1218, 1216, 1176, 1162, 1183, 1180, + /* 250 */ 1160, 1199, 1203, 1133, 1095, 1198, 1194, 1188, 1192, 1171, + /* 260 */ 1169, 1168, 1173, 1174, 1166, 1159, 1141, 1170, 1158, 1167, + /* 270 */ 1157, 1132, 1145, 1143, 1124, 1128, 1103, 1102, 1100, 1096, + /* 280 */ 1150, 1149, 1085, 1125, 1080, 1064, 1120, 1097, 1082, 1078, + /* 290 */ 1073, 1067, 1109, 1107, 1119, 1117, 1116, 1113, 1111, 1108, + /* 300 */ 1007, 1000, 1002, 1076, 1075, 1081, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 635, 870, 959, 959, 959, 870, 899, 899, 959, 759, - /* 10 */ 959, 959, 959, 959, 868, 959, 959, 933, 959, 959, - /* 20 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 30 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 40 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 50 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 60 */ 959, 959, 959, 959, 899, 899, 674, 763, 794, 959, - /* 70 */ 959, 959, 959, 959, 959, 959, 959, 932, 934, 809, - /* 80 */ 808, 802, 801, 912, 774, 799, 792, 785, 796, 871, - /* 90 */ 864, 865, 863, 867, 872, 959, 795, 831, 848, 830, - /* 100 */ 842, 847, 854, 846, 843, 833, 832, 666, 834, 835, - /* 110 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 120 */ 959, 959, 959, 959, 959, 959, 661, 728, 959, 959, - /* 130 */ 959, 959, 959, 959, 836, 837, 851, 850, 849, 959, - /* 140 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 150 */ 959, 939, 937, 959, 883, 959, 959, 959, 959, 959, - /* 160 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 170 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 180 */ 959, 641, 959, 759, 759, 759, 635, 959, 959, 959, - /* 190 */ 959, 951, 763, 753, 719, 959, 959, 959, 959, 959, - /* 200 */ 959, 959, 959, 959, 959, 959, 959, 804, 742, 922, - /* 210 */ 924, 959, 905, 740, 663, 761, 676, 751, 643, 798, - /* 220 */ 776, 776, 917, 798, 917, 700, 959, 788, 959, 788, - /* 230 */ 697, 788, 776, 788, 788, 866, 959, 959, 959, 760, - /* 240 */ 751, 959, 944, 767, 767, 936, 936, 767, 810, 732, - /* 250 */ 798, 739, 739, 739, 739, 767, 798, 810, 732, 732, - /* 260 */ 767, 658, 911, 909, 767, 767, 658, 767, 658, 767, - /* 270 */ 658, 876, 730, 730, 730, 715, 880, 880, 876, 730, - /* 280 */ 700, 730, 715, 730, 730, 780, 775, 780, 775, 780, - /* 290 */ 775, 767, 767, 959, 793, 781, 791, 789, 798, 959, - /* 300 */ 718, 651, 651, 640, 640, 640, 640, 956, 956, 951, - /* 310 */ 702, 702, 684, 959, 959, 959, 959, 959, 959, 959, - /* 320 */ 885, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 330 */ 959, 959, 959, 959, 636, 946, 959, 959, 943, 959, - /* 340 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 350 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 915, - /* 360 */ 959, 959, 959, 959, 959, 959, 908, 907, 959, 959, - /* 370 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 380 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 390 */ 959, 959, 959, 959, 790, 959, 782, 959, 869, 959, - /* 400 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 745, - /* 410 */ 819, 959, 818, 822, 817, 668, 959, 649, 959, 632, - /* 420 */ 637, 955, 958, 957, 954, 953, 952, 947, 945, 942, - /* 430 */ 941, 940, 938, 935, 931, 889, 887, 894, 893, 892, - /* 440 */ 891, 890, 888, 886, 884, 805, 803, 800, 797, 930, - /* 450 */ 882, 741, 738, 737, 657, 948, 914, 923, 921, 811, - /* 460 */ 920, 919, 918, 916, 913, 900, 807, 806, 733, 874, - /* 470 */ 873, 660, 904, 903, 902, 906, 910, 901, 769, 659, - /* 480 */ 656, 665, 722, 721, 729, 727, 726, 725, 724, 723, - /* 490 */ 720, 667, 675, 686, 714, 699, 698, 879, 881, 878, - /* 500 */ 877, 707, 706, 712, 711, 710, 709, 708, 705, 704, - /* 510 */ 703, 696, 695, 701, 694, 717, 716, 713, 693, 736, - /* 520 */ 735, 734, 731, 692, 691, 690, 822, 689, 688, 828, - /* 530 */ 827, 815, 858, 756, 755, 754, 766, 765, 778, 777, - /* 540 */ 813, 812, 779, 764, 758, 757, 773, 772, 771, 770, - /* 550 */ 762, 752, 784, 787, 786, 783, 860, 768, 857, 929, - /* 560 */ 928, 927, 926, 925, 862, 861, 829, 826, 679, 680, - /* 570 */ 898, 896, 897, 895, 682, 681, 678, 677, 859, 747, - /* 580 */ 746, 855, 852, 844, 840, 856, 853, 845, 841, 839, - /* 590 */ 838, 824, 823, 821, 820, 816, 825, 670, 748, 744, - /* 600 */ 743, 814, 750, 749, 687, 685, 683, 664, 662, 655, - /* 610 */ 653, 652, 654, 650, 648, 647, 646, 645, 644, 673, - /* 620 */ 672, 671, 669, 668, 642, 639, 638, 634, 633, 631, + /* 0 */ 647, 964, 964, 964, 878, 878, 969, 964, 774, 802, + /* 10 */ 802, 938, 969, 969, 969, 876, 969, 969, 969, 964, + /* 20 */ 969, 778, 808, 969, 969, 969, 969, 969, 969, 969, + /* 30 */ 969, 937, 939, 816, 815, 918, 789, 813, 806, 810, + /* 40 */ 879, 872, 873, 871, 875, 880, 969, 809, 841, 856, + /* 50 */ 840, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 60 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 70 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 80 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 90 */ 969, 969, 969, 969, 850, 855, 862, 854, 851, 843, + /* 100 */ 842, 844, 845, 969, 969, 673, 739, 969, 969, 846, + /* 110 */ 969, 685, 847, 859, 858, 857, 680, 969, 969, 969, + /* 120 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 130 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 140 */ 969, 969, 969, 969, 647, 964, 969, 969, 964, 964, + /* 150 */ 964, 964, 964, 964, 956, 778, 768, 969, 969, 969, + /* 160 */ 969, 969, 969, 969, 969, 969, 969, 944, 942, 969, + /* 170 */ 891, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 180 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 190 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 200 */ 969, 969, 969, 969, 653, 969, 911, 774, 774, 774, + /* 210 */ 776, 754, 766, 655, 812, 791, 791, 923, 812, 923, + /* 220 */ 710, 733, 707, 802, 791, 874, 802, 802, 775, 766, + /* 230 */ 969, 949, 782, 782, 941, 941, 782, 821, 743, 812, + /* 240 */ 750, 750, 750, 750, 782, 670, 812, 821, 743, 743, + /* 250 */ 812, 782, 670, 917, 915, 782, 782, 670, 782, 670, + /* 260 */ 782, 670, 884, 741, 741, 741, 725, 884, 741, 710, + /* 270 */ 741, 725, 741, 741, 795, 790, 795, 790, 795, 790, + /* 280 */ 782, 782, 969, 884, 888, 888, 884, 807, 796, 805, + /* 290 */ 803, 812, 676, 728, 663, 663, 652, 652, 652, 652, + /* 300 */ 961, 961, 956, 712, 712, 695, 969, 969, 969, 969, + /* 310 */ 969, 969, 687, 969, 893, 969, 969, 969, 969, 969, + /* 320 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 330 */ 969, 828, 969, 648, 951, 969, 969, 948, 969, 969, + /* 340 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 350 */ 969, 969, 969, 969, 969, 969, 921, 969, 969, 969, + /* 360 */ 969, 969, 969, 914, 913, 969, 969, 969, 969, 969, + /* 370 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 380 */ 969, 969, 969, 969, 969, 969, 969, 757, 969, 969, + /* 390 */ 969, 761, 969, 969, 969, 969, 969, 969, 804, 969, + /* 400 */ 797, 969, 877, 969, 969, 969, 969, 969, 969, 969, + /* 410 */ 969, 969, 969, 966, 969, 969, 969, 965, 969, 969, + /* 420 */ 969, 969, 969, 830, 969, 829, 833, 969, 661, 969, + /* 430 */ 644, 649, 960, 963, 962, 959, 958, 957, 952, 950, + /* 440 */ 947, 946, 945, 943, 940, 936, 897, 895, 902, 901, + /* 450 */ 900, 899, 898, 896, 894, 892, 818, 817, 814, 811, + /* 460 */ 753, 935, 890, 752, 749, 748, 669, 953, 920, 929, + /* 470 */ 928, 927, 822, 926, 925, 924, 922, 919, 906, 820, + /* 480 */ 819, 744, 882, 881, 672, 910, 909, 908, 912, 916, + /* 490 */ 907, 784, 751, 671, 668, 675, 679, 731, 732, 740, + /* 500 */ 738, 737, 736, 735, 734, 730, 681, 686, 724, 709, + /* 510 */ 708, 717, 716, 722, 721, 720, 719, 718, 715, 714, + /* 520 */ 713, 706, 705, 711, 704, 727, 726, 723, 703, 747, + /* 530 */ 746, 745, 742, 702, 701, 700, 833, 699, 698, 838, + /* 540 */ 837, 866, 826, 755, 759, 758, 762, 763, 771, 770, + /* 550 */ 769, 780, 781, 793, 792, 824, 823, 794, 779, 773, + /* 560 */ 772, 788, 787, 786, 785, 777, 767, 799, 798, 868, + /* 570 */ 783, 867, 865, 934, 933, 932, 931, 930, 870, 967, + /* 580 */ 968, 887, 889, 886, 801, 800, 885, 869, 839, 836, + /* 590 */ 690, 691, 905, 904, 903, 693, 692, 689, 688, 863, + /* 600 */ 860, 852, 864, 861, 853, 849, 848, 834, 832, 831, + /* 610 */ 827, 835, 760, 756, 825, 765, 764, 697, 696, 694, + /* 620 */ 678, 677, 674, 667, 665, 664, 666, 662, 660, 659, + /* 630 */ 658, 657, 656, 684, 683, 682, 654, 651, 650, 646, + /* 640 */ 645, 643, }; /* The next table maps tokens into fallback tokens. If a construct @@ -104077,71 +120983,74 @@ static const YYACTIONTYPE yy_default[] = { static const YYCODETYPE yyFallback[] = { 0, /* $ => nothing */ 0, /* SEMI => nothing */ - 26, /* EXPLAIN => ID */ - 26, /* QUERY => ID */ - 26, /* PLAN => ID */ - 26, /* BEGIN => ID */ + 27, /* EXPLAIN => ID */ + 27, /* QUERY => ID */ + 27, /* PLAN => ID */ + 27, /* BEGIN => ID */ 0, /* TRANSACTION => nothing */ - 26, /* DEFERRED => ID */ - 26, /* IMMEDIATE => ID */ - 26, /* EXCLUSIVE => ID */ + 27, /* DEFERRED => ID */ + 27, /* IMMEDIATE => ID */ + 27, /* EXCLUSIVE => ID */ 0, /* COMMIT => nothing */ - 26, /* END => ID */ - 26, /* ROLLBACK => ID */ - 26, /* SAVEPOINT => ID */ - 26, /* RELEASE => ID */ + 27, /* END => ID */ + 27, /* ROLLBACK => ID */ + 27, /* SAVEPOINT => ID */ + 27, /* RELEASE => ID */ 0, /* TO => nothing */ 0, /* TABLE => nothing */ 0, /* CREATE => nothing */ - 26, /* IF => ID */ + 27, /* IF => ID */ 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ - 26, /* TEMP => ID */ + 27, /* TEMP => ID */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* AS => nothing */ + 27, /* WITHOUT => ID */ 0, /* COMMA => nothing */ 0, /* ID => nothing */ 0, /* INDEXED => nothing */ - 26, /* ABORT => ID */ - 26, /* ACTION => ID */ - 26, /* AFTER => ID */ - 26, /* ANALYZE => ID */ - 26, /* ASC => ID */ - 26, /* ATTACH => ID */ - 26, /* BEFORE => ID */ - 26, /* BY => ID */ - 26, /* CASCADE => ID */ - 26, /* CAST => ID */ - 26, /* COLUMNKW => ID */ - 26, /* CONFLICT => ID */ - 26, /* DATABASE => ID */ - 26, /* DESC => ID */ - 26, /* DETACH => ID */ - 26, /* EACH => ID */ - 26, /* FAIL => ID */ - 26, /* FOR => ID */ - 26, /* IGNORE => ID */ - 26, /* INITIALLY => ID */ - 26, /* INSTEAD => ID */ - 26, /* LIKE_KW => ID */ - 26, /* MATCH => ID */ - 26, /* NO => ID */ - 26, /* KEY => ID */ - 26, /* OF => ID */ - 26, /* OFFSET => ID */ - 26, /* PRAGMA => ID */ - 26, /* RAISE => ID */ - 26, /* REPLACE => ID */ - 26, /* RESTRICT => ID */ - 26, /* ROW => ID */ - 26, /* TRIGGER => ID */ - 26, /* VACUUM => ID */ - 26, /* VIEW => ID */ - 26, /* VIRTUAL => ID */ - 26, /* REINDEX => ID */ - 26, /* RENAME => ID */ - 26, /* CTIME_KW => ID */ + 27, /* ABORT => ID */ + 27, /* ACTION => ID */ + 27, /* AFTER => ID */ + 27, /* ANALYZE => ID */ + 27, /* ASC => ID */ + 27, /* ATTACH => ID */ + 27, /* BEFORE => ID */ + 27, /* BY => ID */ + 27, /* CASCADE => ID */ + 27, /* CAST => ID */ + 27, /* COLUMNKW => ID */ + 27, /* CONFLICT => ID */ + 27, /* DATABASE => ID */ + 27, /* DESC => ID */ + 27, /* DETACH => ID */ + 27, /* EACH => ID */ + 27, /* FAIL => ID */ + 27, /* FOR => ID */ + 27, /* IGNORE => ID */ + 27, /* INITIALLY => ID */ + 27, /* INSTEAD => ID */ + 27, /* LIKE_KW => ID */ + 27, /* MATCH => ID */ + 27, /* NO => ID */ + 27, /* KEY => ID */ + 27, /* OF => ID */ + 27, /* OFFSET => ID */ + 27, /* PRAGMA => ID */ + 27, /* RAISE => ID */ + 27, /* RECURSIVE => ID */ + 27, /* REPLACE => ID */ + 27, /* RESTRICT => ID */ + 27, /* ROW => ID */ + 27, /* TRIGGER => ID */ + 27, /* VACUUM => ID */ + 27, /* VIEW => ID */ + 27, /* VIRTUAL => ID */ + 27, /* WITH => ID */ + 27, /* REINDEX => ID */ + 27, /* RENAME => ID */ + 27, /* CTIME_KW => ID */ }; #endif /* YYFALLBACK */ @@ -104185,6 +121094,7 @@ struct yyParser { typedef struct yyParser yyParser; #ifndef NDEBUG +/* #include */ static FILE *yyTraceFILE = 0; static char *yyTracePrompt = 0; #endif /* NDEBUG */ @@ -104225,63 +121135,64 @@ static const char *const yyTokenName[] = { "ROLLBACK", "SAVEPOINT", "RELEASE", "TO", "TABLE", "CREATE", "IF", "NOT", "EXISTS", "TEMP", "LP", "RP", - "AS", "COMMA", "ID", "INDEXED", - "ABORT", "ACTION", "AFTER", "ANALYZE", - "ASC", "ATTACH", "BEFORE", "BY", - "CASCADE", "CAST", "COLUMNKW", "CONFLICT", - "DATABASE", "DESC", "DETACH", "EACH", - "FAIL", "FOR", "IGNORE", "INITIALLY", - "INSTEAD", "LIKE_KW", "MATCH", "NO", - "KEY", "OF", "OFFSET", "PRAGMA", - "RAISE", "REPLACE", "RESTRICT", "ROW", - "TRIGGER", "VACUUM", "VIEW", "VIRTUAL", - "REINDEX", "RENAME", "CTIME_KW", "ANY", - "OR", "AND", "IS", "BETWEEN", - "IN", "ISNULL", "NOTNULL", "NE", - "EQ", "GT", "LE", "LT", - "GE", "ESCAPE", "BITAND", "BITOR", - "LSHIFT", "RSHIFT", "PLUS", "MINUS", - "STAR", "SLASH", "REM", "CONCAT", - "COLLATE", "BITNOT", "STRING", "JOIN_KW", - "CONSTRAINT", "DEFAULT", "NULL", "PRIMARY", - "UNIQUE", "CHECK", "REFERENCES", "AUTOINCR", - "ON", "INSERT", "DELETE", "UPDATE", - "SET", "DEFERRABLE", "FOREIGN", "DROP", - "UNION", "ALL", "EXCEPT", "INTERSECT", - "SELECT", "DISTINCT", "DOT", "FROM", + "AS", "WITHOUT", "COMMA", "ID", + "INDEXED", "ABORT", "ACTION", "AFTER", + "ANALYZE", "ASC", "ATTACH", "BEFORE", + "BY", "CASCADE", "CAST", "COLUMNKW", + "CONFLICT", "DATABASE", "DESC", "DETACH", + "EACH", "FAIL", "FOR", "IGNORE", + "INITIALLY", "INSTEAD", "LIKE_KW", "MATCH", + "NO", "KEY", "OF", "OFFSET", + "PRAGMA", "RAISE", "RECURSIVE", "REPLACE", + "RESTRICT", "ROW", "TRIGGER", "VACUUM", + "VIEW", "VIRTUAL", "WITH", "REINDEX", + "RENAME", "CTIME_KW", "ANY", "OR", + "AND", "IS", "BETWEEN", "IN", + "ISNULL", "NOTNULL", "NE", "EQ", + "GT", "LE", "LT", "GE", + "ESCAPE", "BITAND", "BITOR", "LSHIFT", + "RSHIFT", "PLUS", "MINUS", "STAR", + "SLASH", "REM", "CONCAT", "COLLATE", + "BITNOT", "STRING", "JOIN_KW", "CONSTRAINT", + "DEFAULT", "NULL", "PRIMARY", "UNIQUE", + "CHECK", "REFERENCES", "AUTOINCR", "ON", + "INSERT", "DELETE", "UPDATE", "SET", + "DEFERRABLE", "FOREIGN", "DROP", "UNION", + "ALL", "EXCEPT", "INTERSECT", "SELECT", + "VALUES", "DISTINCT", "DOT", "FROM", "JOIN", "USING", "ORDER", "GROUP", "HAVING", "LIMIT", "WHERE", "INTO", - "VALUES", "INTEGER", "FLOAT", "BLOB", - "REGISTER", "VARIABLE", "CASE", "WHEN", - "THEN", "ELSE", "INDEX", "ALTER", - "ADD", "error", "input", "cmdlist", - "ecmd", "explain", "cmdx", "cmd", - "transtype", "trans_opt", "nm", "savepoint_opt", - "create_table", "create_table_args", "createkw", "temp", - "ifnotexists", "dbnm", "columnlist", "conslist_opt", - "select", "column", "columnid", "type", - "carglist", "id", "ids", "typetoken", - "typename", "signed", "plus_num", "minus_num", - "carg", "ccons", "term", "expr", + "INTEGER", "FLOAT", "BLOB", "VARIABLE", + "CASE", "WHEN", "THEN", "ELSE", + "INDEX", "ALTER", "ADD", "error", + "input", "cmdlist", "ecmd", "explain", + "cmdx", "cmd", "transtype", "trans_opt", + "nm", "savepoint_opt", "create_table", "create_table_args", + "createkw", "temp", "ifnotexists", "dbnm", + "columnlist", "conslist_opt", "table_options", "select", + "column", "columnid", "type", "carglist", + "typetoken", "typename", "signed", "plus_num", + "minus_num", "ccons", "term", "expr", "onconf", "sortorder", "autoinc", "idxlist_opt", "refargs", "defer_subclause", "refarg", "refact", - "init_deferred_pred_opt", "conslist", "tcons", "idxlist", - "defer_subclause_opt", "orconf", "resolvetype", "raisetype", - "ifexists", "fullname", "oneselect", "multiselect_op", - "distinct", "selcollist", "from", "where_opt", - "groupby_opt", "having_opt", "orderby_opt", "limit_opt", - "sclp", "as", "seltablist", "stl_prefix", - "joinop", "indexed_opt", "on_opt", "using_opt", - "joinop2", "inscollist", "sortlist", "sortitem", - "nexprlist", "setlist", "insert_cmd", "inscollist_opt", - "itemlist", "exprlist", "likeop", "between_op", - "in_op", "case_operand", "case_exprlist", "case_else", - "uniqueflag", "collate", "nmnum", "plus_opt", - "number", "trigger_decl", "trigger_cmd_list", "trigger_time", + "init_deferred_pred_opt", "conslist", "tconscomma", "tcons", + "idxlist", "defer_subclause_opt", "orconf", "resolvetype", + "raisetype", "ifexists", "fullname", "selectnowith", + "oneselect", "with", "multiselect_op", "distinct", + "selcollist", "from", "where_opt", "groupby_opt", + "having_opt", "orderby_opt", "limit_opt", "values", + "nexprlist", "exprlist", "sclp", "as", + "seltablist", "stl_prefix", "joinop", "indexed_opt", + "on_opt", "using_opt", "joinop2", "idlist", + "sortlist", "setlist", "insert_cmd", "inscollist_opt", + "likeop", "between_op", "in_op", "case_operand", + "case_exprlist", "case_else", "uniqueflag", "collate", + "nmnum", "trigger_decl", "trigger_cmd_list", "trigger_time", "trigger_event", "foreach_clause", "when_clause", "trigger_cmd", "trnm", "tridxby", "database_kw_opt", "key_opt", "add_column_fullname", "kwcolumn_opt", "create_vtab", "vtabarglist", "vtabarg", "vtabargtoken", "lp", "anylist", + "wqlist", }; #endif /* NDEBUG */ @@ -104321,303 +121232,301 @@ static const char *const yyRuleName[] = { /* 29 */ "ifnotexists ::= IF NOT EXISTS", /* 30 */ "temp ::= TEMP", /* 31 */ "temp ::=", - /* 32 */ "create_table_args ::= LP columnlist conslist_opt RP", + /* 32 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", /* 33 */ "create_table_args ::= AS select", - /* 34 */ "columnlist ::= columnlist COMMA column", - /* 35 */ "columnlist ::= column", - /* 36 */ "column ::= columnid type carglist", - /* 37 */ "columnid ::= nm", - /* 38 */ "id ::= ID", - /* 39 */ "id ::= INDEXED", - /* 40 */ "ids ::= ID|STRING", - /* 41 */ "nm ::= id", - /* 42 */ "nm ::= STRING", - /* 43 */ "nm ::= JOIN_KW", - /* 44 */ "type ::=", - /* 45 */ "type ::= typetoken", - /* 46 */ "typetoken ::= typename", - /* 47 */ "typetoken ::= typename LP signed RP", - /* 48 */ "typetoken ::= typename LP signed COMMA signed RP", - /* 49 */ "typename ::= ids", - /* 50 */ "typename ::= typename ids", - /* 51 */ "signed ::= plus_num", - /* 52 */ "signed ::= minus_num", - /* 53 */ "carglist ::= carglist carg", - /* 54 */ "carglist ::=", - /* 55 */ "carg ::= CONSTRAINT nm ccons", - /* 56 */ "carg ::= ccons", - /* 57 */ "ccons ::= DEFAULT term", - /* 58 */ "ccons ::= DEFAULT LP expr RP", - /* 59 */ "ccons ::= DEFAULT PLUS term", - /* 60 */ "ccons ::= DEFAULT MINUS term", - /* 61 */ "ccons ::= DEFAULT id", - /* 62 */ "ccons ::= NULL onconf", - /* 63 */ "ccons ::= NOT NULL onconf", - /* 64 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 65 */ "ccons ::= UNIQUE onconf", - /* 66 */ "ccons ::= CHECK LP expr RP", - /* 67 */ "ccons ::= REFERENCES nm idxlist_opt refargs", - /* 68 */ "ccons ::= defer_subclause", - /* 69 */ "ccons ::= COLLATE ids", - /* 70 */ "autoinc ::=", - /* 71 */ "autoinc ::= AUTOINCR", - /* 72 */ "refargs ::=", - /* 73 */ "refargs ::= refargs refarg", - /* 74 */ "refarg ::= MATCH nm", - /* 75 */ "refarg ::= ON INSERT refact", - /* 76 */ "refarg ::= ON DELETE refact", - /* 77 */ "refarg ::= ON UPDATE refact", - /* 78 */ "refact ::= SET NULL", - /* 79 */ "refact ::= SET DEFAULT", - /* 80 */ "refact ::= CASCADE", - /* 81 */ "refact ::= RESTRICT", - /* 82 */ "refact ::= NO ACTION", - /* 83 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 84 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 85 */ "init_deferred_pred_opt ::=", - /* 86 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 87 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 88 */ "conslist_opt ::=", - /* 89 */ "conslist_opt ::= COMMA conslist", - /* 90 */ "conslist ::= conslist COMMA tcons", - /* 91 */ "conslist ::= conslist tcons", - /* 92 */ "conslist ::= tcons", - /* 93 */ "tcons ::= CONSTRAINT nm", - /* 94 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", - /* 95 */ "tcons ::= UNIQUE LP idxlist RP onconf", - /* 96 */ "tcons ::= CHECK LP expr RP onconf", - /* 97 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", - /* 98 */ "defer_subclause_opt ::=", - /* 99 */ "defer_subclause_opt ::= defer_subclause", - /* 100 */ "onconf ::=", - /* 101 */ "onconf ::= ON CONFLICT resolvetype", - /* 102 */ "orconf ::=", - /* 103 */ "orconf ::= OR resolvetype", - /* 104 */ "resolvetype ::= raisetype", - /* 105 */ "resolvetype ::= IGNORE", - /* 106 */ "resolvetype ::= REPLACE", - /* 107 */ "cmd ::= DROP TABLE ifexists fullname", - /* 108 */ "ifexists ::= IF EXISTS", - /* 109 */ "ifexists ::=", - /* 110 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", - /* 111 */ "cmd ::= DROP VIEW ifexists fullname", - /* 112 */ "cmd ::= select", - /* 113 */ "select ::= oneselect", - /* 114 */ "select ::= select multiselect_op oneselect", + /* 34 */ "table_options ::=", + /* 35 */ "table_options ::= WITHOUT nm", + /* 36 */ "columnlist ::= columnlist COMMA column", + /* 37 */ "columnlist ::= column", + /* 38 */ "column ::= columnid type carglist", + /* 39 */ "columnid ::= nm", + /* 40 */ "nm ::= ID|INDEXED", + /* 41 */ "nm ::= STRING", + /* 42 */ "nm ::= JOIN_KW", + /* 43 */ "type ::=", + /* 44 */ "type ::= typetoken", + /* 45 */ "typetoken ::= typename", + /* 46 */ "typetoken ::= typename LP signed RP", + /* 47 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 48 */ "typename ::= ID|STRING", + /* 49 */ "typename ::= typename ID|STRING", + /* 50 */ "signed ::= plus_num", + /* 51 */ "signed ::= minus_num", + /* 52 */ "carglist ::= carglist ccons", + /* 53 */ "carglist ::=", + /* 54 */ "ccons ::= CONSTRAINT nm", + /* 55 */ "ccons ::= DEFAULT term", + /* 56 */ "ccons ::= DEFAULT LP expr RP", + /* 57 */ "ccons ::= DEFAULT PLUS term", + /* 58 */ "ccons ::= DEFAULT MINUS term", + /* 59 */ "ccons ::= DEFAULT ID|INDEXED", + /* 60 */ "ccons ::= NULL onconf", + /* 61 */ "ccons ::= NOT NULL onconf", + /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 63 */ "ccons ::= UNIQUE onconf", + /* 64 */ "ccons ::= CHECK LP expr RP", + /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", + /* 66 */ "ccons ::= defer_subclause", + /* 67 */ "ccons ::= COLLATE ID|STRING", + /* 68 */ "autoinc ::=", + /* 69 */ "autoinc ::= AUTOINCR", + /* 70 */ "refargs ::=", + /* 71 */ "refargs ::= refargs refarg", + /* 72 */ "refarg ::= MATCH nm", + /* 73 */ "refarg ::= ON INSERT refact", + /* 74 */ "refarg ::= ON DELETE refact", + /* 75 */ "refarg ::= ON UPDATE refact", + /* 76 */ "refact ::= SET NULL", + /* 77 */ "refact ::= SET DEFAULT", + /* 78 */ "refact ::= CASCADE", + /* 79 */ "refact ::= RESTRICT", + /* 80 */ "refact ::= NO ACTION", + /* 81 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 82 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 83 */ "init_deferred_pred_opt ::=", + /* 84 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 85 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 86 */ "conslist_opt ::=", + /* 87 */ "conslist_opt ::= COMMA conslist", + /* 88 */ "conslist ::= conslist tconscomma tcons", + /* 89 */ "conslist ::= tcons", + /* 90 */ "tconscomma ::= COMMA", + /* 91 */ "tconscomma ::=", + /* 92 */ "tcons ::= CONSTRAINT nm", + /* 93 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", + /* 94 */ "tcons ::= UNIQUE LP idxlist RP onconf", + /* 95 */ "tcons ::= CHECK LP expr RP onconf", + /* 96 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", + /* 97 */ "defer_subclause_opt ::=", + /* 98 */ "defer_subclause_opt ::= defer_subclause", + /* 99 */ "onconf ::=", + /* 100 */ "onconf ::= ON CONFLICT resolvetype", + /* 101 */ "orconf ::=", + /* 102 */ "orconf ::= OR resolvetype", + /* 103 */ "resolvetype ::= raisetype", + /* 104 */ "resolvetype ::= IGNORE", + /* 105 */ "resolvetype ::= REPLACE", + /* 106 */ "cmd ::= DROP TABLE ifexists fullname", + /* 107 */ "ifexists ::= IF EXISTS", + /* 108 */ "ifexists ::=", + /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", + /* 110 */ "cmd ::= DROP VIEW ifexists fullname", + /* 111 */ "cmd ::= select", + /* 112 */ "select ::= with selectnowith", + /* 113 */ "selectnowith ::= oneselect", + /* 114 */ "selectnowith ::= selectnowith multiselect_op oneselect", /* 115 */ "multiselect_op ::= UNION", /* 116 */ "multiselect_op ::= UNION ALL", /* 117 */ "multiselect_op ::= EXCEPT|INTERSECT", /* 118 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 119 */ "distinct ::= DISTINCT", - /* 120 */ "distinct ::= ALL", - /* 121 */ "distinct ::=", - /* 122 */ "sclp ::= selcollist COMMA", - /* 123 */ "sclp ::=", - /* 124 */ "selcollist ::= sclp expr as", - /* 125 */ "selcollist ::= sclp STAR", - /* 126 */ "selcollist ::= sclp nm DOT STAR", - /* 127 */ "as ::= AS nm", - /* 128 */ "as ::= ids", - /* 129 */ "as ::=", - /* 130 */ "from ::=", - /* 131 */ "from ::= FROM seltablist", - /* 132 */ "stl_prefix ::= seltablist joinop", - /* 133 */ "stl_prefix ::=", - /* 134 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 135 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 136 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 137 */ "dbnm ::=", - /* 138 */ "dbnm ::= DOT nm", - /* 139 */ "fullname ::= nm dbnm", - /* 140 */ "joinop ::= COMMA|JOIN", - /* 141 */ "joinop ::= JOIN_KW JOIN", - /* 142 */ "joinop ::= JOIN_KW nm JOIN", - /* 143 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 144 */ "on_opt ::= ON expr", - /* 145 */ "on_opt ::=", - /* 146 */ "indexed_opt ::=", - /* 147 */ "indexed_opt ::= INDEXED BY nm", - /* 148 */ "indexed_opt ::= NOT INDEXED", - /* 149 */ "using_opt ::= USING LP inscollist RP", - /* 150 */ "using_opt ::=", - /* 151 */ "orderby_opt ::=", - /* 152 */ "orderby_opt ::= ORDER BY sortlist", - /* 153 */ "sortlist ::= sortlist COMMA sortitem sortorder", - /* 154 */ "sortlist ::= sortitem sortorder", - /* 155 */ "sortitem ::= expr", - /* 156 */ "sortorder ::= ASC", - /* 157 */ "sortorder ::= DESC", - /* 158 */ "sortorder ::=", - /* 159 */ "groupby_opt ::=", - /* 160 */ "groupby_opt ::= GROUP BY nexprlist", - /* 161 */ "having_opt ::=", - /* 162 */ "having_opt ::= HAVING expr", - /* 163 */ "limit_opt ::=", - /* 164 */ "limit_opt ::= LIMIT expr", - /* 165 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 166 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 167 */ "cmd ::= DELETE FROM fullname indexed_opt where_opt", - /* 168 */ "where_opt ::=", - /* 169 */ "where_opt ::= WHERE expr", - /* 170 */ "cmd ::= UPDATE orconf fullname indexed_opt SET setlist where_opt", - /* 171 */ "setlist ::= setlist COMMA nm EQ expr", - /* 172 */ "setlist ::= nm EQ expr", - /* 173 */ "cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP", - /* 174 */ "cmd ::= insert_cmd INTO fullname inscollist_opt select", - /* 175 */ "cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", - /* 176 */ "insert_cmd ::= INSERT orconf", - /* 177 */ "insert_cmd ::= REPLACE", - /* 178 */ "itemlist ::= itemlist COMMA expr", - /* 179 */ "itemlist ::= expr", - /* 180 */ "inscollist_opt ::=", - /* 181 */ "inscollist_opt ::= LP inscollist RP", - /* 182 */ "inscollist ::= inscollist COMMA nm", - /* 183 */ "inscollist ::= nm", - /* 184 */ "expr ::= term", - /* 185 */ "expr ::= LP expr RP", - /* 186 */ "term ::= NULL", - /* 187 */ "expr ::= id", - /* 188 */ "expr ::= JOIN_KW", - /* 189 */ "expr ::= nm DOT nm", - /* 190 */ "expr ::= nm DOT nm DOT nm", - /* 191 */ "term ::= INTEGER|FLOAT|BLOB", - /* 192 */ "term ::= STRING", - /* 193 */ "expr ::= REGISTER", - /* 194 */ "expr ::= VARIABLE", - /* 195 */ "expr ::= expr COLLATE ids", - /* 196 */ "expr ::= CAST LP expr AS typetoken RP", - /* 197 */ "expr ::= ID LP distinct exprlist RP", - /* 198 */ "expr ::= ID LP STAR RP", - /* 199 */ "term ::= CTIME_KW", - /* 200 */ "expr ::= expr AND expr", - /* 201 */ "expr ::= expr OR expr", - /* 202 */ "expr ::= expr LT|GT|GE|LE expr", - /* 203 */ "expr ::= expr EQ|NE expr", - /* 204 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 205 */ "expr ::= expr PLUS|MINUS expr", - /* 206 */ "expr ::= expr STAR|SLASH|REM expr", - /* 207 */ "expr ::= expr CONCAT expr", - /* 208 */ "likeop ::= LIKE_KW", - /* 209 */ "likeop ::= NOT LIKE_KW", - /* 210 */ "likeop ::= MATCH", - /* 211 */ "likeop ::= NOT MATCH", - /* 212 */ "expr ::= expr likeop expr", - /* 213 */ "expr ::= expr likeop expr ESCAPE expr", - /* 214 */ "expr ::= expr ISNULL|NOTNULL", - /* 215 */ "expr ::= expr NOT NULL", - /* 216 */ "expr ::= expr IS expr", - /* 217 */ "expr ::= expr IS NOT expr", - /* 218 */ "expr ::= NOT expr", - /* 219 */ "expr ::= BITNOT expr", - /* 220 */ "expr ::= MINUS expr", - /* 221 */ "expr ::= PLUS expr", - /* 222 */ "between_op ::= BETWEEN", - /* 223 */ "between_op ::= NOT BETWEEN", - /* 224 */ "expr ::= expr between_op expr AND expr", - /* 225 */ "in_op ::= IN", - /* 226 */ "in_op ::= NOT IN", - /* 227 */ "expr ::= expr in_op LP exprlist RP", - /* 228 */ "expr ::= LP select RP", - /* 229 */ "expr ::= expr in_op LP select RP", - /* 230 */ "expr ::= expr in_op nm dbnm", - /* 231 */ "expr ::= EXISTS LP select RP", - /* 232 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 233 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 234 */ "case_exprlist ::= WHEN expr THEN expr", - /* 235 */ "case_else ::= ELSE expr", - /* 236 */ "case_else ::=", - /* 237 */ "case_operand ::= expr", - /* 238 */ "case_operand ::=", - /* 239 */ "exprlist ::= nexprlist", - /* 240 */ "exprlist ::=", - /* 241 */ "nexprlist ::= nexprlist COMMA expr", - /* 242 */ "nexprlist ::= expr", - /* 243 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP", - /* 244 */ "uniqueflag ::= UNIQUE", - /* 245 */ "uniqueflag ::=", - /* 246 */ "idxlist_opt ::=", - /* 247 */ "idxlist_opt ::= LP idxlist RP", - /* 248 */ "idxlist ::= idxlist COMMA nm collate sortorder", - /* 249 */ "idxlist ::= nm collate sortorder", - /* 250 */ "collate ::=", - /* 251 */ "collate ::= COLLATE ids", - /* 252 */ "cmd ::= DROP INDEX ifexists fullname", - /* 253 */ "cmd ::= VACUUM", - /* 254 */ "cmd ::= VACUUM nm", - /* 255 */ "cmd ::= PRAGMA nm dbnm", - /* 256 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 257 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 258 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 259 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 260 */ "nmnum ::= plus_num", - /* 261 */ "nmnum ::= nm", - /* 262 */ "nmnum ::= ON", - /* 263 */ "nmnum ::= DELETE", - /* 264 */ "nmnum ::= DEFAULT", - /* 265 */ "plus_num ::= plus_opt number", - /* 266 */ "minus_num ::= MINUS number", - /* 267 */ "number ::= INTEGER|FLOAT", - /* 268 */ "plus_opt ::= PLUS", - /* 269 */ "plus_opt ::=", - /* 270 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 271 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 272 */ "trigger_time ::= BEFORE", - /* 273 */ "trigger_time ::= AFTER", - /* 274 */ "trigger_time ::= INSTEAD OF", - /* 275 */ "trigger_time ::=", - /* 276 */ "trigger_event ::= DELETE|INSERT", - /* 277 */ "trigger_event ::= UPDATE", - /* 278 */ "trigger_event ::= UPDATE OF inscollist", - /* 279 */ "foreach_clause ::=", - /* 280 */ "foreach_clause ::= FOR EACH ROW", - /* 281 */ "when_clause ::=", - /* 282 */ "when_clause ::= WHEN expr", - /* 283 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 284 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 285 */ "trnm ::= nm", - /* 286 */ "trnm ::= nm DOT nm", - /* 287 */ "tridxby ::=", - /* 288 */ "tridxby ::= INDEXED BY nm", - /* 289 */ "tridxby ::= NOT INDEXED", - /* 290 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 291 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP", - /* 292 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", - /* 293 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 294 */ "trigger_cmd ::= select", - /* 295 */ "expr ::= RAISE LP IGNORE RP", - /* 296 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 297 */ "raisetype ::= ROLLBACK", - /* 298 */ "raisetype ::= ABORT", - /* 299 */ "raisetype ::= FAIL", - /* 300 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 301 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 302 */ "cmd ::= DETACH database_kw_opt expr", - /* 303 */ "key_opt ::=", - /* 304 */ "key_opt ::= KEY expr", - /* 305 */ "database_kw_opt ::= DATABASE", - /* 306 */ "database_kw_opt ::=", - /* 307 */ "cmd ::= REINDEX", - /* 308 */ "cmd ::= REINDEX nm dbnm", - /* 309 */ "cmd ::= ANALYZE", - /* 310 */ "cmd ::= ANALYZE nm dbnm", - /* 311 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 312 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", - /* 313 */ "add_column_fullname ::= fullname", - /* 314 */ "kwcolumn_opt ::=", - /* 315 */ "kwcolumn_opt ::= COLUMNKW", - /* 316 */ "cmd ::= create_vtab", - /* 317 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 318 */ "create_vtab ::= createkw VIRTUAL TABLE nm dbnm USING nm", - /* 319 */ "vtabarglist ::= vtabarg", - /* 320 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 321 */ "vtabarg ::=", - /* 322 */ "vtabarg ::= vtabarg vtabargtoken", - /* 323 */ "vtabargtoken ::= ANY", - /* 324 */ "vtabargtoken ::= lp anylist RP", - /* 325 */ "lp ::= LP", - /* 326 */ "anylist ::=", - /* 327 */ "anylist ::= anylist LP anylist RP", - /* 328 */ "anylist ::= anylist ANY", + /* 119 */ "oneselect ::= values", + /* 120 */ "values ::= VALUES LP nexprlist RP", + /* 121 */ "values ::= values COMMA LP exprlist RP", + /* 122 */ "distinct ::= DISTINCT", + /* 123 */ "distinct ::= ALL", + /* 124 */ "distinct ::=", + /* 125 */ "sclp ::= selcollist COMMA", + /* 126 */ "sclp ::=", + /* 127 */ "selcollist ::= sclp expr as", + /* 128 */ "selcollist ::= sclp STAR", + /* 129 */ "selcollist ::= sclp nm DOT STAR", + /* 130 */ "as ::= AS nm", + /* 131 */ "as ::= ID|STRING", + /* 132 */ "as ::=", + /* 133 */ "from ::=", + /* 134 */ "from ::= FROM seltablist", + /* 135 */ "stl_prefix ::= seltablist joinop", + /* 136 */ "stl_prefix ::=", + /* 137 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 138 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 139 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 140 */ "dbnm ::=", + /* 141 */ "dbnm ::= DOT nm", + /* 142 */ "fullname ::= nm dbnm", + /* 143 */ "joinop ::= COMMA|JOIN", + /* 144 */ "joinop ::= JOIN_KW JOIN", + /* 145 */ "joinop ::= JOIN_KW nm JOIN", + /* 146 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 147 */ "on_opt ::= ON expr", + /* 148 */ "on_opt ::=", + /* 149 */ "indexed_opt ::=", + /* 150 */ "indexed_opt ::= INDEXED BY nm", + /* 151 */ "indexed_opt ::= NOT INDEXED", + /* 152 */ "using_opt ::= USING LP idlist RP", + /* 153 */ "using_opt ::=", + /* 154 */ "orderby_opt ::=", + /* 155 */ "orderby_opt ::= ORDER BY sortlist", + /* 156 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 157 */ "sortlist ::= expr sortorder", + /* 158 */ "sortorder ::= ASC", + /* 159 */ "sortorder ::= DESC", + /* 160 */ "sortorder ::=", + /* 161 */ "groupby_opt ::=", + /* 162 */ "groupby_opt ::= GROUP BY nexprlist", + /* 163 */ "having_opt ::=", + /* 164 */ "having_opt ::= HAVING expr", + /* 165 */ "limit_opt ::=", + /* 166 */ "limit_opt ::= LIMIT expr", + /* 167 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 168 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 169 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", + /* 170 */ "where_opt ::=", + /* 171 */ "where_opt ::= WHERE expr", + /* 172 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 173 */ "setlist ::= setlist COMMA nm EQ expr", + /* 174 */ "setlist ::= nm EQ expr", + /* 175 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt select", + /* 176 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", + /* 177 */ "insert_cmd ::= INSERT orconf", + /* 178 */ "insert_cmd ::= REPLACE", + /* 179 */ "inscollist_opt ::=", + /* 180 */ "inscollist_opt ::= LP idlist RP", + /* 181 */ "idlist ::= idlist COMMA nm", + /* 182 */ "idlist ::= nm", + /* 183 */ "expr ::= term", + /* 184 */ "expr ::= LP expr RP", + /* 185 */ "term ::= NULL", + /* 186 */ "expr ::= ID|INDEXED", + /* 187 */ "expr ::= JOIN_KW", + /* 188 */ "expr ::= nm DOT nm", + /* 189 */ "expr ::= nm DOT nm DOT nm", + /* 190 */ "term ::= INTEGER|FLOAT|BLOB", + /* 191 */ "term ::= STRING", + /* 192 */ "expr ::= VARIABLE", + /* 193 */ "expr ::= expr COLLATE ID|STRING", + /* 194 */ "expr ::= CAST LP expr AS typetoken RP", + /* 195 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 196 */ "expr ::= ID|INDEXED LP STAR RP", + /* 197 */ "term ::= CTIME_KW", + /* 198 */ "expr ::= expr AND expr", + /* 199 */ "expr ::= expr OR expr", + /* 200 */ "expr ::= expr LT|GT|GE|LE expr", + /* 201 */ "expr ::= expr EQ|NE expr", + /* 202 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 203 */ "expr ::= expr PLUS|MINUS expr", + /* 204 */ "expr ::= expr STAR|SLASH|REM expr", + /* 205 */ "expr ::= expr CONCAT expr", + /* 206 */ "likeop ::= LIKE_KW|MATCH", + /* 207 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 208 */ "expr ::= expr likeop expr", + /* 209 */ "expr ::= expr likeop expr ESCAPE expr", + /* 210 */ "expr ::= expr ISNULL|NOTNULL", + /* 211 */ "expr ::= expr NOT NULL", + /* 212 */ "expr ::= expr IS expr", + /* 213 */ "expr ::= expr IS NOT expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= MINUS expr", + /* 217 */ "expr ::= PLUS expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::= expr", + /* 234 */ "case_operand ::=", + /* 235 */ "exprlist ::= nexprlist", + /* 236 */ "exprlist ::=", + /* 237 */ "nexprlist ::= nexprlist COMMA expr", + /* 238 */ "nexprlist ::= expr", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "idxlist_opt ::=", + /* 243 */ "idxlist_opt ::= LP idxlist RP", + /* 244 */ "idxlist ::= idxlist COMMA nm collate sortorder", + /* 245 */ "idxlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM", + /* 250 */ "cmd ::= VACUUM nm", + /* 251 */ "cmd ::= PRAGMA nm dbnm", + /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 256 */ "nmnum ::= plus_num", + /* 257 */ "nmnum ::= nm", + /* 258 */ "nmnum ::= ON", + /* 259 */ "nmnum ::= DELETE", + /* 260 */ "nmnum ::= DEFAULT", + /* 261 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 262 */ "plus_num ::= INTEGER|FLOAT", + /* 263 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 264 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 265 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 266 */ "trigger_time ::= BEFORE", + /* 267 */ "trigger_time ::= AFTER", + /* 268 */ "trigger_time ::= INSTEAD OF", + /* 269 */ "trigger_time ::=", + /* 270 */ "trigger_event ::= DELETE|INSERT", + /* 271 */ "trigger_event ::= UPDATE", + /* 272 */ "trigger_event ::= UPDATE OF idlist", + /* 273 */ "foreach_clause ::=", + /* 274 */ "foreach_clause ::= FOR EACH ROW", + /* 275 */ "when_clause ::=", + /* 276 */ "when_clause ::= WHEN expr", + /* 277 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 278 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 279 */ "trnm ::= nm", + /* 280 */ "trnm ::= nm DOT nm", + /* 281 */ "tridxby ::=", + /* 282 */ "tridxby ::= INDEXED BY nm", + /* 283 */ "tridxby ::= NOT INDEXED", + /* 284 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 285 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", + /* 286 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 287 */ "trigger_cmd ::= select", + /* 288 */ "expr ::= RAISE LP IGNORE RP", + /* 289 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 290 */ "raisetype ::= ROLLBACK", + /* 291 */ "raisetype ::= ABORT", + /* 292 */ "raisetype ::= FAIL", + /* 293 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 294 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 295 */ "cmd ::= DETACH database_kw_opt expr", + /* 296 */ "key_opt ::=", + /* 297 */ "key_opt ::= KEY expr", + /* 298 */ "database_kw_opt ::= DATABASE", + /* 299 */ "database_kw_opt ::=", + /* 300 */ "cmd ::= REINDEX", + /* 301 */ "cmd ::= REINDEX nm dbnm", + /* 302 */ "cmd ::= ANALYZE", + /* 303 */ "cmd ::= ANALYZE nm dbnm", + /* 304 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 305 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", + /* 306 */ "add_column_fullname ::= fullname", + /* 307 */ "kwcolumn_opt ::=", + /* 308 */ "kwcolumn_opt ::= COLUMNKW", + /* 309 */ "cmd ::= create_vtab", + /* 310 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 311 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 312 */ "vtabarglist ::= vtabarg", + /* 313 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 314 */ "vtabarg ::=", + /* 315 */ "vtabarg ::= vtabarg vtabargtoken", + /* 316 */ "vtabargtoken ::= ANY", + /* 317 */ "vtabargtoken ::= lp anylist RP", + /* 318 */ "lp ::= LP", + /* 319 */ "anylist ::=", + /* 320 */ "anylist ::= anylist LP anylist RP", + /* 321 */ "anylist ::= anylist ANY", + /* 322 */ "with ::=", + /* 323 */ "with ::= WITH wqlist", + /* 324 */ "with ::= WITH RECURSIVE wqlist", + /* 325 */ "wqlist ::= nm idxlist_opt AS LP select RP", + /* 326 */ "wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP", }; #endif /* NDEBUG */ @@ -104657,9 +121566,9 @@ static void yyGrowStack(yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to sqlite3Parser and sqlite3ParserFree. */ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(size_t)){ +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(u64)){ yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + pParser = (yyParser*)(*mallocProc)( (u64)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH @@ -104696,70 +121605,76 @@ static void yy_destructor( ** which appear on the RHS of the rule, but which are not used ** inside the C code. */ - case 160: /* select */ - case 194: /* oneselect */ + case 163: /* select */ + case 195: /* selectnowith */ + case 196: /* oneselect */ + case 207: /* values */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy387)); +sqlite3SelectDelete(pParse->db, (yypminor->yy3)); } break; case 174: /* term */ case 175: /* expr */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy118).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy346).pExpr); } break; case 179: /* idxlist_opt */ - case 187: /* idxlist */ - case 197: /* selcollist */ - case 200: /* groupby_opt */ - case 202: /* orderby_opt */ - case 204: /* sclp */ - case 214: /* sortlist */ - case 216: /* nexprlist */ - case 217: /* setlist */ - case 220: /* itemlist */ - case 221: /* exprlist */ - case 226: /* case_exprlist */ + case 188: /* idxlist */ + case 200: /* selcollist */ + case 203: /* groupby_opt */ + case 205: /* orderby_opt */ + case 208: /* nexprlist */ + case 209: /* exprlist */ + case 210: /* sclp */ + case 220: /* sortlist */ + case 221: /* setlist */ + case 228: /* case_exprlist */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy322)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; - case 193: /* fullname */ - case 198: /* from */ - case 206: /* seltablist */ - case 207: /* stl_prefix */ + case 194: /* fullname */ + case 201: /* from */ + case 212: /* seltablist */ + case 213: /* stl_prefix */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy259)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy65)); } break; - case 199: /* where_opt */ - case 201: /* having_opt */ - case 210: /* on_opt */ - case 215: /* sortitem */ - case 225: /* case_operand */ - case 227: /* case_else */ + case 197: /* with */ + case 252: /* wqlist */ +{ +sqlite3WithDelete(pParse->db, (yypminor->yy59)); +} + break; + case 202: /* where_opt */ + case 204: /* having_opt */ + case 216: /* on_opt */ + case 227: /* case_operand */ + case 229: /* case_else */ case 238: /* when_clause */ case 243: /* key_opt */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy314)); +sqlite3ExprDelete(pParse->db, (yypminor->yy132)); } break; - case 211: /* using_opt */ - case 213: /* inscollist */ - case 219: /* inscollist_opt */ + case 217: /* using_opt */ + case 219: /* idlist */ + case 223: /* inscollist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy384)); +sqlite3IdListDelete(pParse->db, (yypminor->yy408)); } break; case 234: /* trigger_cmd_list */ case 239: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy203)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy473)); } break; case 236: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy90).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy378).b); } break; default: break; /* If no destructor action specified: do nothing */ @@ -104948,7 +121863,6 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ sqlite3ErrorMsg(pParse, "parser stack overflow"); - pParse->parseError = 1; sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ } @@ -105005,63 +121919,61 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ unsigned char nrhs; /* Number of right-hand side symbols in the rule */ } yyRuleInfo[] = { - { 142, 1 }, - { 143, 2 }, - { 143, 1 }, { 144, 1 }, - { 144, 3 }, - { 145, 0 }, + { 145, 2 }, { 145, 1 }, - { 145, 3 }, { 146, 1 }, + { 146, 3 }, + { 147, 0 }, + { 147, 1 }, { 147, 3 }, - { 149, 0 }, - { 149, 1 }, - { 149, 2 }, - { 148, 0 }, { 148, 1 }, - { 148, 1 }, - { 148, 1 }, - { 147, 2 }, - { 147, 2 }, - { 147, 2 }, - { 151, 1 }, + { 149, 3 }, { 151, 0 }, - { 147, 2 }, - { 147, 3 }, - { 147, 5 }, - { 147, 2 }, - { 152, 6 }, - { 154, 1 }, - { 156, 0 }, - { 156, 3 }, - { 155, 1 }, - { 155, 0 }, - { 153, 4 }, - { 153, 2 }, + { 151, 1 }, + { 151, 2 }, + { 150, 0 }, + { 150, 1 }, + { 150, 1 }, + { 150, 1 }, + { 149, 2 }, + { 149, 2 }, + { 149, 2 }, + { 153, 1 }, + { 153, 0 }, + { 149, 2 }, + { 149, 3 }, + { 149, 5 }, + { 149, 2 }, + { 154, 6 }, + { 156, 1 }, + { 158, 0 }, { 158, 3 }, - { 158, 1 }, - { 161, 3 }, - { 162, 1 }, - { 165, 1 }, + { 157, 1 }, + { 157, 0 }, + { 155, 5 }, + { 155, 2 }, + { 162, 0 }, + { 162, 2 }, + { 160, 3 }, + { 160, 1 }, + { 164, 3 }, { 165, 1 }, + { 152, 1 }, + { 152, 1 }, + { 152, 1 }, + { 166, 0 }, { 166, 1 }, - { 150, 1 }, - { 150, 1 }, - { 150, 1 }, - { 163, 0 }, - { 163, 1 }, - { 167, 1 }, - { 167, 4 }, - { 167, 6 }, { 168, 1 }, - { 168, 2 }, + { 168, 4 }, + { 168, 6 }, { 169, 1 }, - { 169, 1 }, - { 164, 2 }, - { 164, 0 }, - { 172, 3 }, - { 172, 1 }, + { 169, 2 }, + { 170, 1 }, + { 170, 1 }, + { 167, 2 }, + { 167, 0 }, + { 173, 2 }, { 173, 2 }, { 173, 4 }, { 173, 3 }, @@ -105093,102 +122005,103 @@ static const struct { { 184, 0 }, { 184, 2 }, { 184, 2 }, - { 159, 0 }, - { 159, 2 }, + { 161, 0 }, + { 161, 2 }, { 185, 3 }, - { 185, 2 }, { 185, 1 }, - { 186, 2 }, - { 186, 7 }, - { 186, 5 }, - { 186, 5 }, - { 186, 10 }, - { 188, 0 }, - { 188, 1 }, + { 186, 1 }, + { 186, 0 }, + { 187, 2 }, + { 187, 7 }, + { 187, 5 }, + { 187, 5 }, + { 187, 10 }, + { 189, 0 }, + { 189, 1 }, { 176, 0 }, { 176, 3 }, - { 189, 0 }, - { 189, 2 }, - { 190, 1 }, - { 190, 1 }, - { 190, 1 }, - { 147, 4 }, - { 192, 2 }, - { 192, 0 }, - { 147, 8 }, - { 147, 4 }, - { 147, 1 }, - { 160, 1 }, - { 160, 3 }, - { 195, 1 }, - { 195, 2 }, - { 195, 1 }, - { 194, 9 }, - { 196, 1 }, - { 196, 1 }, - { 196, 0 }, - { 204, 2 }, - { 204, 0 }, - { 197, 3 }, - { 197, 2 }, - { 197, 4 }, - { 205, 2 }, - { 205, 1 }, - { 205, 0 }, - { 198, 0 }, - { 198, 2 }, - { 207, 2 }, - { 207, 0 }, - { 206, 7 }, - { 206, 7 }, - { 206, 7 }, - { 157, 0 }, - { 157, 2 }, + { 190, 0 }, + { 190, 2 }, + { 191, 1 }, + { 191, 1 }, + { 191, 1 }, + { 149, 4 }, { 193, 2 }, - { 208, 1 }, - { 208, 2 }, - { 208, 3 }, - { 208, 4 }, + { 193, 0 }, + { 149, 8 }, + { 149, 4 }, + { 149, 1 }, + { 163, 2 }, + { 195, 1 }, + { 195, 3 }, + { 198, 1 }, + { 198, 2 }, + { 198, 1 }, + { 196, 9 }, + { 196, 1 }, + { 207, 4 }, + { 207, 5 }, + { 199, 1 }, + { 199, 1 }, + { 199, 0 }, { 210, 2 }, { 210, 0 }, - { 209, 0 }, - { 209, 3 }, - { 209, 2 }, - { 211, 4 }, + { 200, 3 }, + { 200, 2 }, + { 200, 4 }, + { 211, 2 }, + { 211, 1 }, { 211, 0 }, - { 202, 0 }, - { 202, 3 }, - { 214, 4 }, + { 201, 0 }, + { 201, 2 }, + { 213, 2 }, + { 213, 0 }, + { 212, 7 }, + { 212, 7 }, + { 212, 7 }, + { 159, 0 }, + { 159, 2 }, + { 194, 2 }, + { 214, 1 }, { 214, 2 }, - { 215, 1 }, + { 214, 3 }, + { 214, 4 }, + { 216, 2 }, + { 216, 0 }, + { 215, 0 }, + { 215, 3 }, + { 215, 2 }, + { 217, 4 }, + { 217, 0 }, + { 205, 0 }, + { 205, 3 }, + { 220, 4 }, + { 220, 2 }, { 177, 1 }, { 177, 1 }, { 177, 0 }, - { 200, 0 }, - { 200, 3 }, - { 201, 0 }, - { 201, 2 }, { 203, 0 }, - { 203, 2 }, - { 203, 4 }, - { 203, 4 }, - { 147, 5 }, - { 199, 0 }, - { 199, 2 }, - { 147, 7 }, - { 217, 5 }, - { 217, 3 }, - { 147, 8 }, - { 147, 5 }, - { 147, 6 }, - { 218, 2 }, - { 218, 1 }, - { 220, 3 }, - { 220, 1 }, - { 219, 0 }, + { 203, 3 }, + { 204, 0 }, + { 204, 2 }, + { 206, 0 }, + { 206, 2 }, + { 206, 4 }, + { 206, 4 }, + { 149, 6 }, + { 202, 0 }, + { 202, 2 }, + { 149, 8 }, + { 221, 5 }, + { 221, 3 }, + { 149, 6 }, + { 149, 7 }, + { 222, 2 }, + { 222, 1 }, + { 223, 0 }, + { 223, 3 }, { 219, 3 }, - { 213, 3 }, - { 213, 1 }, + { 219, 1 }, { 175, 1 }, { 175, 3 }, { 174, 1 }, @@ -105199,7 +122112,6 @@ static const struct { { 174, 1 }, { 174, 1 }, { 175, 1 }, - { 175, 1 }, { 175, 3 }, { 175, 6 }, { 175, 5 }, @@ -105213,69 +122125,65 @@ static const struct { { 175, 3 }, { 175, 3 }, { 175, 3 }, - { 222, 1 }, - { 222, 2 }, - { 222, 1 }, - { 222, 2 }, - { 175, 3 }, - { 175, 5 }, - { 175, 2 }, - { 175, 3 }, - { 175, 3 }, - { 175, 4 }, - { 175, 2 }, - { 175, 2 }, - { 175, 2 }, - { 175, 2 }, - { 223, 1 }, - { 223, 2 }, - { 175, 5 }, { 224, 1 }, { 224, 2 }, + { 175, 3 }, + { 175, 5 }, + { 175, 2 }, + { 175, 3 }, + { 175, 3 }, + { 175, 4 }, + { 175, 2 }, + { 175, 2 }, + { 175, 2 }, + { 175, 2 }, + { 225, 1 }, + { 225, 2 }, + { 175, 5 }, + { 226, 1 }, + { 226, 2 }, { 175, 5 }, { 175, 3 }, { 175, 5 }, { 175, 4 }, { 175, 4 }, { 175, 5 }, - { 226, 5 }, - { 226, 4 }, - { 227, 2 }, + { 228, 5 }, + { 228, 4 }, + { 229, 2 }, + { 229, 0 }, + { 227, 1 }, { 227, 0 }, - { 225, 1 }, - { 225, 0 }, - { 221, 1 }, - { 221, 0 }, - { 216, 3 }, - { 216, 1 }, - { 147, 11 }, - { 228, 1 }, - { 228, 0 }, + { 209, 1 }, + { 209, 0 }, + { 208, 3 }, + { 208, 1 }, + { 149, 12 }, + { 230, 1 }, + { 230, 0 }, { 179, 0 }, { 179, 3 }, - { 187, 5 }, - { 187, 3 }, - { 229, 0 }, - { 229, 2 }, - { 147, 4 }, - { 147, 1 }, - { 147, 2 }, - { 147, 3 }, - { 147, 5 }, - { 147, 6 }, - { 147, 5 }, - { 147, 6 }, - { 230, 1 }, - { 230, 1 }, - { 230, 1 }, - { 230, 1 }, - { 230, 1 }, - { 170, 2 }, - { 171, 2 }, - { 232, 1 }, - { 231, 1 }, + { 188, 5 }, + { 188, 3 }, { 231, 0 }, - { 147, 5 }, + { 231, 2 }, + { 149, 4 }, + { 149, 1 }, + { 149, 2 }, + { 149, 3 }, + { 149, 5 }, + { 149, 6 }, + { 149, 5 }, + { 149, 6 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 171, 2 }, + { 171, 1 }, + { 172, 2 }, + { 149, 5 }, { 233, 11 }, { 235, 1 }, { 235, 1 }, @@ -105296,34 +122204,33 @@ static const struct { { 241, 3 }, { 241, 2 }, { 239, 7 }, - { 239, 8 }, { 239, 5 }, { 239, 5 }, { 239, 1 }, { 175, 4 }, { 175, 6 }, - { 191, 1 }, - { 191, 1 }, - { 191, 1 }, - { 147, 4 }, - { 147, 6 }, - { 147, 3 }, + { 192, 1 }, + { 192, 1 }, + { 192, 1 }, + { 149, 4 }, + { 149, 6 }, + { 149, 3 }, { 243, 0 }, { 243, 2 }, { 242, 1 }, { 242, 0 }, - { 147, 1 }, - { 147, 3 }, - { 147, 1 }, - { 147, 3 }, - { 147, 6 }, - { 147, 6 }, + { 149, 1 }, + { 149, 3 }, + { 149, 1 }, + { 149, 3 }, + { 149, 6 }, + { 149, 6 }, { 244, 1 }, { 245, 0 }, { 245, 1 }, - { 147, 1 }, - { 147, 4 }, - { 246, 7 }, + { 149, 1 }, + { 149, 4 }, + { 246, 8 }, { 247, 1 }, { 247, 3 }, { 248, 0 }, @@ -105334,6 +122241,11 @@ static const struct { { 251, 0 }, { 251, 4 }, { 251, 2 }, + { 197, 0 }, + { 197, 2 }, + { 197, 3 }, + { 252, 6 }, + { 252, 8 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -105401,17 +122313,17 @@ static void yy_reduce( { sqlite3FinishCoding(pParse); } break; case 9: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy4);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy328);} break; case 13: /* transtype ::= */ -{yygotominor.yy4 = TK_DEFERRED;} +{yygotominor.yy328 = TK_DEFERRED;} break; case 14: /* transtype ::= DEFERRED */ case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); -{yygotominor.yy4 = yymsp[0].major;} +{yygotominor.yy328 = yymsp[0].major;} break; case 17: /* cmd ::= COMMIT trans_opt */ case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); @@ -105437,7 +122349,7 @@ static void yy_reduce( break; case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy4,0,0,yymsp[-2].minor.yy4); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy328,0,0,yymsp[-2].minor.yy328); } break; case 27: /* createkw ::= CREATE */ @@ -105448,651 +122360,746 @@ static void yy_reduce( break; case 28: /* ifnotexists ::= */ case 31: /* temp ::= */ yytestcase(yyruleno==31); - case 70: /* autoinc ::= */ yytestcase(yyruleno==70); - case 83: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==83); - case 85: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==85); - case 87: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==87); - case 98: /* defer_subclause_opt ::= */ yytestcase(yyruleno==98); - case 109: /* ifexists ::= */ yytestcase(yyruleno==109); - case 120: /* distinct ::= ALL */ yytestcase(yyruleno==120); - case 121: /* distinct ::= */ yytestcase(yyruleno==121); - case 222: /* between_op ::= BETWEEN */ yytestcase(yyruleno==222); - case 225: /* in_op ::= IN */ yytestcase(yyruleno==225); -{yygotominor.yy4 = 0;} + case 68: /* autoinc ::= */ yytestcase(yyruleno==68); + case 81: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==81); + case 83: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==83); + case 85: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==85); + case 97: /* defer_subclause_opt ::= */ yytestcase(yyruleno==97); + case 108: /* ifexists ::= */ yytestcase(yyruleno==108); + case 218: /* between_op ::= BETWEEN */ yytestcase(yyruleno==218); + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yygotominor.yy328 = 0;} break; case 29: /* ifnotexists ::= IF NOT EXISTS */ case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); - case 71: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==71); - case 86: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==86); - case 108: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==108); - case 119: /* distinct ::= DISTINCT */ yytestcase(yyruleno==119); - case 223: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==223); - case 226: /* in_op ::= NOT IN */ yytestcase(yyruleno==226); -{yygotominor.yy4 = 1;} + case 69: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==69); + case 84: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==84); + case 107: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==107); + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); +{yygotominor.yy328 = 1;} break; - case 32: /* create_table_args ::= LP columnlist conslist_opt RP */ + case 32: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ { - sqlite3EndTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy186,0); } break; case 33: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,yymsp[0].minor.yy387); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy387); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy3); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); } break; - case 36: /* column ::= columnid type carglist */ + case 34: /* table_options ::= */ +{yygotominor.yy186 = 0;} + break; + case 35: /* table_options ::= WITHOUT nm */ +{ + if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ + yygotominor.yy186 = TF_WithoutRowid; + }else{ + yygotominor.yy186 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + } +} + break; + case 38: /* column ::= columnid type carglist */ { yygotominor.yy0.z = yymsp[-2].minor.yy0.z; yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; } break; - case 37: /* columnid ::= nm */ + case 39: /* columnid ::= nm */ { sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); yygotominor.yy0 = yymsp[0].minor.yy0; + pParse->constraintName.n = 0; } break; - case 38: /* id ::= ID */ - case 39: /* id ::= INDEXED */ yytestcase(yyruleno==39); - case 40: /* ids ::= ID|STRING */ yytestcase(yyruleno==40); - case 41: /* nm ::= id */ yytestcase(yyruleno==41); - case 42: /* nm ::= STRING */ yytestcase(yyruleno==42); - case 43: /* nm ::= JOIN_KW */ yytestcase(yyruleno==43); - case 46: /* typetoken ::= typename */ yytestcase(yyruleno==46); - case 49: /* typename ::= ids */ yytestcase(yyruleno==49); - case 127: /* as ::= AS nm */ yytestcase(yyruleno==127); - case 128: /* as ::= ids */ yytestcase(yyruleno==128); - case 138: /* dbnm ::= DOT nm */ yytestcase(yyruleno==138); - case 147: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==147); - case 251: /* collate ::= COLLATE ids */ yytestcase(yyruleno==251); - case 260: /* nmnum ::= plus_num */ yytestcase(yyruleno==260); - case 261: /* nmnum ::= nm */ yytestcase(yyruleno==261); - case 262: /* nmnum ::= ON */ yytestcase(yyruleno==262); - case 263: /* nmnum ::= DELETE */ yytestcase(yyruleno==263); - case 264: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==264); - case 265: /* plus_num ::= plus_opt number */ yytestcase(yyruleno==265); - case 266: /* minus_num ::= MINUS number */ yytestcase(yyruleno==266); - case 267: /* number ::= INTEGER|FLOAT */ yytestcase(yyruleno==267); - case 285: /* trnm ::= nm */ yytestcase(yyruleno==285); + case 40: /* nm ::= ID|INDEXED */ + case 41: /* nm ::= STRING */ yytestcase(yyruleno==41); + case 42: /* nm ::= JOIN_KW */ yytestcase(yyruleno==42); + case 45: /* typetoken ::= typename */ yytestcase(yyruleno==45); + case 48: /* typename ::= ID|STRING */ yytestcase(yyruleno==48); + case 130: /* as ::= AS nm */ yytestcase(yyruleno==130); + case 131: /* as ::= ID|STRING */ yytestcase(yyruleno==131); + case 141: /* dbnm ::= DOT nm */ yytestcase(yyruleno==141); + case 150: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==150); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); + case 256: /* nmnum ::= plus_num */ yytestcase(yyruleno==256); + case 257: /* nmnum ::= nm */ yytestcase(yyruleno==257); + case 258: /* nmnum ::= ON */ yytestcase(yyruleno==258); + case 259: /* nmnum ::= DELETE */ yytestcase(yyruleno==259); + case 260: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==260); + case 261: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==261); + case 262: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==262); + case 263: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==263); + case 279: /* trnm ::= nm */ yytestcase(yyruleno==279); {yygotominor.yy0 = yymsp[0].minor.yy0;} break; - case 45: /* type ::= typetoken */ + case 44: /* type ::= typetoken */ {sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} break; - case 47: /* typetoken ::= typename LP signed RP */ + case 46: /* typetoken ::= typename LP signed RP */ { yygotominor.yy0.z = yymsp[-3].minor.yy0.z; yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); } break; - case 48: /* typetoken ::= typename LP signed COMMA signed RP */ + case 47: /* typetoken ::= typename LP signed COMMA signed RP */ { yygotominor.yy0.z = yymsp[-5].minor.yy0.z; yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); } break; - case 50: /* typename ::= typename ids */ + case 49: /* typename ::= typename ID|STRING */ {yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} break; - case 57: /* ccons ::= DEFAULT term */ - case 59: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==59); -{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy118);} + case 54: /* ccons ::= CONSTRAINT nm */ + case 92: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==92); +{pParse->constraintName = yymsp[0].minor.yy0;} break; - case 58: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy118);} + case 55: /* ccons ::= DEFAULT term */ + case 57: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==57); +{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy346);} break; - case 60: /* ccons ::= DEFAULT MINUS term */ + case 56: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy346);} + break; + case 58: /* ccons ::= DEFAULT MINUS term */ { ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy118.pExpr, 0, 0); + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy346.pExpr, 0, 0); v.zStart = yymsp[-1].minor.yy0.z; - v.zEnd = yymsp[0].minor.yy118.zEnd; + v.zEnd = yymsp[0].minor.yy346.zEnd; sqlite3AddDefaultValue(pParse,&v); } break; - case 61: /* ccons ::= DEFAULT id */ + case 59: /* ccons ::= DEFAULT ID|INDEXED */ { ExprSpan v; spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); sqlite3AddDefaultValue(pParse,&v); } break; - case 63: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy4);} + case 61: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy328);} break; - case 64: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy4,yymsp[0].minor.yy4,yymsp[-2].minor.yy4);} + case 62: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy328,yymsp[0].minor.yy328,yymsp[-2].minor.yy328);} break; - case 65: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy4,0,0,0,0);} + case 63: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy328,0,0,0,0);} break; - case 66: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy118.pExpr);} + case 64: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy346.pExpr);} break; - case 67: /* ccons ::= REFERENCES nm idxlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy322,yymsp[0].minor.yy4);} + case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy328);} break; - case 68: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy4);} + case 66: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy328);} break; - case 69: /* ccons ::= COLLATE ids */ + case 67: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; - case 72: /* refargs ::= */ -{ yygotominor.yy4 = OE_None*0x0101; /* EV: R-19803-45884 */} + case 70: /* refargs ::= */ +{ yygotominor.yy328 = OE_None*0x0101; /* EV: R-19803-45884 */} break; - case 73: /* refargs ::= refargs refarg */ -{ yygotominor.yy4 = (yymsp[-1].minor.yy4 & ~yymsp[0].minor.yy215.mask) | yymsp[0].minor.yy215.value; } + case 71: /* refargs ::= refargs refarg */ +{ yygotominor.yy328 = (yymsp[-1].minor.yy328 & ~yymsp[0].minor.yy429.mask) | yymsp[0].minor.yy429.value; } break; - case 74: /* refarg ::= MATCH nm */ - case 75: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==75); -{ yygotominor.yy215.value = 0; yygotominor.yy215.mask = 0x000000; } + case 72: /* refarg ::= MATCH nm */ + case 73: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==73); +{ yygotominor.yy429.value = 0; yygotominor.yy429.mask = 0x000000; } break; - case 76: /* refarg ::= ON DELETE refact */ -{ yygotominor.yy215.value = yymsp[0].minor.yy4; yygotominor.yy215.mask = 0x0000ff; } + case 74: /* refarg ::= ON DELETE refact */ +{ yygotominor.yy429.value = yymsp[0].minor.yy328; yygotominor.yy429.mask = 0x0000ff; } break; - case 77: /* refarg ::= ON UPDATE refact */ -{ yygotominor.yy215.value = yymsp[0].minor.yy4<<8; yygotominor.yy215.mask = 0x00ff00; } + case 75: /* refarg ::= ON UPDATE refact */ +{ yygotominor.yy429.value = yymsp[0].minor.yy328<<8; yygotominor.yy429.mask = 0x00ff00; } break; - case 78: /* refact ::= SET NULL */ -{ yygotominor.yy4 = OE_SetNull; /* EV: R-33326-45252 */} + case 76: /* refact ::= SET NULL */ +{ yygotominor.yy328 = OE_SetNull; /* EV: R-33326-45252 */} break; - case 79: /* refact ::= SET DEFAULT */ -{ yygotominor.yy4 = OE_SetDflt; /* EV: R-33326-45252 */} + case 77: /* refact ::= SET DEFAULT */ +{ yygotominor.yy328 = OE_SetDflt; /* EV: R-33326-45252 */} break; - case 80: /* refact ::= CASCADE */ -{ yygotominor.yy4 = OE_Cascade; /* EV: R-33326-45252 */} + case 78: /* refact ::= CASCADE */ +{ yygotominor.yy328 = OE_Cascade; /* EV: R-33326-45252 */} break; - case 81: /* refact ::= RESTRICT */ -{ yygotominor.yy4 = OE_Restrict; /* EV: R-33326-45252 */} + case 79: /* refact ::= RESTRICT */ +{ yygotominor.yy328 = OE_Restrict; /* EV: R-33326-45252 */} break; - case 82: /* refact ::= NO ACTION */ -{ yygotominor.yy4 = OE_None; /* EV: R-33326-45252 */} + case 80: /* refact ::= NO ACTION */ +{ yygotominor.yy328 = OE_None; /* EV: R-33326-45252 */} break; - case 84: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 99: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==99); - case 101: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==101); - case 104: /* resolvetype ::= raisetype */ yytestcase(yyruleno==104); -{yygotominor.yy4 = yymsp[0].minor.yy4;} + case 82: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 98: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==98); + case 100: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==100); + case 103: /* resolvetype ::= raisetype */ yytestcase(yyruleno==103); +{yygotominor.yy328 = yymsp[0].minor.yy328;} break; - case 88: /* conslist_opt ::= */ + case 86: /* conslist_opt ::= */ {yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} break; - case 89: /* conslist_opt ::= COMMA conslist */ + case 87: /* conslist_opt ::= COMMA conslist */ {yygotominor.yy0 = yymsp[-1].minor.yy0;} break; - case 94: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy322,yymsp[0].minor.yy4,yymsp[-2].minor.yy4,0);} + case 90: /* tconscomma ::= COMMA */ +{pParse->constraintName.n = 0;} break; - case 95: /* tcons ::= UNIQUE LP idxlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy322,yymsp[0].minor.yy4,0,0,0,0);} + case 93: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy328,yymsp[-2].minor.yy328,0);} break; - case 96: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy118.pExpr);} + case 94: /* tcons ::= UNIQUE LP idxlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy328,0,0,0,0);} break; - case 97: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ + case 95: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy346.pExpr);} + break; + case 96: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy322, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[-1].minor.yy4); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy4); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy328); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy328); } break; - case 100: /* onconf ::= */ -{yygotominor.yy4 = OE_Default;} + case 99: /* onconf ::= */ +{yygotominor.yy328 = OE_Default;} break; - case 102: /* orconf ::= */ -{yygotominor.yy210 = OE_Default;} + case 101: /* orconf ::= */ +{yygotominor.yy186 = OE_Default;} break; - case 103: /* orconf ::= OR resolvetype */ -{yygotominor.yy210 = (u8)yymsp[0].minor.yy4;} + case 102: /* orconf ::= OR resolvetype */ +{yygotominor.yy186 = (u8)yymsp[0].minor.yy328;} break; - case 105: /* resolvetype ::= IGNORE */ -{yygotominor.yy4 = OE_Ignore;} + case 104: /* resolvetype ::= IGNORE */ +{yygotominor.yy328 = OE_Ignore;} break; - case 106: /* resolvetype ::= REPLACE */ -{yygotominor.yy4 = OE_Replace;} + case 105: /* resolvetype ::= REPLACE */ +{yygotominor.yy328 = OE_Replace;} break; - case 107: /* cmd ::= DROP TABLE ifexists fullname */ + case 106: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy259, 0, yymsp[-1].minor.yy4); + sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328); } break; - case 110: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ + case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ { - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy387, yymsp[-6].minor.yy4, yymsp[-4].minor.yy4); + sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy3, yymsp[-6].minor.yy328, yymsp[-4].minor.yy328); } break; - case 111: /* cmd ::= DROP VIEW ifexists fullname */ + case 110: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy259, 1, yymsp[-1].minor.yy4); + sqlite3DropTable(pParse, yymsp[0].minor.yy65, 1, yymsp[-1].minor.yy328); } break; - case 112: /* cmd ::= select */ + case 111: /* cmd ::= select */ { - SelectDest dest = {SRT_Output, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy387, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy387); + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy3, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); } break; - case 113: /* select ::= oneselect */ -{yygotominor.yy387 = yymsp[0].minor.yy387;} - break; - case 114: /* select ::= select multiselect_op oneselect */ + case 112: /* select ::= with selectnowith */ { - if( yymsp[0].minor.yy387 ){ - yymsp[0].minor.yy387->op = (u8)yymsp[-1].minor.yy4; - yymsp[0].minor.yy387->pPrior = yymsp[-2].minor.yy387; + Select *p = yymsp[0].minor.yy3, *pNext, *pLoop; + if( p ){ + int cnt = 0, mxSelect; + p->pWith = yymsp[-1].minor.yy59; + if( p->pPrior ){ + pNext = 0; + for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + } + mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; + if( mxSelect && cnt>mxSelect ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + } + } }else{ - sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy387); + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59); } - yygotominor.yy387 = yymsp[0].minor.yy387; + yygotominor.yy3 = p; +} + break; + case 113: /* selectnowith ::= oneselect */ + case 119: /* oneselect ::= values */ yytestcase(yyruleno==119); +{yygotominor.yy3 = yymsp[0].minor.yy3;} + break; + case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ +{ + Select *pRhs = yymsp[0].minor.yy3; + if( pRhs && pRhs->pPrior ){ + SrcList *pFrom; + Token x; + x.n = 0; + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); + } + if( pRhs ){ + pRhs->op = (u8)yymsp[-1].minor.yy328; + pRhs->pPrior = yymsp[-2].minor.yy3; + if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy3); + } + yygotominor.yy3 = pRhs; } break; case 116: /* multiselect_op ::= UNION ALL */ -{yygotominor.yy4 = TK_ALL;} +{yygotominor.yy328 = TK_ALL;} break; case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yygotominor.yy387 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy322,yymsp[-5].minor.yy259,yymsp[-4].minor.yy314,yymsp[-3].minor.yy322,yymsp[-2].minor.yy314,yymsp[-1].minor.yy322,yymsp[-7].minor.yy4,yymsp[0].minor.yy292.pLimit,yymsp[0].minor.yy292.pOffset); + yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy65,yymsp[-4].minor.yy132,yymsp[-3].minor.yy14,yymsp[-2].minor.yy132,yymsp[-1].minor.yy14,yymsp[-7].minor.yy381,yymsp[0].minor.yy476.pLimit,yymsp[0].minor.yy476.pOffset); +#if SELECTTRACE_ENABLED + /* Populate the Select.zSelName[] string that is used to help with + ** query planner debugging, to differentiate between multiple Select + ** objects in a complex query. + ** + ** If the SELECT keyword is immediately followed by a C-style comment + ** then extract the first few alphanumeric characters from within that + ** comment to be the zSelName value. Otherwise, the label is #N where + ** is an integer that is incremented with each SELECT statement seen. + */ + if( yygotominor.yy3!=0 ){ + const char *z = yymsp[-8].minor.yy0.z+6; + int i; + sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "#%d", + ++pParse->nSelect); + while( z[0]==' ' ) z++; + if( z[0]=='/' && z[1]=='*' ){ + z += 2; + while( z[0]==' ' ) z++; + for(i=0; sqlite3Isalnum(z[i]); i++){} + sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "%.*s", i, z); + } + } +#endif /* SELECTRACE_ENABLED */ } break; - case 122: /* sclp ::= selcollist COMMA */ - case 247: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==247); -{yygotominor.yy322 = yymsp[-1].minor.yy322;} - break; - case 123: /* sclp ::= */ - case 151: /* orderby_opt ::= */ yytestcase(yyruleno==151); - case 159: /* groupby_opt ::= */ yytestcase(yyruleno==159); - case 240: /* exprlist ::= */ yytestcase(yyruleno==240); - case 246: /* idxlist_opt ::= */ yytestcase(yyruleno==246); -{yygotominor.yy322 = 0;} - break; - case 124: /* selcollist ::= sclp expr as */ + case 120: /* values ::= VALUES LP nexprlist RP */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy322, yymsp[-1].minor.yy118.pExpr); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yygotominor.yy322,&yymsp[-1].minor.yy118); + yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); } break; - case 125: /* selcollist ::= sclp STAR */ + case 121: /* values ::= values COMMA LP exprlist RP */ +{ + Select *pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); + if( pRight ){ + pRight->op = TK_ALL; + pRight->pPrior = yymsp[-4].minor.yy3; + yygotominor.yy3 = pRight; + }else{ + yygotominor.yy3 = yymsp[-4].minor.yy3; + } +} + break; + case 122: /* distinct ::= DISTINCT */ +{yygotominor.yy381 = SF_Distinct;} + break; + case 123: /* distinct ::= ALL */ + case 124: /* distinct ::= */ yytestcase(yyruleno==124); +{yygotominor.yy381 = 0;} + break; + case 125: /* sclp ::= selcollist COMMA */ + case 243: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==243); +{yygotominor.yy14 = yymsp[-1].minor.yy14;} + break; + case 126: /* sclp ::= */ + case 154: /* orderby_opt ::= */ yytestcase(yyruleno==154); + case 161: /* groupby_opt ::= */ yytestcase(yyruleno==161); + case 236: /* exprlist ::= */ yytestcase(yyruleno==236); + case 242: /* idxlist_opt ::= */ yytestcase(yyruleno==242); +{yygotominor.yy14 = 0;} + break; + case 127: /* selcollist ::= sclp expr as */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, yymsp[-1].minor.yy346.pExpr); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yygotominor.yy14,&yymsp[-1].minor.yy346); +} + break; + case 128: /* selcollist ::= sclp STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); - yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy322, p); + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy14, p); } break; - case 126: /* selcollist ::= sclp nm DOT STAR */ + case 129: /* selcollist ::= sclp nm DOT STAR */ { Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, pDot); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, pDot); } break; - case 129: /* as ::= */ + case 132: /* as ::= */ {yygotominor.yy0.n = 0;} break; - case 130: /* from ::= */ -{yygotominor.yy259 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy259));} + case 133: /* from ::= */ +{yygotominor.yy65 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy65));} break; - case 131: /* from ::= FROM seltablist */ + case 134: /* from ::= FROM seltablist */ { - yygotominor.yy259 = yymsp[0].minor.yy259; - sqlite3SrcListShiftJoinType(yygotominor.yy259); + yygotominor.yy65 = yymsp[0].minor.yy65; + sqlite3SrcListShiftJoinType(yygotominor.yy65); } break; - case 132: /* stl_prefix ::= seltablist joinop */ + case 135: /* stl_prefix ::= seltablist joinop */ { - yygotominor.yy259 = yymsp[-1].minor.yy259; - if( ALWAYS(yygotominor.yy259 && yygotominor.yy259->nSrc>0) ) yygotominor.yy259->a[yygotominor.yy259->nSrc-1].jointype = (u8)yymsp[0].minor.yy4; + yygotominor.yy65 = yymsp[-1].minor.yy65; + if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].jointype = (u8)yymsp[0].minor.yy328; } break; - case 133: /* stl_prefix ::= */ -{yygotominor.yy259 = 0;} + case 136: /* stl_prefix ::= */ +{yygotominor.yy65 = 0;} break; - case 134: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + case 137: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ { - yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); - sqlite3SrcListIndexedBy(pParse, yygotominor.yy259, &yymsp[-2].minor.yy0); + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + sqlite3SrcListIndexedBy(pParse, yygotominor.yy65, &yymsp[-2].minor.yy0); } break; - case 135: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 138: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ { - yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy387,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy3,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); } break; - case 136: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 139: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { - if( yymsp[-6].minor.yy259==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy314==0 && yymsp[0].minor.yy384==0 ){ - yygotominor.yy259 = yymsp[-4].minor.yy259; + if( yymsp[-6].minor.yy65==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy132==0 && yymsp[0].minor.yy408==0 ){ + yygotominor.yy65 = yymsp[-4].minor.yy65; + }else if( yymsp[-4].minor.yy65->nSrc==1 ){ + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + if( yygotominor.yy65 ){ + struct SrcList_item *pNew = &yygotominor.yy65->a[yygotominor.yy65->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy65->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; + } + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy65); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy259); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy259,0,0,0,0,0,0,0); - yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy65); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy65,0,0,0,0,SF_NestedFrom,0,0); + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); } } break; - case 137: /* dbnm ::= */ - case 146: /* indexed_opt ::= */ yytestcase(yyruleno==146); + case 140: /* dbnm ::= */ + case 149: /* indexed_opt ::= */ yytestcase(yyruleno==149); {yygotominor.yy0.z=0; yygotominor.yy0.n=0;} break; - case 139: /* fullname ::= nm dbnm */ -{yygotominor.yy259 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + case 142: /* fullname ::= nm dbnm */ +{yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} break; - case 140: /* joinop ::= COMMA|JOIN */ -{ yygotominor.yy4 = JT_INNER; } + case 143: /* joinop ::= COMMA|JOIN */ +{ yygotominor.yy328 = JT_INNER; } break; - case 141: /* joinop ::= JOIN_KW JOIN */ -{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } + case 144: /* joinop ::= JOIN_KW JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } break; - case 142: /* joinop ::= JOIN_KW nm JOIN */ -{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } + case 145: /* joinop ::= JOIN_KW nm JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } break; - case 143: /* joinop ::= JOIN_KW nm nm JOIN */ -{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } + case 146: /* joinop ::= JOIN_KW nm nm JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } break; - case 144: /* on_opt ::= ON expr */ - case 155: /* sortitem ::= expr */ yytestcase(yyruleno==155); - case 162: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==162); - case 169: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==169); - case 235: /* case_else ::= ELSE expr */ yytestcase(yyruleno==235); - case 237: /* case_operand ::= expr */ yytestcase(yyruleno==237); -{yygotominor.yy314 = yymsp[0].minor.yy118.pExpr;} + case 147: /* on_opt ::= ON expr */ + case 164: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==164); + case 171: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==171); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 233: /* case_operand ::= expr */ yytestcase(yyruleno==233); +{yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;} break; - case 145: /* on_opt ::= */ - case 161: /* having_opt ::= */ yytestcase(yyruleno==161); - case 168: /* where_opt ::= */ yytestcase(yyruleno==168); - case 236: /* case_else ::= */ yytestcase(yyruleno==236); - case 238: /* case_operand ::= */ yytestcase(yyruleno==238); -{yygotominor.yy314 = 0;} + case 148: /* on_opt ::= */ + case 163: /* having_opt ::= */ yytestcase(yyruleno==163); + case 170: /* where_opt ::= */ yytestcase(yyruleno==170); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 234: /* case_operand ::= */ yytestcase(yyruleno==234); +{yygotominor.yy132 = 0;} break; - case 148: /* indexed_opt ::= NOT INDEXED */ + case 151: /* indexed_opt ::= NOT INDEXED */ {yygotominor.yy0.z=0; yygotominor.yy0.n=1;} break; - case 149: /* using_opt ::= USING LP inscollist RP */ - case 181: /* inscollist_opt ::= LP inscollist RP */ yytestcase(yyruleno==181); -{yygotominor.yy384 = yymsp[-1].minor.yy384;} + case 152: /* using_opt ::= USING LP idlist RP */ + case 180: /* inscollist_opt ::= LP idlist RP */ yytestcase(yyruleno==180); +{yygotominor.yy408 = yymsp[-1].minor.yy408;} break; - case 150: /* using_opt ::= */ - case 180: /* inscollist_opt ::= */ yytestcase(yyruleno==180); -{yygotominor.yy384 = 0;} + case 153: /* using_opt ::= */ + case 179: /* inscollist_opt ::= */ yytestcase(yyruleno==179); +{yygotominor.yy408 = 0;} break; - case 152: /* orderby_opt ::= ORDER BY sortlist */ - case 160: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==160); - case 239: /* exprlist ::= nexprlist */ yytestcase(yyruleno==239); -{yygotominor.yy322 = yymsp[0].minor.yy322;} + case 155: /* orderby_opt ::= ORDER BY sortlist */ + case 162: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==162); + case 235: /* exprlist ::= nexprlist */ yytestcase(yyruleno==235); +{yygotominor.yy14 = yymsp[0].minor.yy14;} break; - case 153: /* sortlist ::= sortlist COMMA sortitem sortorder */ + case 156: /* sortlist ::= sortlist COMMA expr sortorder */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322,yymsp[-1].minor.yy314); - if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; } break; - case 154: /* sortlist ::= sortitem sortorder */ + case 157: /* sortlist ::= expr sortorder */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy314); - if( yygotominor.yy322 && ALWAYS(yygotominor.yy322->a) ) yygotominor.yy322->a[0].sortOrder = (u8)yymsp[0].minor.yy4; + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr); + if( yygotominor.yy14 && ALWAYS(yygotominor.yy14->a) ) yygotominor.yy14->a[0].sortOrder = (u8)yymsp[0].minor.yy328; } break; - case 156: /* sortorder ::= ASC */ - case 158: /* sortorder ::= */ yytestcase(yyruleno==158); -{yygotominor.yy4 = SQLITE_SO_ASC;} + case 158: /* sortorder ::= ASC */ + case 160: /* sortorder ::= */ yytestcase(yyruleno==160); +{yygotominor.yy328 = SQLITE_SO_ASC;} break; - case 157: /* sortorder ::= DESC */ -{yygotominor.yy4 = SQLITE_SO_DESC;} + case 159: /* sortorder ::= DESC */ +{yygotominor.yy328 = SQLITE_SO_DESC;} break; - case 163: /* limit_opt ::= */ -{yygotominor.yy292.pLimit = 0; yygotominor.yy292.pOffset = 0;} + case 165: /* limit_opt ::= */ +{yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;} break; - case 164: /* limit_opt ::= LIMIT expr */ -{yygotominor.yy292.pLimit = yymsp[0].minor.yy118.pExpr; yygotominor.yy292.pOffset = 0;} + case 166: /* limit_opt ::= LIMIT expr */ +{yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;} break; - case 165: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yygotominor.yy292.pLimit = yymsp[-2].minor.yy118.pExpr; yygotominor.yy292.pOffset = yymsp[0].minor.yy118.pExpr;} + case 167: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;} break; - case 166: /* limit_opt ::= LIMIT expr COMMA expr */ -{yygotominor.yy292.pOffset = yymsp[-2].minor.yy118.pExpr; yygotominor.yy292.pLimit = yymsp[0].minor.yy118.pExpr;} + case 168: /* limit_opt ::= LIMIT expr COMMA expr */ +{yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;} break; - case 167: /* cmd ::= DELETE FROM fullname indexed_opt where_opt */ + case 169: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy259, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy259,yymsp[0].minor.yy314); + sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132); } break; - case 170: /* cmd ::= UPDATE orconf fullname indexed_opt SET setlist where_opt */ + case 172: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy259, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy322,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy259,yymsp[-1].minor.yy322,yymsp[0].minor.yy314,yymsp[-5].minor.yy210); + sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy14,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186); } break; - case 171: /* setlist ::= setlist COMMA nm EQ expr */ + case 173: /* setlist ::= setlist COMMA nm EQ expr */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[0].minor.yy118.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 172: /* setlist ::= nm EQ expr */ + case 174: /* setlist ::= nm EQ expr */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy118.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); + yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 173: /* cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP */ -{sqlite3Insert(pParse, yymsp[-5].minor.yy259, yymsp[-1].minor.yy322, 0, yymsp[-4].minor.yy384, yymsp[-7].minor.yy210);} + case 175: /* cmd ::= with insert_cmd INTO fullname inscollist_opt select */ +{ + sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); + sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186); +} break; - case 174: /* cmd ::= insert_cmd INTO fullname inscollist_opt select */ -{sqlite3Insert(pParse, yymsp[-2].minor.yy259, 0, yymsp[0].minor.yy387, yymsp[-1].minor.yy384, yymsp[-4].minor.yy210);} + case 176: /* cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ +{ + sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1); + sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186); +} break; - case 175: /* cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ -{sqlite3Insert(pParse, yymsp[-3].minor.yy259, 0, 0, yymsp[-2].minor.yy384, yymsp[-5].minor.yy210);} + case 177: /* insert_cmd ::= INSERT orconf */ +{yygotominor.yy186 = yymsp[0].minor.yy186;} break; - case 176: /* insert_cmd ::= INSERT orconf */ -{yygotominor.yy210 = yymsp[0].minor.yy210;} + case 178: /* insert_cmd ::= REPLACE */ +{yygotominor.yy186 = OE_Replace;} break; - case 177: /* insert_cmd ::= REPLACE */ -{yygotominor.yy210 = OE_Replace;} + case 181: /* idlist ::= idlist COMMA nm */ +{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);} break; - case 178: /* itemlist ::= itemlist COMMA expr */ - case 241: /* nexprlist ::= nexprlist COMMA expr */ yytestcase(yyruleno==241); -{yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy118.pExpr);} + case 182: /* idlist ::= nm */ +{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} break; - case 179: /* itemlist ::= expr */ - case 242: /* nexprlist ::= expr */ yytestcase(yyruleno==242); -{yygotominor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy118.pExpr);} + case 183: /* expr ::= term */ +{yygotominor.yy346 = yymsp[0].minor.yy346;} break; - case 182: /* inscollist ::= inscollist COMMA nm */ -{yygotominor.yy384 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy384,&yymsp[0].minor.yy0);} + case 184: /* expr ::= LP expr RP */ +{yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} break; - case 183: /* inscollist ::= nm */ -{yygotominor.yy384 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} + case 185: /* term ::= NULL */ + case 190: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==190); + case 191: /* term ::= STRING */ yytestcase(yyruleno==191); +{spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} break; - case 184: /* expr ::= term */ -{yygotominor.yy118 = yymsp[0].minor.yy118;} + case 186: /* expr ::= ID|INDEXED */ + case 187: /* expr ::= JOIN_KW */ yytestcase(yyruleno==187); +{spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);} break; - case 185: /* expr ::= LP expr RP */ -{yygotominor.yy118.pExpr = yymsp[-1].minor.yy118.pExpr; spanSet(&yygotominor.yy118,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} - break; - case 186: /* term ::= NULL */ - case 191: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==191); - case 192: /* term ::= STRING */ yytestcase(yyruleno==192); -{spanExpr(&yygotominor.yy118, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} - break; - case 187: /* expr ::= id */ - case 188: /* expr ::= JOIN_KW */ yytestcase(yyruleno==188); -{spanExpr(&yygotominor.yy118, pParse, TK_ID, &yymsp[0].minor.yy0);} - break; - case 189: /* expr ::= nm DOT nm */ + case 188: /* expr ::= nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); - spanSet(&yygotominor.yy118,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); + spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); } break; - case 190: /* expr ::= nm DOT nm DOT nm */ + case 189: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); - spanSet(&yygotominor.yy118,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); + spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); } break; - case 193: /* expr ::= REGISTER */ + case 192: /* expr ::= VARIABLE */ { - /* When doing a nested parse, one can include terms in an expression - ** that look like this: #1 #2 ... These terms refer to registers - ** in the virtual machine. #N is the N-th register. */ - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); - yygotominor.yy118.pExpr = 0; + if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = 0; + }else{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); + if( yygotominor.yy346.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy346.pExpr->iTable); + } }else{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); - if( yygotominor.yy118.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy118.pExpr->iTable); + spanExpr(&yygotominor.yy346, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yygotominor.yy346.pExpr); } - spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 194: /* expr ::= VARIABLE */ + case 193: /* expr ::= expr COLLATE ID|STRING */ { - spanExpr(&yygotominor.yy118, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yygotominor.yy118.pExpr); - spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0); + yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 195: /* expr ::= expr COLLATE ids */ + case 194: /* expr ::= CAST LP expr AS typetoken RP */ { - yygotominor.yy118.pExpr = sqlite3ExprSetCollByToken(pParse, yymsp[-2].minor.yy118.pExpr, &yymsp[0].minor.yy0); - yygotominor.yy118.zStart = yymsp[-2].minor.yy118.zStart; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); } break; - case 196: /* expr ::= CAST LP expr AS typetoken RP */ + case 195: /* expr ::= ID|INDEXED LP distinct exprlist RP */ { - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy118.pExpr, 0, &yymsp[-1].minor.yy0); - spanSet(&yygotominor.yy118,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); -} - break; - case 197: /* expr ::= ID LP distinct exprlist RP */ -{ - if( yymsp[-1].minor.yy322 && yymsp[-1].minor.yy322->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); } - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); - spanSet(&yygotominor.yy118,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy4 && yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->flags |= EP_Distinct; + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy381 && yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->flags |= EP_Distinct; } } break; - case 198: /* expr ::= ID LP STAR RP */ + case 196: /* expr ::= ID|INDEXED LP STAR RP */ { - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); - spanSet(&yygotominor.yy118,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); } break; - case 199: /* term ::= CTIME_KW */ + case 197: /* term ::= CTIME_KW */ { - /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are - ** treated as functions that return constants */ - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, 0,&yymsp[0].minor.yy0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->op = TK_CONST_FUNC; - } - spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); + spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 200: /* expr ::= expr AND expr */ - case 201: /* expr ::= expr OR expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==202); - case 203: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==203); - case 204: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==204); - case 205: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==205); - case 206: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==206); - case 207: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==207); -{spanBinaryExpr(&yygotominor.yy118,pParse,yymsp[-1].major,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy118);} + case 198: /* expr ::= expr AND expr */ + case 199: /* expr ::= expr OR expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==204); + case 205: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==205); +{spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);} break; - case 208: /* likeop ::= LIKE_KW */ - case 210: /* likeop ::= MATCH */ yytestcase(yyruleno==210); -{yygotominor.yy342.eOperator = yymsp[0].minor.yy0; yygotominor.yy342.not = 0;} + case 206: /* likeop ::= LIKE_KW|MATCH */ +{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;} break; - case 209: /* likeop ::= NOT LIKE_KW */ - case 211: /* likeop ::= NOT MATCH */ yytestcase(yyruleno==211); -{yygotominor.yy342.eOperator = yymsp[0].minor.yy0; yygotominor.yy342.not = 1;} + case 207: /* likeop ::= NOT LIKE_KW|MATCH */ +{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;} break; - case 212: /* expr ::= expr likeop expr */ + case 208: /* expr ::= expr likeop expr */ { ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy118.pExpr); - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy342.eOperator); - if( yymsp[-1].minor.yy342.not ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-2].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; - if( yygotominor.yy118.pExpr ) yygotominor.yy118.pExpr->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy96.eOperator); + if( yymsp[-1].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; } break; - case 213: /* expr ::= expr likeop expr ESCAPE expr */ + case 209: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy118.pExpr); - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy342.eOperator); - if( yymsp[-3].minor.yy342.not ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; - if( yygotominor.yy118.pExpr ) yygotominor.yy118.pExpr->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy96.eOperator); + if( yymsp[-3].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; } break; - case 214: /* expr ::= expr ISNULL|NOTNULL */ -{spanUnaryPostfix(&yygotominor.yy118,pParse,yymsp[0].major,&yymsp[-1].minor.yy118,&yymsp[0].minor.yy0);} + case 210: /* expr ::= expr ISNULL|NOTNULL */ +{spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);} break; - case 215: /* expr ::= expr NOT NULL */ -{spanUnaryPostfix(&yygotominor.yy118,pParse,TK_NOTNULL,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy0);} + case 211: /* expr ::= expr NOT NULL */ +{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);} break; - case 216: /* expr ::= expr IS expr */ + case 212: /* expr ::= expr IS expr */ { - spanBinaryExpr(&yygotominor.yy118,pParse,TK_IS,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy118); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy118.pExpr, yygotominor.yy118.pExpr, TK_ISNULL); + spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL); } break; - case 217: /* expr ::= expr IS NOT expr */ + case 213: /* expr ::= expr IS NOT expr */ { - spanBinaryExpr(&yygotominor.yy118,pParse,TK_ISNOT,&yymsp[-3].minor.yy118,&yymsp[0].minor.yy118); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy118.pExpr, yygotominor.yy118.pExpr, TK_NOTNULL); + spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL); } break; - case 218: /* expr ::= NOT expr */ - case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219); -{spanUnaryPrefix(&yygotominor.yy118,pParse,yymsp[-1].major,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 220: /* expr ::= MINUS expr */ -{spanUnaryPrefix(&yygotominor.yy118,pParse,TK_UMINUS,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} + case 216: /* expr ::= MINUS expr */ +{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 221: /* expr ::= PLUS expr */ -{spanUnaryPrefix(&yygotominor.yy118,pParse,TK_UPLUS,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} + case 217: /* expr ::= PLUS expr */ +{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 224: /* expr ::= expr between_op expr AND expr */ + case 220: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy118.pExpr); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; } break; - case 227: /* expr ::= expr in_op LP exprlist RP */ + case 223: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy322==0 ){ + if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -106101,226 +123108,252 @@ static void yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy4]); - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy118.pExpr); - }else{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pList = yymsp[-1].minor.yy322; - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy328]); + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy346.pExpr); + }else if( yymsp[-1].minor.yy14->nExpr==1 ){ + /* Expressions of the form: + ** + ** expr1 IN (?1) + ** expr1 NOT IN (?2) + ** + ** with exactly one value on the RHS can be simplified to something + ** like this: + ** + ** expr1 == ?1 + ** expr1 <> ?2 + ** + ** But, the RHS of the == or <> is marked with the EP_Generic flag + ** so that it may not contribute to the computation of comparison + ** affinity or the collating sequence to use for comparison. Otherwise, + ** the semantics would be subtly different from IN or NOT IN. + */ + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + /* pRHS cannot be NULL because a malloc error would have been detected + ** before now and control would have never reached this point */ + if( ALWAYS(pRHS) ){ + pRHS->flags &= ~EP_Collate; + pRHS->flags |= EP_Generic; } - if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - } - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 228: /* expr ::= LP select RP */ -{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pSelect = yymsp[-1].minor.yy387; - ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy328 ? TK_NE : TK_EQ, yymsp[-4].minor.yy346.pExpr, pRHS, 0); }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); } - yygotominor.yy118.zStart = yymsp[-2].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 229: /* expr ::= expr in_op LP select RP */ + case 224: /* expr ::= LP select RP */ { - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pSelect = yymsp[-1].minor.yy387; - ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); } - if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy346.zStart = yymsp[-2].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 230: /* expr ::= expr in_op nm dbnm */ + case 225: /* expr ::= expr in_op LP select RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 226: /* expr ::= expr in_op nm dbnm */ { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); }else{ sqlite3SrcListDelete(pParse->db, pSrc); } - if( yymsp[-2].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-3].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; + if( yymsp[-2].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-3].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; } break; - case 231: /* expr ::= EXISTS LP select RP */ + case 227: /* expr ::= EXISTS LP select RP */ { - Expr *p = yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); if( p ){ - p->x.pSelect = yymsp[-1].minor.yy387; + p->x.pSelect = yymsp[-1].minor.yy3; ExprSetProperty(p, EP_xIsSelect); sqlite3ExprSetHeight(pParse, p); }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); } - yygotominor.yy118.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 232: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy314, yymsp[-1].minor.yy314, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pList = yymsp[-2].minor.yy322; - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy132 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy132) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeight(pParse, yygotominor.yy346.pExpr); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy322); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy132); } - yygotominor.yy118.zStart = yymsp[-4].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy346.zStart = yymsp[-4].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 233: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy118.pExpr); - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yygotominor.yy322, yymsp[0].minor.yy118.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); } break; - case 234: /* case_exprlist ::= WHEN expr THEN expr */ + case 230: /* case_exprlist ::= WHEN expr THEN expr */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yygotominor.yy322, yymsp[0].minor.yy118.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); } break; - case 243: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP */ + case 237: /* nexprlist ::= nexprlist COMMA expr */ +{yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);} + break; + case 238: /* nexprlist ::= expr */ +{yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);} + break; + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt */ { - sqlite3CreateIndex(pParse, &yymsp[-6].minor.yy0, &yymsp[-5].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-3].minor.yy0,0), yymsp[-1].minor.yy322, yymsp[-9].minor.yy4, - &yymsp[-10].minor.yy0, &yymsp[0].minor.yy0, SQLITE_SO_ASC, yymsp[-7].minor.yy4); + sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, + sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328); } break; - case 244: /* uniqueflag ::= UNIQUE */ - case 298: /* raisetype ::= ABORT */ yytestcase(yyruleno==298); -{yygotominor.yy4 = OE_Abort;} + case 240: /* uniqueflag ::= UNIQUE */ + case 291: /* raisetype ::= ABORT */ yytestcase(yyruleno==291); +{yygotominor.yy328 = OE_Abort;} break; - case 245: /* uniqueflag ::= */ -{yygotominor.yy4 = OE_None;} + case 241: /* uniqueflag ::= */ +{yygotominor.yy328 = OE_None;} break; - case 248: /* idxlist ::= idxlist COMMA nm collate sortorder */ + case 244: /* idxlist ::= idxlist COMMA nm collate sortorder */ { - Expr *p = 0; - if( yymsp[-1].minor.yy0.n>0 ){ - p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); - } - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, p); - sqlite3ExprListSetName(pParse,yygotominor.yy322,&yymsp[-2].minor.yy0,1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy322, "index"); - if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, p); + sqlite3ExprListSetName(pParse,yygotominor.yy14,&yymsp[-2].minor.yy0,1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; } break; - case 249: /* idxlist ::= nm collate sortorder */ + case 245: /* idxlist ::= nm collate sortorder */ { - Expr *p = 0; - if( yymsp[-1].minor.yy0.n>0 ){ - p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); - } - yygotominor.yy322 = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy322, "index"); - if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, p); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; } break; - case 250: /* collate ::= */ + case 246: /* collate ::= */ {yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} break; - case 252: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy259, yymsp[-1].minor.yy4);} + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);} break; - case 253: /* cmd ::= VACUUM */ - case 254: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==254); + case 249: /* cmd ::= VACUUM */ + case 250: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==250); {sqlite3Vacuum(pParse);} break; - case 255: /* cmd ::= PRAGMA nm dbnm */ + case 251: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 256: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 257: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 258: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 259: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 270: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 264: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy203, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all); } break; - case 271: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 265: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy4, yymsp[-4].minor.yy90.a, yymsp[-4].minor.yy90.b, yymsp[-2].minor.yy259, yymsp[0].minor.yy314, yymsp[-10].minor.yy4, yymsp[-8].minor.yy4); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328); yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); } break; - case 272: /* trigger_time ::= BEFORE */ - case 275: /* trigger_time ::= */ yytestcase(yyruleno==275); -{ yygotominor.yy4 = TK_BEFORE; } + case 266: /* trigger_time ::= BEFORE */ + case 269: /* trigger_time ::= */ yytestcase(yyruleno==269); +{ yygotominor.yy328 = TK_BEFORE; } break; - case 273: /* trigger_time ::= AFTER */ -{ yygotominor.yy4 = TK_AFTER; } + case 267: /* trigger_time ::= AFTER */ +{ yygotominor.yy328 = TK_AFTER; } break; - case 274: /* trigger_time ::= INSTEAD OF */ -{ yygotominor.yy4 = TK_INSTEAD;} + case 268: /* trigger_time ::= INSTEAD OF */ +{ yygotominor.yy328 = TK_INSTEAD;} break; - case 276: /* trigger_event ::= DELETE|INSERT */ - case 277: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==277); -{yygotominor.yy90.a = yymsp[0].major; yygotominor.yy90.b = 0;} + case 270: /* trigger_event ::= DELETE|INSERT */ + case 271: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==271); +{yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;} break; - case 278: /* trigger_event ::= UPDATE OF inscollist */ -{yygotominor.yy90.a = TK_UPDATE; yygotominor.yy90.b = yymsp[0].minor.yy384;} + case 272: /* trigger_event ::= UPDATE OF idlist */ +{yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;} break; - case 281: /* when_clause ::= */ - case 303: /* key_opt ::= */ yytestcase(yyruleno==303); -{ yygotominor.yy314 = 0; } + case 275: /* when_clause ::= */ + case 296: /* key_opt ::= */ yytestcase(yyruleno==296); +{ yygotominor.yy132 = 0; } break; - case 282: /* when_clause ::= WHEN expr */ - case 304: /* key_opt ::= KEY expr */ yytestcase(yyruleno==304); -{ yygotominor.yy314 = yymsp[0].minor.yy118.pExpr; } + case 276: /* when_clause ::= WHEN expr */ + case 297: /* key_opt ::= KEY expr */ yytestcase(yyruleno==297); +{ yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; } break; - case 283: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 277: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy203!=0 ); - yymsp[-2].minor.yy203->pLast->pNext = yymsp[-1].minor.yy203; - yymsp[-2].minor.yy203->pLast = yymsp[-1].minor.yy203; - yygotominor.yy203 = yymsp[-2].minor.yy203; + assert( yymsp[-2].minor.yy473!=0 ); + yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473; + yymsp[-2].minor.yy473->pLast = yymsp[-1].minor.yy473; + yygotominor.yy473 = yymsp[-2].minor.yy473; } break; - case 284: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 278: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy203!=0 ); - yymsp[-1].minor.yy203->pLast = yymsp[-1].minor.yy203; - yygotominor.yy203 = yymsp[-1].minor.yy203; + assert( yymsp[-1].minor.yy473!=0 ); + yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473; + yygotominor.yy473 = yymsp[-1].minor.yy473; } break; - case 286: /* trnm ::= nm DOT nm */ + case 280: /* trnm ::= nm DOT nm */ { yygotominor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -106328,123 +123361,137 @@ static void yy_reduce( "statements within triggers"); } break; - case 288: /* tridxby ::= INDEXED BY nm */ + case 282: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 289: /* tridxby ::= NOT INDEXED */ + case 283: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 290: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ -{ yygotominor.yy203 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy322, yymsp[0].minor.yy314, yymsp[-5].minor.yy210); } + case 284: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ +{ yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); } break; - case 291: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP */ -{yygotominor.yy203 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy384, yymsp[-1].minor.yy322, 0, yymsp[-7].minor.yy210);} + case 285: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ +{yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);} break; - case 292: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ -{yygotominor.yy203 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy384, 0, yymsp[0].minor.yy387, yymsp[-4].minor.yy210);} + case 286: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ +{yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);} break; - case 293: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ -{yygotominor.yy203 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy314);} + case 287: /* trigger_cmd ::= select */ +{yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); } break; - case 294: /* trigger_cmd ::= select */ -{yygotominor.yy203 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy387); } - break; - case 295: /* expr ::= RAISE LP IGNORE RP */ + case 288: /* expr ::= RAISE LP IGNORE RP */ { - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->affinity = OE_Ignore; + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->affinity = OE_Ignore; } - yygotominor.yy118.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 296: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 289: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); - if( yygotominor.yy118.pExpr ) { - yygotominor.yy118.pExpr->affinity = (char)yymsp[-3].minor.yy4; + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yygotominor.yy346.pExpr ) { + yygotominor.yy346.pExpr->affinity = (char)yymsp[-3].minor.yy328; } - yygotominor.yy118.zStart = yymsp[-5].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy346.zStart = yymsp[-5].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 297: /* raisetype ::= ROLLBACK */ -{yygotominor.yy4 = OE_Rollback;} + case 290: /* raisetype ::= ROLLBACK */ +{yygotominor.yy328 = OE_Rollback;} break; - case 299: /* raisetype ::= FAIL */ -{yygotominor.yy4 = OE_Fail;} + case 292: /* raisetype ::= FAIL */ +{yygotominor.yy328 = OE_Fail;} break; - case 300: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 293: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy259,yymsp[-1].minor.yy4); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328); } break; - case 301: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 294: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy118.pExpr, yymsp[-1].minor.yy118.pExpr, yymsp[0].minor.yy314); + sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132); } break; - case 302: /* cmd ::= DETACH database_kw_opt expr */ + case 295: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy118.pExpr); + sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr); } break; - case 307: /* cmd ::= REINDEX */ + case 300: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 308: /* cmd ::= REINDEX nm dbnm */ + case 301: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 309: /* cmd ::= ANALYZE */ + case 302: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 310: /* cmd ::= ANALYZE nm dbnm */ + case 303: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 311: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 304: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy259,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0); } break; - case 312: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ + case 305: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ { sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); } break; - case 313: /* add_column_fullname ::= fullname */ + case 306: /* add_column_fullname ::= fullname */ { pParse->db->lookaside.bEnabled = 0; - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy259); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65); } break; - case 316: /* cmd ::= create_vtab */ + case 309: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 317: /* cmd ::= create_vtab LP vtabarglist RP */ + case 310: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 318: /* create_vtab ::= createkw VIRTUAL TABLE nm dbnm USING nm */ + case 311: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328); } break; - case 321: /* vtabarg ::= */ + case 314: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 323: /* vtabargtoken ::= ANY */ - case 324: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==324); - case 325: /* lp ::= LP */ yytestcase(yyruleno==325); + case 316: /* vtabargtoken ::= ANY */ + case 317: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==317); + case 318: /* lp ::= LP */ yytestcase(yyruleno==318); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; + case 322: /* with ::= */ +{yygotominor.yy59 = 0;} + break; + case 323: /* with ::= WITH wqlist */ + case 324: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==324); +{ yygotominor.yy59 = yymsp[0].minor.yy59; } + break; + case 325: /* wqlist ::= nm idxlist_opt AS LP select RP */ +{ + yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); +} + break; + case 326: /* wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP */ +{ + yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); +} + break; default: /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); @@ -106457,37 +123504,33 @@ static void yy_reduce( /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20); /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21); /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25); - /* (34) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==34); - /* (35) columnlist ::= column */ yytestcase(yyruleno==35); - /* (44) type ::= */ yytestcase(yyruleno==44); - /* (51) signed ::= plus_num */ yytestcase(yyruleno==51); - /* (52) signed ::= minus_num */ yytestcase(yyruleno==52); - /* (53) carglist ::= carglist carg */ yytestcase(yyruleno==53); - /* (54) carglist ::= */ yytestcase(yyruleno==54); - /* (55) carg ::= CONSTRAINT nm ccons */ yytestcase(yyruleno==55); - /* (56) carg ::= ccons */ yytestcase(yyruleno==56); - /* (62) ccons ::= NULL onconf */ yytestcase(yyruleno==62); - /* (90) conslist ::= conslist COMMA tcons */ yytestcase(yyruleno==90); - /* (91) conslist ::= conslist tcons */ yytestcase(yyruleno==91); - /* (92) conslist ::= tcons */ yytestcase(yyruleno==92); - /* (93) tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==93); - /* (268) plus_opt ::= PLUS */ yytestcase(yyruleno==268); - /* (269) plus_opt ::= */ yytestcase(yyruleno==269); - /* (279) foreach_clause ::= */ yytestcase(yyruleno==279); - /* (280) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==280); - /* (287) tridxby ::= */ yytestcase(yyruleno==287); - /* (305) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==305); - /* (306) database_kw_opt ::= */ yytestcase(yyruleno==306); - /* (314) kwcolumn_opt ::= */ yytestcase(yyruleno==314); - /* (315) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==315); - /* (319) vtabarglist ::= vtabarg */ yytestcase(yyruleno==319); - /* (320) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==320); - /* (322) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==322); - /* (326) anylist ::= */ yytestcase(yyruleno==326); - /* (327) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==327); - /* (328) anylist ::= anylist ANY */ yytestcase(yyruleno==328); + /* (36) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==36); + /* (37) columnlist ::= column */ yytestcase(yyruleno==37); + /* (43) type ::= */ yytestcase(yyruleno==43); + /* (50) signed ::= plus_num */ yytestcase(yyruleno==50); + /* (51) signed ::= minus_num */ yytestcase(yyruleno==51); + /* (52) carglist ::= carglist ccons */ yytestcase(yyruleno==52); + /* (53) carglist ::= */ yytestcase(yyruleno==53); + /* (60) ccons ::= NULL onconf */ yytestcase(yyruleno==60); + /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88); + /* (89) conslist ::= tcons */ yytestcase(yyruleno==89); + /* (91) tconscomma ::= */ yytestcase(yyruleno==91); + /* (273) foreach_clause ::= */ yytestcase(yyruleno==273); + /* (274) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==274); + /* (281) tridxby ::= */ yytestcase(yyruleno==281); + /* (298) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==298); + /* (299) database_kw_opt ::= */ yytestcase(yyruleno==299); + /* (307) kwcolumn_opt ::= */ yytestcase(yyruleno==307); + /* (308) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==308); + /* (312) vtabarglist ::= vtabarg */ yytestcase(yyruleno==312); + /* (313) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==313); + /* (315) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==315); + /* (319) anylist ::= */ yytestcase(yyruleno==319); + /* (320) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==320); + /* (321) anylist ::= anylist ANY */ yytestcase(yyruleno==321); break; }; + assert( yyruleno>=0 && yyrulenoyyidx -= yysize; @@ -106549,7 +123592,6 @@ static void yy_syntax_error( UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); - pParse->parseError = 1; sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } @@ -106598,7 +123640,9 @@ SQLITE_PRIVATE void sqlite3Parser( ){ YYMINORTYPE yyminorunion; int yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ +#endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif @@ -106621,7 +123665,9 @@ SQLITE_PRIVATE void sqlite3Parser( yypParser->yystack[0].major = 0; } yyminorunion.yy0 = yyminor; +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); +#endif sqlite3ParserARG_STORE; #ifndef NDEBUG @@ -106633,7 +123679,6 @@ SQLITE_PRIVATE void sqlite3Parser( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyactyyerrcnt--; yymajor = YYNOCODE; @@ -106760,6 +123805,7 @@ SQLITE_PRIVATE void sqlite3Parser( ** individual tokens and sends those tokens one-by-one over to the ** parser for analysis. */ +/* #include */ /* ** The charMap() macro maps alphabetic characters into their @@ -106820,20 +123866,20 @@ const unsigned char ebcdicToAscii[] = { ** is substantially reduced. This is important for embedded applications ** on platforms with limited memory. */ -/* Hash score: 175 */ +/* Hash score: 182 */ static int keywordCode(const char *z, int n){ - /* zText[] encodes 811 bytes of keywords in 541 bytes */ + /* zText[] encodes 834 bytes of keywords in 554 bytes */ /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ - /* UNIQUERYATTACHAVINGROUPDATEBEGINNERELEASEBETWEENOTNULLIKE */ - /* CASCADELETECASECOLLATECREATECURRENT_DATEDETACHIMMEDIATEJOIN */ - /* SERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHENWHERENAME */ - /* AFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSS */ - /* CURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAILFROMFULLGLOBYIF */ - /* ISNULLORDERESTRICTOUTERIGHTROLLBACKROWUNIONUSINGVACUUMVIEW */ - /* INITIALLY */ - static const char zText[540] = { + /* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */ + /* BETWEENOTNULLIKECASCADELETECASECOLLATECREATECURRENT_DATEDETACH */ + /* IMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHEN */ + /* WHERENAMEAFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMIT */ + /* CONFLICTCROSSCURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAIL */ + /* FROMFULLGLOBYIFISNULLORDERESTRICTRIGHTROLLBACKROWUNIONUSING */ + /* VACUUMVIEWINITIALLY */ + static const char zText[553] = { 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', @@ -106844,76 +123890,77 @@ static int keywordCode(const char *z, int n){ 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', - 'U','E','R','Y','A','T','T','A','C','H','A','V','I','N','G','R','O','U', - 'P','D','A','T','E','B','E','G','I','N','N','E','R','E','L','E','A','S', - 'E','B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C', - 'A','S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L', - 'A','T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D', - 'A','T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E', - 'J','O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A', - 'L','Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U', - 'E','S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W', - 'H','E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C', - 'E','A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R', - 'E','M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M', - 'M','I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U', - 'R','R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M', - 'A','R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T', - 'D','R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L', - 'O','B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S', - 'T','R','I','C','T','O','U','T','E','R','I','G','H','T','R','O','L','L', - 'B','A','C','K','R','O','W','U','N','I','O','N','U','S','I','N','G','V', - 'A','C','U','U','M','V','I','E','W','I','N','I','T','I','A','L','L','Y', + 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', + 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', + 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E', + 'B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C','A', + 'S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L','A', + 'T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A', + 'T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E','J', + 'O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A','L', + 'Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U','E', + 'S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W','H', + 'E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C','E', + 'A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R','E', + 'M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M','M', + 'I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U','R', + 'R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M','A', + 'R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T','D', + 'R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L','O', + 'B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T', + 'R','I','C','T','R','I','G','H','T','R','O','L','L','B','A','C','K','R', + 'O','W','U','N','I','O','N','U','S','I','N','G','V','A','C','U','U','M', + 'V','I','E','W','I','N','I','T','I','A','L','L','Y', }; static const unsigned char aHash[127] = { - 72, 101, 114, 70, 0, 45, 0, 0, 78, 0, 73, 0, 0, - 42, 12, 74, 15, 0, 113, 81, 50, 108, 0, 19, 0, 0, - 118, 0, 116, 111, 0, 22, 89, 0, 9, 0, 0, 66, 67, - 0, 65, 6, 0, 48, 86, 98, 0, 115, 97, 0, 0, 44, - 0, 99, 24, 0, 17, 0, 119, 49, 23, 0, 5, 106, 25, - 92, 0, 0, 121, 102, 56, 120, 53, 28, 51, 0, 87, 0, - 96, 26, 0, 95, 0, 0, 0, 91, 88, 93, 84, 105, 14, - 39, 104, 0, 77, 0, 18, 85, 107, 32, 0, 117, 76, 109, - 58, 46, 80, 0, 0, 90, 40, 0, 112, 0, 36, 0, 0, - 29, 0, 82, 59, 60, 0, 20, 57, 0, 52, + 76, 105, 117, 74, 0, 45, 0, 0, 82, 0, 77, 0, 0, + 42, 12, 78, 15, 0, 116, 85, 54, 112, 0, 19, 0, 0, + 121, 0, 119, 115, 0, 22, 93, 0, 9, 0, 0, 70, 71, + 0, 69, 6, 0, 48, 90, 102, 0, 118, 101, 0, 0, 44, + 0, 103, 24, 0, 17, 0, 122, 53, 23, 0, 5, 110, 25, + 96, 0, 0, 124, 106, 60, 123, 57, 28, 55, 0, 91, 0, + 100, 26, 0, 99, 0, 0, 0, 95, 92, 97, 88, 109, 14, + 39, 108, 0, 81, 0, 18, 89, 111, 32, 0, 120, 80, 113, + 62, 46, 84, 0, 0, 94, 40, 59, 114, 0, 36, 0, 0, + 29, 0, 86, 63, 64, 0, 20, 61, 0, 56, }; - static const unsigned char aNext[121] = { + static const unsigned char aNext[124] = { 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 43, 3, 47, - 0, 0, 0, 0, 30, 0, 54, 0, 38, 0, 0, 0, 1, - 62, 0, 0, 63, 0, 41, 0, 0, 0, 0, 0, 0, 0, - 61, 0, 0, 0, 0, 31, 55, 16, 34, 10, 0, 0, 0, - 0, 0, 0, 0, 11, 68, 75, 0, 8, 0, 100, 94, 0, - 103, 0, 83, 0, 71, 0, 0, 110, 27, 37, 69, 79, 0, - 35, 64, 0, 0, + 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, + 0, 43, 3, 47, 0, 0, 0, 0, 30, 0, 58, 0, 38, + 0, 0, 0, 1, 66, 0, 0, 67, 0, 41, 0, 0, 0, + 0, 0, 0, 49, 65, 0, 0, 0, 0, 31, 52, 16, 34, + 10, 0, 0, 0, 0, 0, 0, 0, 11, 72, 79, 0, 8, + 0, 104, 98, 0, 107, 0, 87, 0, 75, 51, 0, 27, 37, + 73, 83, 0, 35, 68, 0, 0, }; - static const unsigned char aLen[121] = { + static const unsigned char aLen[124] = { 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, - 4, 6, 2, 3, 9, 4, 2, 6, 5, 6, 6, 5, 6, - 5, 5, 7, 7, 7, 3, 2, 4, 4, 7, 3, 6, 4, - 7, 6, 12, 6, 9, 4, 6, 5, 4, 7, 6, 5, 6, - 7, 5, 4, 5, 6, 5, 7, 3, 7, 13, 2, 2, 4, - 6, 6, 8, 5, 17, 12, 7, 8, 8, 2, 4, 4, 4, - 4, 4, 2, 2, 6, 5, 8, 5, 5, 8, 3, 5, 5, - 6, 4, 9, 3, + 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, + 6, 6, 5, 6, 5, 5, 9, 7, 7, 3, 2, 4, 4, + 7, 3, 6, 4, 7, 6, 12, 6, 9, 4, 6, 5, 4, + 7, 6, 5, 6, 7, 5, 4, 5, 6, 5, 7, 3, 7, + 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 8, 8, + 2, 4, 4, 4, 4, 4, 2, 2, 6, 5, 8, 5, 8, + 3, 5, 5, 6, 4, 9, 3, }; - static const unsigned short int aOffset[121] = { + static const unsigned short int aOffset[124] = { 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, - 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 189, 194, 197, - 203, 206, 210, 217, 223, 223, 223, 226, 229, 233, 234, 238, 244, - 248, 255, 261, 273, 279, 288, 290, 296, 301, 303, 310, 315, 320, - 326, 332, 337, 341, 344, 350, 354, 361, 363, 370, 372, 374, 383, - 387, 393, 399, 407, 412, 412, 428, 435, 442, 443, 450, 454, 458, - 462, 466, 469, 471, 473, 479, 483, 491, 495, 500, 508, 511, 516, - 521, 527, 531, 536, + 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, + 199, 204, 209, 212, 218, 221, 225, 234, 240, 240, 240, 243, 246, + 250, 251, 255, 261, 265, 272, 278, 290, 296, 305, 307, 313, 318, + 320, 327, 332, 337, 343, 349, 354, 358, 361, 367, 371, 378, 380, + 387, 389, 391, 400, 404, 410, 416, 424, 429, 429, 445, 452, 459, + 460, 467, 471, 475, 479, 483, 486, 488, 490, 496, 500, 508, 513, + 521, 524, 529, 534, 540, 544, 549, }; - static const unsigned char aCode[121] = { + static const unsigned char aCode[124] = { TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, @@ -106923,22 +123970,22 @@ static int keywordCode(const char *z, int n){ TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, - TK_OR, TK_UNIQUE, TK_QUERY, TK_ATTACH, TK_HAVING, - TK_GROUP, TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RELEASE, - TK_BETWEEN, TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, - TK_LIKE_KW, TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, - TK_COLLATE, TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, - TK_JOIN, TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, - TK_PRAGMA, TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, - TK_WHEN, TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, - TK_AND, TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, - TK_CAST, TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, - TK_CTIME_KW, TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, - TK_IS, TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, - TK_LIKE_KW, TK_BY, TK_IF, TK_ISNULL, TK_ORDER, - TK_RESTRICT, TK_JOIN_KW, TK_JOIN_KW, TK_ROLLBACK, TK_ROW, - TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, - TK_ALL, + TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, + TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, + TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN, + TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, + TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE, + TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN, + TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, + TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, + TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, TK_AND, + TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, + TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, + TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, TK_IS, + TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, TK_LIKE_KW, + TK_BY, TK_IF, TK_ISNULL, TK_ORDER, TK_RESTRICT, + TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, + TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, }; int h, i; if( n<2 ) return TK_ID; @@ -106995,79 +124042,82 @@ static int keywordCode(const char *z, int n){ testcase( i==45 ); /* OR */ testcase( i==46 ); /* UNIQUE */ testcase( i==47 ); /* QUERY */ - testcase( i==48 ); /* ATTACH */ - testcase( i==49 ); /* HAVING */ - testcase( i==50 ); /* GROUP */ - testcase( i==51 ); /* UPDATE */ - testcase( i==52 ); /* BEGIN */ - testcase( i==53 ); /* INNER */ - testcase( i==54 ); /* RELEASE */ - testcase( i==55 ); /* BETWEEN */ - testcase( i==56 ); /* NOTNULL */ - testcase( i==57 ); /* NOT */ - testcase( i==58 ); /* NO */ - testcase( i==59 ); /* NULL */ - testcase( i==60 ); /* LIKE */ - testcase( i==61 ); /* CASCADE */ - testcase( i==62 ); /* ASC */ - testcase( i==63 ); /* DELETE */ - testcase( i==64 ); /* CASE */ - testcase( i==65 ); /* COLLATE */ - testcase( i==66 ); /* CREATE */ - testcase( i==67 ); /* CURRENT_DATE */ - testcase( i==68 ); /* DETACH */ - testcase( i==69 ); /* IMMEDIATE */ - testcase( i==70 ); /* JOIN */ - testcase( i==71 ); /* INSERT */ - testcase( i==72 ); /* MATCH */ - testcase( i==73 ); /* PLAN */ - testcase( i==74 ); /* ANALYZE */ - testcase( i==75 ); /* PRAGMA */ - testcase( i==76 ); /* ABORT */ - testcase( i==77 ); /* VALUES */ - testcase( i==78 ); /* VIRTUAL */ - testcase( i==79 ); /* LIMIT */ - testcase( i==80 ); /* WHEN */ - testcase( i==81 ); /* WHERE */ - testcase( i==82 ); /* RENAME */ - testcase( i==83 ); /* AFTER */ - testcase( i==84 ); /* REPLACE */ - testcase( i==85 ); /* AND */ - testcase( i==86 ); /* DEFAULT */ - testcase( i==87 ); /* AUTOINCREMENT */ - testcase( i==88 ); /* TO */ - testcase( i==89 ); /* IN */ - testcase( i==90 ); /* CAST */ - testcase( i==91 ); /* COLUMN */ - testcase( i==92 ); /* COMMIT */ - testcase( i==93 ); /* CONFLICT */ - testcase( i==94 ); /* CROSS */ - testcase( i==95 ); /* CURRENT_TIMESTAMP */ - testcase( i==96 ); /* CURRENT_TIME */ - testcase( i==97 ); /* PRIMARY */ - testcase( i==98 ); /* DEFERRED */ - testcase( i==99 ); /* DISTINCT */ - testcase( i==100 ); /* IS */ - testcase( i==101 ); /* DROP */ - testcase( i==102 ); /* FAIL */ - testcase( i==103 ); /* FROM */ - testcase( i==104 ); /* FULL */ - testcase( i==105 ); /* GLOB */ - testcase( i==106 ); /* BY */ - testcase( i==107 ); /* IF */ - testcase( i==108 ); /* ISNULL */ - testcase( i==109 ); /* ORDER */ - testcase( i==110 ); /* RESTRICT */ - testcase( i==111 ); /* OUTER */ - testcase( i==112 ); /* RIGHT */ - testcase( i==113 ); /* ROLLBACK */ - testcase( i==114 ); /* ROW */ - testcase( i==115 ); /* UNION */ - testcase( i==116 ); /* USING */ - testcase( i==117 ); /* VACUUM */ - testcase( i==118 ); /* VIEW */ - testcase( i==119 ); /* INITIALLY */ - testcase( i==120 ); /* ALL */ + testcase( i==48 ); /* WITHOUT */ + testcase( i==49 ); /* WITH */ + testcase( i==50 ); /* OUTER */ + testcase( i==51 ); /* RELEASE */ + testcase( i==52 ); /* ATTACH */ + testcase( i==53 ); /* HAVING */ + testcase( i==54 ); /* GROUP */ + testcase( i==55 ); /* UPDATE */ + testcase( i==56 ); /* BEGIN */ + testcase( i==57 ); /* INNER */ + testcase( i==58 ); /* RECURSIVE */ + testcase( i==59 ); /* BETWEEN */ + testcase( i==60 ); /* NOTNULL */ + testcase( i==61 ); /* NOT */ + testcase( i==62 ); /* NO */ + testcase( i==63 ); /* NULL */ + testcase( i==64 ); /* LIKE */ + testcase( i==65 ); /* CASCADE */ + testcase( i==66 ); /* ASC */ + testcase( i==67 ); /* DELETE */ + testcase( i==68 ); /* CASE */ + testcase( i==69 ); /* COLLATE */ + testcase( i==70 ); /* CREATE */ + testcase( i==71 ); /* CURRENT_DATE */ + testcase( i==72 ); /* DETACH */ + testcase( i==73 ); /* IMMEDIATE */ + testcase( i==74 ); /* JOIN */ + testcase( i==75 ); /* INSERT */ + testcase( i==76 ); /* MATCH */ + testcase( i==77 ); /* PLAN */ + testcase( i==78 ); /* ANALYZE */ + testcase( i==79 ); /* PRAGMA */ + testcase( i==80 ); /* ABORT */ + testcase( i==81 ); /* VALUES */ + testcase( i==82 ); /* VIRTUAL */ + testcase( i==83 ); /* LIMIT */ + testcase( i==84 ); /* WHEN */ + testcase( i==85 ); /* WHERE */ + testcase( i==86 ); /* RENAME */ + testcase( i==87 ); /* AFTER */ + testcase( i==88 ); /* REPLACE */ + testcase( i==89 ); /* AND */ + testcase( i==90 ); /* DEFAULT */ + testcase( i==91 ); /* AUTOINCREMENT */ + testcase( i==92 ); /* TO */ + testcase( i==93 ); /* IN */ + testcase( i==94 ); /* CAST */ + testcase( i==95 ); /* COLUMN */ + testcase( i==96 ); /* COMMIT */ + testcase( i==97 ); /* CONFLICT */ + testcase( i==98 ); /* CROSS */ + testcase( i==99 ); /* CURRENT_TIMESTAMP */ + testcase( i==100 ); /* CURRENT_TIME */ + testcase( i==101 ); /* PRIMARY */ + testcase( i==102 ); /* DEFERRED */ + testcase( i==103 ); /* DISTINCT */ + testcase( i==104 ); /* IS */ + testcase( i==105 ); /* DROP */ + testcase( i==106 ); /* FAIL */ + testcase( i==107 ); /* FROM */ + testcase( i==108 ); /* FULL */ + testcase( i==109 ); /* GLOB */ + testcase( i==110 ); /* BY */ + testcase( i==111 ); /* IF */ + testcase( i==112 ); /* ISNULL */ + testcase( i==113 ); /* ORDER */ + testcase( i==114 ); /* RESTRICT */ + testcase( i==115 ); /* RIGHT */ + testcase( i==116 ); /* ROLLBACK */ + testcase( i==117 ); /* ROW */ + testcase( i==118 ); /* UNION */ + testcase( i==119 ); /* USING */ + testcase( i==120 ); /* VACUUM */ + testcase( i==121 ); /* VIEW */ + testcase( i==122 ); /* INITIALLY */ + testcase( i==123 ); /* ALL */ return aCode[i]; } } @@ -107076,7 +124126,7 @@ static int keywordCode(const char *z, int n){ SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ return keywordCode((char*)z, n); } -#define SQLITE_N_KEYWORD 121 +#define SQLITE_N_KEYWORD 124 /************** End of keywordhash.h *****************************************/ /************** Continuing where we left off in tokenize.c *******************/ @@ -107094,7 +124144,7 @@ SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ ** end result. ** ** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identfiers. But many SQL implementations do. +** middle of identifiers. But many SQL implementations do. ** SQLite will allow '$' in identifiers for compatibility. ** But the feature is undocumented. */ @@ -107119,6 +124169,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { }; #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif +SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } /* @@ -107140,7 +124191,6 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } case '-': { if( z[1]=='-' ){ - /* IMP: R-15891-05542 -- syntax diagram for comments */ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; @@ -107173,7 +124223,6 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_SLASH; return 1; } - /* IMP: R-15891-05542 -- syntax diagram for comments */ for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} if( c ) i++; *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ @@ -107289,6 +124338,12 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); testcase( z[0]=='9' ); *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif for(i=0; sqlite3Isdigit(z[i]); i++){} #ifndef SQLITE_OMIT_FLOATING_POINT if( z[i]=='.' ){ @@ -107322,24 +124377,15 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ for(i=1; sqlite3Isdigit(z[i]); i++){} return i; } - case '#': { - for(i=1; sqlite3Isdigit(z[i]); i++){} - if( i>1 ){ - /* Parameters of the form #NNN (where NNN is a number) are used - ** internally by sqlite3NestedParse. */ - *tokenType = TK_REGISTER; - return i; - } - /* Fall through into the next case if the '#' is not followed by - ** a digit. Try to match #AAAA where AAAA is a parameter name. */ - } #ifndef SQLITE_OMIT_TCL_VARIABLE case '$': #endif case '@': /* For compatibility with MS SQL Server */ + case '#': case ':': { int n = 0; - testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; for(i=1; (c=z[i])!=0; i++){ if( IdChar(c) ){ @@ -107413,14 +124459,14 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; - if( db->activeVdbeCnt==0 ){ + if( db->nVdbeActive==0 ){ db->u1.isInterrupted = 0; } pParse->rc = SQLITE_OK; pParse->zTail = zSql; i = 0; assert( pzErrMsg!=0 ); - pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); + pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ db->mallocFailed = 1; return SQLITE_NOMEM; @@ -107522,10 +124568,10 @@ abort_parse: sqlite3DeleteTable(db, pParse->pNewTable); } + if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); sqlite3DeleteTrigger(db, pParse->pNewTrigger); for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); sqlite3DbFree(db, pParse->azVar); - sqlite3DbFree(db, pParse->aAlias); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; pParse->pAinc = p->pNext; @@ -107615,7 +124661,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** a statement. ** ** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceeded by EXPLAIN and/or followed by +** statement, possibly preceded by EXPLAIN and/or followed by ** TEMP or TEMPORARY ** ** (5) TRIGGER We are in the middle of a trigger definition that must be @@ -107625,7 +124671,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** the end of a trigger definition. ** ** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger difinition. +** of a trigger definition. ** ** Transitions between states above are determined by tokens extracted ** from the input. The following tokens are significant: @@ -107668,7 +124714,7 @@ SQLITE_API int sqlite3_complete(const char *zSql){ }; #else /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simplier + ** used to detect the end of a statement is much simpler */ static const u8 trans[3][3] = { /* Token: */ @@ -107967,8 +125013,8 @@ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } */ SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } -/* IMPLEMENTATION-OF: R-54823-41343 The sqlite3_threadsafe() function returns -** zero if and only if SQLite was compiled mutexing code omitted due to +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to ** the SQLITE_THREADSAFE compile-time option being set to 0. */ SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } @@ -107992,6 +125038,15 @@ SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*, ...) = 0; */ SQLITE_API char *sqlite3_temp_directory = 0; +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_data_directory = 0; + /* ** Initialize SQLite. ** @@ -108024,8 +125079,11 @@ SQLITE_API char *sqlite3_temp_directory = 0; ** without blocking. */ SQLITE_API int sqlite3_initialize(void){ - sqlite3_mutex *pMaster; /* The main static mutex */ + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif #ifdef SQLITE_OMIT_WSD rc = sqlite3_wsd_init(4096, 24); @@ -108058,7 +125116,7 @@ SQLITE_API int sqlite3_initialize(void){ ** malloc subsystem - this implies that the allocation of a static ** mutex must not require support from the malloc subsystem. */ - pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(pMaster); sqlite3GlobalConfig.isMutexInit = 1; if( !sqlite3GlobalConfig.isMallocInit ){ @@ -108116,6 +125174,9 @@ SQLITE_API int sqlite3_initialize(void){ sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif } sqlite3GlobalConfig.inProgress = 0; } @@ -108152,6 +125213,16 @@ SQLITE_API int sqlite3_initialize(void){ #endif #endif + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } +#endif + return rc; } @@ -108165,6 +125236,10 @@ SQLITE_API int sqlite3_initialize(void){ */ SQLITE_API int sqlite3_shutdown(void){ if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif sqlite3_os_end(); sqlite3_reset_auto_extension(); sqlite3GlobalConfig.isInit = 0; @@ -108176,6 +125251,18 @@ SQLITE_API int sqlite3_shutdown(void){ if( sqlite3GlobalConfig.isMallocInit ){ sqlite3MallocEnd(); sqlite3GlobalConfig.isMallocInit = 0; + +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif } if( sqlite3GlobalConfig.isMutexInit ){ sqlite3MutexEnd(); @@ -108273,16 +125360,25 @@ SQLITE_API int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_PCACHE: { - /* Specify an alternative page cache implementation */ - sqlite3GlobalConfig.pcache = *va_arg(ap, sqlite3_pcache_methods*); + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; break; } - case SQLITE_CONFIG_GETPCACHE: { - if( sqlite3GlobalConfig.pcache.xInit==0 ){ + case SQLITE_CONFIG_PCACHE2: { + /* Specify an alternative page cache implementation */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ sqlite3PCacheSetDefault(); } - *va_arg(ap, sqlite3_pcache_methods*) = sqlite3GlobalConfig.pcache; + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; break; } @@ -108309,8 +125405,8 @@ SQLITE_API int sqlite3_config(int op, ...){ memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); }else{ /* The heap pointer is not NULL, then install one of the - ** mem5.c/mem3.c methods. If neither ENABLE_MEMSYS3 nor - ** ENABLE_MEMSYS5 is defined, return an error. + ** mem5.c/mem3.c methods. The enclosing #if guarantees at + ** least one of these methods is currently enabled. */ #ifdef SQLITE_ENABLE_MEMSYS3 sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); @@ -108329,7 +125425,7 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } - /* Record a pointer to the logger funcction and its first argument. + /* Record a pointer to the logger function and its first argument. ** The default is NULL. Logging is disabled if the function pointer is ** NULL. */ @@ -108344,11 +125440,50 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ case SQLITE_CONFIG_URI: { sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); break; } + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif + default: { rc = SQLITE_ERROR; break; @@ -108381,21 +125516,21 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ if( db->lookaside.bMalloced ){ sqlite3_free(db->lookaside.pStart); } - /* The size of a lookaside slot needs to be larger than a pointer - ** to be useful. + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; if( cnt<0 ) cnt = 0; if( sz==0 || cnt==0 ){ sz = 0; pStart = 0; }else if( pBuf==0 ){ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ sqlite3BeginBenignMalloc(); pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); + if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; }else{ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ pStart = pBuf; } db->lookaside.pStart = pStart; @@ -108415,7 +125550,8 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ db->lookaside.bEnabled = 1; db->lookaside.bMalloced = pBuf==0 ?1:0; }else{ - db->lookaside.pEnd = 0; + db->lookaside.pStart = db; + db->lookaside.pEnd = db; db->lookaside.bEnabled = 0; db->lookaside.bMalloced = 0; } @@ -108429,6 +125565,26 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ return db->mutex; } +/* +** Free up as much memory as we can from the given database +** connection. +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ + int i; + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + /* ** Configuration settings for an individual database connection */ @@ -108521,7 +125677,7 @@ static int binCollFunc( /* ** Another built-in collating sequence: NOCASE. ** -** This collating sequence is intended to be used for "case independant +** This collating sequence is intended to be used for "case independent ** comparison". SQLite's knowledge of upper and lower case equivalents ** extends only to the 26 characters used in the English language. ** @@ -108595,14 +125751,53 @@ static void functionDestroy(sqlite3 *db, FuncDef *p){ } } +/* +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. +*/ +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( db->aDb[i].pSchema ){ + HashElem *p; + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + } + } + } + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif +} + +/* +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. +*/ +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; +} + /* ** Close an existing SQLite database */ -SQLITE_API int sqlite3_close(sqlite3 *db){ - HashElem *i; /* Hash table iterator */ - int j; - +static int sqlite3Close(sqlite3 *db, int forceZombie){ if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ return SQLITE_OK; } if( !sqlite3SafetyCheckSickOrOk(db) ){ @@ -108610,10 +125805,10 @@ SQLITE_API int sqlite3_close(sqlite3 *db){ } sqlite3_mutex_enter(db->mutex); - /* Force xDestroy calls on all virtual tables */ - sqlite3ResetInternalSchema(db, -1); + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); - /* If a transaction is open, the ResetInternalSchema() call above + /* If a transaction is open, the disconnectAllVtab() call above ** will not have called the xDisconnect() method on any virtual ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() ** call will do so. We need to do this before the check for active @@ -108622,31 +125817,93 @@ SQLITE_API int sqlite3_close(sqlite3 *db){ */ sqlite3VtabRollback(db); - /* If there are any outstanding VMs, return SQLITE_BUSY. */ - if( db->pVdbe ){ - sqlite3Error(db, SQLITE_BUSY, - "unable to close due to unfinalised statements"); + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); sqlite3_mutex_leave(db->mutex); return SQLITE_BUSY; } - assert( sqlite3SafetyCheckSickOrOk(db) ); - for(j=0; jnDb; j++){ - Btree *pBt = db->aDb[j].pBt; - if( pBt && sqlite3BtreeIsInBackup(pBt) ){ - sqlite3Error(db, SQLITE_BUSY, - "unable to close due to unfinished backup operation"); - sqlite3_mutex_leave(db->mutex); - return SQLITE_BUSY; - } +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); } +#endif + + /* Convert the connection into a zombie and then close it. + */ + db->magic = SQLITE_MAGIC_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; +} + +/* +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection option if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. +*/ +SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + + +/* +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. +*/ +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. + */ + if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; + } + + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); /* Free any outstanding Savepoint structures. */ sqlite3CloseSavepoints(db); + /* Close all database connections */ for(j=0; jnDb; j++){ struct Db *pDb = &db->aDb[j]; if( pDb->pBt ){ + if( pDb->pSchema ){ + /* Must clear the KeyInfo cache. See ticket [e4a18565a36884b00edf] */ + sqlite3BtreeEnter(pDb->pBt); + for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } + sqlite3BtreeLeave(pDb->pBt); + } sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; if( j!=1 ){ @@ -108654,15 +125911,22 @@ SQLITE_API int sqlite3_close(sqlite3 *db){ } } } - sqlite3ResetInternalSchema(db, -1); + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); + + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); /* Tell the code in notify.c that the connection no longer holds any ** locks and does not require any further unlock-notify callbacks. */ sqlite3ConnectionClosed(db); - assert( db->nDb<=2 ); - assert( db->aDb==db->aDbStatic ); for(j=0; jaFunc.a); j++){ FuncDef *pNext, *pHash, *p; for(p=db->aFunc.a[j]; p; p=pHash){ @@ -108697,11 +125961,13 @@ SQLITE_API int sqlite3_close(sqlite3 *db){ sqlite3HashClear(&db->aModule); #endif - sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ - if( db->pErr ){ - sqlite3ValueFree(db->pErr); - } + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif db->magic = SQLITE_MAGIC_ERROR; @@ -108720,36 +125986,53 @@ SQLITE_API int sqlite3_close(sqlite3 *db){ sqlite3_free(db->lookaside.pStart); } sqlite3_free(db); - return SQLITE_OK; } /* -** Rollback all database files. +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any write cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ -SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; + int schemaChange; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; + for(i=0; inDb; i++){ - if( db->aDb[i].pBt ){ - if( sqlite3BtreeIsInTrans(db->aDb[i].pBt) ){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(db->aDb[i].pBt); - db->aDb[i].inTrans = 0; + sqlite3BtreeRollback(p, tripCode, !schemaChange); } } sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( db->flags&SQLITE_InternChanges ){ + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ sqlite3ExpirePreparedStatements(db); - sqlite3ResetInternalSchema(db, -1); + sqlite3ResetAllSchemasOfConnection(db); } + sqlite3BtreeLeaveAll(db); /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~SQLITE_DeferFKs; /* If one has been configured, invoke the rollback-hook callback */ if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ @@ -108757,6 +126040,115 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db){ } } +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) || defined(SQLITE_TEST) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + /* ** Return a static string that describes the kind of error specified in the ** argument. @@ -108791,12 +126183,21 @@ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ /* SQLITE_RANGE */ "bind or column index out of range", /* SQLITE_NOTADB */ "file is encrypted or is not a database", }; - rc &= 0xff; - if( ALWAYS(rc>=0) && rc<(int)(sizeof(aMsg)/sizeof(aMsg[0])) && aMsg[rc]!=0 ){ - return aMsg[rc]; - }else{ - return "unknown error"; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcbusyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; db->busyHandler.nBusy = 0; + db->busyTimeout = 0; sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -108895,7 +126297,7 @@ SQLITE_API void sqlite3_progress_handler( sqlite3_mutex_enter(db->mutex); if( nOps>0 ){ db->xProgress = xProgress; - db->nProgressOps = nOps; + db->nProgressOps = (unsigned)nOps; db->pProgressArg = pArg; }else{ db->xProgress = 0; @@ -108913,8 +126315,8 @@ SQLITE_API void sqlite3_progress_handler( */ SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ if( ms>0 ){ - db->busyTimeout = ms; sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; }else{ sqlite3_busy_handler(db, 0, 0); } @@ -108948,6 +126350,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( ){ FuncDef *p; int nName; + int extraFlags; assert( sqlite3_mutex_held(db->mutex) ); if( zFunctionName==0 || @@ -108958,6 +126361,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( (255<(nName = sqlite3Strlen30( zFunctionName))) ){ return SQLITE_MISUSE_BKPT; } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + extraFlags = enc & SQLITE_DETERMINISTIC; + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); #ifndef SQLITE_OMIT_UTF16 /* If SQLITE_UTF16 is specified as the encoding type, transform this @@ -108971,10 +126378,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( enc = SQLITE_UTF16NATIVE; }else if( enc==SQLITE_ANY ){ int rc; - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8, + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, pUserData, xFunc, xStep, xFinal, pDestructor); if( rc==SQLITE_OK ){ - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, pUserData, xFunc, xStep, xFinal, pDestructor); } if( rc!=SQLITE_OK ){ @@ -108992,9 +126399,9 @@ SQLITE_PRIVATE int sqlite3CreateFunc( ** operation to continue but invalidate all precompiled statements. */ p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); - if( p && p->iPrefEnc==enc && p->nArg==nArg ){ - if( db->activeVdbeCnt ){ - sqlite3Error(db, SQLITE_BUSY, + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); assert( !db->mallocFailed ); return SQLITE_BUSY; @@ -109017,7 +126424,8 @@ SQLITE_PRIVATE int sqlite3CreateFunc( pDestructor->nRef++; } p->pDestructor = pDestructor; - p->flags = 0; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); p->xFunc = xFunc; p->xStep = xStep; p->xFinalize = xFinal; @@ -109122,13 +126530,13 @@ SQLITE_API int sqlite3_overload_function( int nArg ){ int nName = sqlite3Strlen30(zName); - int rc; + int rc = SQLITE_OK; sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ - sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0, 0); + rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, + 0, sqlite3InvalidFunction, 0, 0, 0); } - rc = sqlite3ApiExit(db, SQLITE_OK); + rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -109174,9 +126582,8 @@ SQLITE_API void *sqlite3_profile( } #endif /* SQLITE_OMIT_TRACE */ -/*** EXPERIMENTAL *** -** -** Register a function to be invoked when a transaction comments. +/* +** Register a function to be invoked when a transaction commits. ** If the invoked function returns non-zero, then the commit becomes a ** rollback. */ @@ -109332,10 +126739,10 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( } if( iDb<0 ){ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -109448,6 +126855,7 @@ SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ if( db->mallocFailed ){ z = sqlite3ErrStr(SQLITE_NOMEM); }else{ + testcase( db->pErr==0 ); z = (char*)sqlite3_value_text(db->pErr); assert( !db->mallocFailed ); if( z==0 ){ @@ -109489,8 +126897,7 @@ SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ }else{ z = sqlite3_value_text16(db->pErr); if( z==0 ){ - sqlite3ValueSetStr(db->pErr, -1, sqlite3ErrStr(db->errCode), - SQLITE_UTF8, SQLITE_STATIC); + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); z = sqlite3_value_text16(db->pErr); } /* A malloc() may have failed within the call to sqlite3_value_text16() @@ -109528,6 +126935,41 @@ SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ return db->errCode; } +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* +** Invalidate all cached KeyInfo objects for database connection "db" +*/ +static void invalidateCachedKeyInfo(sqlite3 *db){ + Db *pDb; /* A single database */ + int iDb; /* The database index number */ + HashElem *k; /* For looping over tables in pDb */ + Table *pTab; /* A table in the database */ + Index *pIdx; /* Each index */ + + for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ + if( pDb->pBt==0 ) continue; + sqlite3BtreeEnter(pDb->pBt); + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pKeyInfo && pIdx->pKeyInfo->db==db ){ + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } + } + } + sqlite3BtreeLeave(pDb->pBt); + } +} + /* ** Create a new collating function for database "db". The name is zName ** and the encoding is enc. @@ -109536,14 +126978,12 @@ static int createCollation( sqlite3* db, const char *zName, u8 enc, - u8 collType, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDel)(void*) ){ CollSeq *pColl; int enc2; - int nName = sqlite3Strlen30(zName); assert( sqlite3_mutex_held(db->mutex) ); @@ -109567,12 +127007,13 @@ static int createCollation( */ pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); if( pColl && pColl->xCmp ){ - if( db->activeVdbeCnt ){ - sqlite3Error(db, SQLITE_BUSY, + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify collation sequence due to active statements"); return SQLITE_BUSY; } sqlite3ExpirePreparedStatements(db); + invalidateCachedKeyInfo(db); /* If collation sequence pColl was created directly by a call to ** sqlite3_create_collation, and not generated by synthCollSeq(), @@ -109581,7 +127022,7 @@ static int createCollation( ** to be called. */ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); int j; for(j=0; j<3; j++){ CollSeq *p = &aColl[j]; @@ -109601,8 +127042,7 @@ static int createCollation( pColl->pUser = pCtx; pColl->xDel = xDel; pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - pColl->type = collType; - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); return SQLITE_OK; } @@ -109622,8 +127062,9 @@ static const int aHardLimit[] = { SQLITE_MAX_FUNCTION_ARG, SQLITE_MAX_ATTACHED, SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, }; /* @@ -109647,8 +127088,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 # error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 #endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>62 -# error SQLITE_MAX_ATTACHED must be between 0 and 62 +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 #endif #if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 # error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 @@ -109659,6 +127100,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_TRIGGER_DEPTH<1 # error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 #endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif /* @@ -109692,7 +127136,8 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ SQLITE_MAX_LIKE_PATTERN_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -109750,7 +127195,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( assert( *pzErrMsg==0 ); if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) - && nUri>=5 && memcmp(zUri, "file:", 5)==0 + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ @@ -109766,20 +127211,20 @@ SQLITE_PRIVATE int sqlite3ParseUri( zFile = sqlite3_malloc(nByte); if( !zFile ) return SQLITE_NOMEM; + iIn = 5; +#ifndef SQLITE_ALLOW_URI_AUTHORITY /* Discard the scheme and authority segments of the URI. */ if( zUri[5]=='/' && zUri[6]=='/' ){ iIn = 7; while( zUri[iIn] && zUri[iIn]!='/' ) iIn++; - if( iIn!=7 && (iIn!=16 || memcmp("localhost", &zUri[7], 9)) ){ *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s", iIn-7, &zUri[7]); rc = SQLITE_ERROR; goto parse_uri_out; } - }else{ - iIn = 5; } +#endif /* Copy the filename and any query parameters into the zFile buffer. ** Decode %HH escape codes along the way. @@ -109877,10 +127322,12 @@ SQLITE_PRIVATE int sqlite3ParseUri( { "ro", SQLITE_OPEN_READONLY }, { "rw", SQLITE_OPEN_READWRITE }, { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, { 0, 0 } }; - mask = SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; aMode = aOpenMode; limit = mask & flags; zModeType = "access"; @@ -109901,7 +127348,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( rc = SQLITE_ERROR; goto parse_uri_out; } - if( mode>limit ){ + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", zModeType, zVal); rc = SQLITE_PERM; @@ -109920,6 +127367,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( memcpy(zFile, zUri, nUri); zFile[nUri] = '\0'; zFile[nUri+1] = '\0'; + flags &= ~SQLITE_OPEN_URI; } *ppVfs = sqlite3_vfs_find(zVfs); @@ -109977,7 +127425,9 @@ static int openDatabase( testcase( (1<<(flags&7))==0x02 ); /* READONLY */ testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT; + if( ((1<<(flags&7)) & 0x46)==0 ){ + return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ + } if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; @@ -110036,10 +127486,16 @@ static int openDatabase( assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; - db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger + db->nMaxSorterMmap = 0x7FFFFFFF; + db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex +#endif #if SQLITE_DEFAULT_FILE_FORMAT<4 | SQLITE_LegacyFileFmt #endif @@ -110062,14 +127518,10 @@ static int openDatabase( ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. */ - createCollation(db, "BINARY", SQLITE_UTF8, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16BE, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16LE, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, SQLITE_COLL_USER, (void*)1, - binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); if( db->mallocFailed ){ goto opendb_out; } @@ -110077,15 +127529,14 @@ static int openDatabase( assert( db->pDfltColl!=0 ); /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, SQLITE_COLL_NOCASE, 0, - nocaseCollatingFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); /* Parse the filename/URI argument. */ db->openFlags = flags; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; - sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); sqlite3_free(zErrMsg); goto opendb_out; } @@ -110097,13 +127548,14 @@ static int openDatabase( if( rc==SQLITE_IOERR_NOMEM ){ rc = SQLITE_NOMEM; } - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); goto opendb_out; } + sqlite3BtreeEnter(db->aDb[0].pBt); db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - /* The default safety_level for the main database is 'full'; for the temp ** database it is 'NONE'. This matches the pager layer defaults. */ @@ -110121,16 +127573,19 @@ static int openDatabase( ** database schema yet. This is delayed until the first time the database ** is accessed. */ - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); sqlite3RegisterBuiltinFunctions(db); /* Load automatic extensions - extensions that have been registered ** using the sqlite3_automatic_extension() API. */ - sqlite3AutoLoadExtensions(db); rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; + if( rc==SQLITE_OK ){ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; + } } #ifdef SQLITE_ENABLE_FTS1 @@ -110165,8 +127620,6 @@ static int openDatabase( } #endif - sqlite3Error(db, rc, 0); - /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking ** mode. Doing nothing at all also makes NORMAL the default. @@ -110177,6 +127630,8 @@ static int openDatabase( SQLITE_DEFAULT_LOCKING_MODE); #endif + if( rc ) sqlite3Error(db, rc); + /* Enable the lookaside-malloc subsystem */ setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, sqlite3GlobalConfig.nLookaside); @@ -110190,6 +127645,7 @@ opendb_out: sqlite3_mutex_leave(db->mutex); } rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); if( rc==SQLITE_NOMEM ){ sqlite3_close(db); db = 0; @@ -110197,6 +127653,13 @@ opendb_out: db->magic = SQLITE_MAGIC_SICK; } *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif return sqlite3ApiExit(0, rc); } @@ -110270,7 +127733,7 @@ SQLITE_API int sqlite3_create_collation( int rc; sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -110290,7 +127753,7 @@ SQLITE_API int sqlite3_create_collation_v2( int rc; sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, xDel); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -110313,7 +127776,7 @@ SQLITE_API int sqlite3_create_collation16( assert( !db->mallocFailed ); zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); if( zName8 ){ - rc = createCollation(db, zName8, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); sqlite3DbFree(db, zName8); } rc = sqlite3ApiExit(db, rc); @@ -110373,17 +127836,15 @@ SQLITE_API int sqlite3_global_recover(void){ ** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on ** by default. Autocommit is disabled by a BEGIN statement and reenabled ** by the next COMMIT or ROLLBACK. -** -******* THIS IS AN EXPERIMENTAL API AND IS SUBJECT TO CHANGE ****** */ SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ return db->autoCommit; } /* -** The following routines are subtitutes for constants SQLITE_CORRUPT, +** The following routines are substitutes for constants SQLITE_CORRUPT, ** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error -** constants. They server two purposes: +** constants. They serve two purposes: ** ** 1. Serve as a convenient place to set a breakpoint in a debugger ** to detect when version error conditions occurs. @@ -110502,7 +127963,7 @@ SQLITE_API int sqlite3_table_column_metadata( zDataType = pCol->zType; zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; - primarykey = pCol->isPrimKey!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; }else{ zDataType = "INTEGER"; @@ -110531,7 +127992,7 @@ error_out: zColumnName); rc = SQLITE_ERROR; } - sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); sqlite3DbFree(db, zErrMsg); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -110570,35 +128031,27 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ */ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ int rc = SQLITE_ERROR; - int iDb; + Btree *pBtree; + sqlite3_mutex_enter(db->mutex); - if( zDbName==0 ){ - iDb = 0; - }else{ - for(iDb=0; iDbnDb; iDb++){ - if( strcmp(db->aDb[iDb].zName, zDbName)==0 ) break; - } - } - if( iDbnDb ){ - Btree *pBtree = db->aDb[iDb].pBt; - if( pBtree ){ - Pager *pPager; - sqlite3_file *fd; - sqlite3BtreeEnter(pBtree); - pPager = sqlite3BtreePager(pBtree); - assert( pPager!=0 ); - fd = sqlite3PagerFile(pPager); - assert( fd!=0 ); - if( op==SQLITE_FCNTL_FILE_POINTER ){ - *(sqlite3_file**)pArg = fd; - rc = SQLITE_OK; - }else if( fd->pMethods ){ - rc = sqlite3OsFileControl(fd, op, pArg); - }else{ - rc = SQLITE_NOTFOUND; - } - sqlite3BtreeLeave(pBtree); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + }else{ + rc = SQLITE_NOTFOUND; } + sqlite3BtreeLeave(pBtree); } sqlite3_mutex_leave(db->mutex); return rc; @@ -110638,7 +128091,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** to the xRandomness method of the default VFS. */ case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3PrngResetState(); + sqlite3_randomness(0,0); break; } @@ -110657,6 +128110,28 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + /* ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) ** @@ -110683,7 +128158,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in ** an incompatible database file format. Changing the PENDING byte ** while any database connection is open results in undefined and - ** dileterious behavior. + ** deleterious behavior. */ case SQLITE_TESTCTRL_PENDING_BYTE: { rc = PENDING_BYTE; @@ -110748,6 +128223,22 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) ** ** Set the nReserve size to N for the main database on the database @@ -110773,8 +128264,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); break; } @@ -110796,15 +128286,6 @@ SQLITE_API int sqlite3_test_control(int op, ...){ } #endif - /* sqlite3_test_control(SQLITE_TESTCTRL_PGHDRSZ) - ** - ** Return the size of a pcache header in bytes. - */ - case SQLITE_TESTCTRL_PGHDRSZ: { - rc = sizeof(PgHdr); - break; - } - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); ** ** Pass pFree into sqlite3ScratchFree(). @@ -110832,6 +128313,50 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } + + + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,int,u8,u8); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ @@ -110850,6 +128375,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** returns a NULL pointer. */ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + if( zFilename==0 ) return 0; zFilename += sqlite3Strlen30(zFilename) + 1; while( zFilename[0] ){ int x = strcmp(zFilename, zParam); @@ -110860,6 +128386,64 @@ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char * return 0; } +/* +** Return a boolean value for a query parameter. +*/ +SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} + +/* +** Return a 64-bit integer value for a query parameter. +*/ +SQLITE_API sqlite3_int64 sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ + bDflt = v; + } + return bDflt; +} + +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int i; + for(i=0; inDb; i++){ + if( db->aDb[i].pBt + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + ){ + return db->aDb[i].pBt; + } + } + return 0; +} + +/* +** Return the filename of the database associated with a database +** connection. +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} + +/* +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; +} + /************** End of main.c ************************************************/ /************** Begin file notify.c ******************************************/ /* @@ -111046,7 +128630,7 @@ SQLITE_API int sqlite3_unlock_notify( leaveMutex(); assert( !db->mallocFailed ); - sqlite3Error(db, rc, (rc?"database is deadlocked":0)); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); sqlite3_mutex_leave(db->mutex); return rc; } @@ -111267,7 +128851,7 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** A doclist is stored like this: ** ** array { -** varint docid; +** varint docid; (delta from previous doclist) ** array { (position list for column 0) ** varint position; (2 more than the delta from previous position) ** } @@ -111298,8 +128882,8 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** at D signals the start of a new column; the 1 at E indicates that the ** new column is column number 1. There are two positions at 12 and 45 ** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the next docid. It has one position 72 (72-2) and then -** terminates with the 0 at K. +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. ** ** A "position-list" is the list of positions for multiple columns for ** a single docid. A "column-list" is the set of positions for a single @@ -111483,10 +129067,6 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** will eventually overtake the earlier data and knock it out. The ** query logic likewise merges doclists so that newer data knocks out ** older data. -** -** TODO(shess) Provide a VACUUM type operation to clear out all -** deletions and duplications. This would basically be a forced merge -** into a single segment. */ /************** Include fts3Int.h in the middle of fts3.c ********************/ @@ -111520,7 +129100,13 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ # define SQLITE_ENABLE_FTS3 #endif -#ifdef SQLITE_ENABLE_FTS3 +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT3 +#endif + /************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ /************** Begin file fts3_tokenizer.h **********************************/ /* @@ -111576,7 +129162,7 @@ typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; struct sqlite3_tokenizer_module { /* - ** Structure version. Should always be set to 0. + ** Structure version. Should always be set to 0 or 1. */ int iVersion; @@ -111594,7 +129180,7 @@ struct sqlite3_tokenizer_module { ** This method should return either SQLITE_OK (0), or an SQLite error ** code. If SQLITE_OK is returned, then *ppTokenizer should be set ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialised by + ** sqlite3_tokenizer.pModule variable should not be initialized by ** this callback. The caller will do so. */ int (*xCreate)( @@ -111657,6 +129243,15 @@ struct sqlite3_tokenizer_module { int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ int *piPosition /* OUT: Number of tokens returned before this one */ ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); }; struct sqlite3_tokenizer { @@ -111690,7 +129285,7 @@ int fts3_term_cnt(int iTerm, int iCol); ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. We've modified it slightly to serve as a standalone ** hash table implementation for the full-text indexing module. ** @@ -111795,6 +129390,18 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi /************** End of fts3_hash.h *******************************************/ /************** Continuing where we left off in fts3Int.h ********************/ +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single @@ -111823,6 +129430,9 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi #ifndef MIN # define MIN(x,y) ((x)<(y)?(x):(y)) #endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif /* ** Maximum length of a varint encoded integer. The varint format is different @@ -111877,7 +129487,7 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi # define NEVER(X) (0) #else # define ALWAYS(x) (x) -# define NEVER(X) (x) +# define NEVER(x) (x) #endif /* @@ -111887,6 +129497,7 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ /* ** Macro used to suppress compiler warnings for unused parameters. @@ -111913,6 +129524,13 @@ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ #endif /* SQLITE_AMALGAMATION */ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; @@ -111939,37 +129557,47 @@ struct Fts3Table { const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[27]; + sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ - u8 bHasStat; /* True if %_stat table exists */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* TODO: Fix the first paragraph of this comment. - ** - ** The following hash table is used to buffer pending index updates during - ** transactions. Variable nPendingData estimates the memory size of the - ** pending data, including hash table overhead, but not malloc overhead. - ** When nPendingData exceeds nMaxPendingData, the buffer is flushed - ** automatically. Variable iPrevDocid is the docid of the most recently - ** inserted record. + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. ** ** A single FTS4 table may have multiple full-text indexes. For each index ** there is an entry in the aIndex[] array. Index 0 is an index of all the ** terms that appear in the document set. Each subsequent index in aIndex[] ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. */ int nIndex; /* Size of aIndex[] */ struct Fts3Index { @@ -111979,16 +129607,23 @@ struct Fts3Table { int nMaxPendingData; /* Max pending data before flush to disk */ int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These - ** values do not contribution to the FTS computation; they are used for - ** verifying the SQLite core. + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. */ int inTransaction; /* True after xBegin but before xCommit/xRollback */ int mxSavepoint; /* Largest valid xSavepoint integer */ #endif + +#ifdef SQLITE_TEST + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; +#endif }; /* @@ -112003,6 +129638,7 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ int nPhrase; /* Number of matchable phrases in query */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ @@ -112013,7 +129649,8 @@ struct Fts3Cursor { int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ sqlite3_int64 nDoc; /* Documents in table */ - + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ int nMatchinfo; /* Number of elements in aMatchinfo[] */ @@ -112043,6 +129680,15 @@ struct Fts3Cursor { #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ struct Fts3Doclist { char *aAll; /* Array containing doclist (or NULL) */ @@ -112053,7 +129699,7 @@ struct Fts3Doclist { int bFreeList; /* True if pList should be sqlite3_free()d */ char *pList; /* Pointer to position list following iDocid */ int nList; /* Length of position list */ -} doclist; +}; /* ** A "phrase" is a sequence of one or more tokens that must match in @@ -112065,6 +129711,7 @@ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are @@ -112148,23 +129795,33 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sql SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64, +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( Fts3Table*,int,const char*,int,int,Fts3SegReader**); SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); +#ifndef SQLITE_DISABLE_FTS4_DEFERRED SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 @@ -112174,8 +129831,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Ft SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -112183,6 +129840,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 #define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 /* Type passed as 4th argument to SegmentReaderIterate() */ struct Fts3SegFilter { @@ -112215,6 +129873,12 @@ struct Fts3MultiSegReader { int nDoclist; /* Size of aDoclist[] in bytes */ }; +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + /* fts3.c */ SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); @@ -112222,8 +129886,9 @@ SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); - SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); /* fts3_tokenizer.c */ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); @@ -112241,8 +129906,8 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const ch SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ -SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, - char **, int, int, const char *, int, Fts3Expr ** +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** ); SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST @@ -112250,33 +129915,34 @@ SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); #endif +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + /* fts3_aux.c */ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); -SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ -); - SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); -SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); -SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); - SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); -SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); -#endif /* SQLITE_ENABLE_FTS3 */ +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); +#endif + +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ /************** End of fts3Int.h *********************************************/ @@ -112287,11 +129953,22 @@ SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, in # define SQLITE_CORE 1 #endif +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -112309,21 +129986,37 @@ SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ return (int) (q - (unsigned char *)p); } +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*ptr++) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + /* ** Read a 64-bit variable-length integer from memory starting at p[0]. ** Return the number of bytes read, or 0 on error. ** The value is stored in *v. */ SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const unsigned char *q = (const unsigned char *) p; - sqlite_uint64 x = 0, y = 1; - while( (*q&0x80)==0x80 && q-(unsigned char *)pzSegmentsTbl); sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); @@ -112482,16 +130188,19 @@ static void fts3DbExec( ** The xDestroy() virtual table method. */ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - int rc = SQLITE_OK; /* Return code */ Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ /* Drop the shadow tables */ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName); + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); /* If everything has worked, invoke fts3DisconnectMethod() to free the ** memory associated with the Fts3Table structure and return SQLITE_OK. @@ -112516,7 +130225,9 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ int rc; /* Return code */ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ + const char *zLanguageid; + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Create a list of user columns for the virtual table */ @@ -112527,7 +130238,8 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ /* Create the whole "CREATE TABLE" statement to pass to SQLite */ zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid ); if( !zCols || !zSql ){ rc = SQLITE_NOMEM; @@ -112541,6 +130253,18 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ } } +/* +** Create the %_stat table if it does not already exist. +*/ +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + /* ** Create the backing store tables (%_content, %_segments and %_segdir) ** required by the FTS3 table passed as the only argument. This is done @@ -112553,23 +130277,31 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ static int fts3CreateTables(Fts3Table *p){ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ - char *zContentCols; /* Columns of %_content table */ sqlite3 *db = p->db; /* The database connection */ - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); /* Create other tables */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", @@ -112593,11 +130325,9 @@ static int fts3CreateTables(Fts3Table *p){ p->zDb, p->zName ); } + assert( p->bHasStat==p->bFts4 ); if( p->bHasStat ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); + sqlite3Fts3CreateStatTable(&rc, p); } return rc; } @@ -112679,6 +130409,7 @@ static void fts3Appendf( char *z; va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); if( z && *pz ){ char *z2 = sqlite3_mprintf("%s%s", *pz, z); sqlite3_free(z); @@ -112703,7 +130434,7 @@ static void fts3Appendf( static char *fts3QuoteId(char const *zInput){ int nRet; char *zRet; - nRet = 2 + strlen(zInput)*2 + 1; + nRet = 2 + (int)strlen(zInput)*2 + 1; zRet = sqlite3_malloc(nRet); if( zRet ){ int i; @@ -112720,8 +130451,8 @@ static char *fts3QuoteId(char const *zInput){ } /* -** Return a list of comma separated SQL expressions that could be used -** in a SELECT statement such as the following: +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: ** ** SELECT FROM %_content AS x ... ** @@ -112732,7 +130463,7 @@ static char *fts3QuoteId(char const *zInput){ ** table has the three user-defined columns "a", "b", and "c", the following ** string is returned: ** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. @@ -112748,16 +130479,34 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zFunction; int i; - if( !zFunc ){ - zFunction = ""; + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); }else{ - zFree = zFunction = fts3QuoteId(zFunc); + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - sqlite3_free(zFree); + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); return zRet; } @@ -112796,13 +130545,30 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } sqlite3_free(zFree); return zRet; } +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ static int fts3GobbleInt(const char **pp, int *pnOut){ - const char *p = *pp; - int nInt = 0; + const char *p; /* Iterator pointer */ + int nInt = 0; /* Output value */ + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ nInt = nInt * 10 + (p[0] - '0'); } @@ -112812,15 +130578,30 @@ static int fts3GobbleInt(const char **pp, int *pnOut){ return SQLITE_OK; } - +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ static int fts3PrefixParameter( const char *zParam, /* ABC in prefix=ABC parameter to parse */ int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ - struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ ){ - struct Fts3Index *aIndex; - int nIndex = 1; + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ if( zParam && zParam[0] ){ const char *p; @@ -112831,7 +130612,7 @@ static int fts3PrefixParameter( } aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = *apFree = aIndex; + *apIndex = aIndex; *pnIndex = nIndex; if( !aIndex ){ return SQLITE_NOMEM; @@ -112852,6 +130633,91 @@ static int fts3PrefixParameter( return SQLITE_OK; } +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr /* OUT: Bytes of string content */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + int nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i MATCHINFO */ - { "prefix", 6, 0 }, /* 1 -> PREFIX */ - { "compress", 8, 0 }, /* 2 -> COMPRESS */ - { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ - { "order", 5, 0 } /* 4 -> ORDER */ + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ }; int iOpt; @@ -112992,13 +130873,31 @@ static int fts3InitVtab( case 4: /* ORDER */ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) ){ *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); rc = SQLITE_ERROR; } bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); @@ -113011,6 +130910,39 @@ static int fts3InitVtab( aCol[nCol++] = z; } } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jnMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bFts4 = isFts4; p->bDescIdx = bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -113064,9 +131003,10 @@ static int fts3InitVtab( for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->aIndex[nIndex]; + zCsr = (char *)&p->abNotindexed[nCol]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -113087,7 +131027,28 @@ static int fts3InitVtab( assert( zCsr <= &((char *)p)[nByte] ); } - if( (zCompress==0)!=(zUncompress==0) ){ + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; ibHasStat = 2; + } + /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database. */ fts3DatabasePageSize(&rc, p); @@ -113113,10 +131081,14 @@ static int fts3InitVtab( fts3_init_out: sqlite3_free(zPrefix); - sqlite3_free(aFree); + sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; iestimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: @@ -113168,20 +131153,39 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ int i; /* Iterator variable */ int iCons = -1; /* Index of constraint to use */ + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + /* By default use a full table scan. This is an expensive option, ** so search through the constraints to see if a more efficient ** strategy is possible. */ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 500000; + pInfo->estimatedCost = 5000000; for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ) continue; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ - && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) - ){ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ pInfo->idxNum = FTS3_DOCID_SEARCH; pInfo->estimatedCost = 1.0; iCons = i; @@ -113202,14 +131206,47 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; pInfo->estimatedCost = 2.0; iCons = i; - break; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } } } + iIdx = 1; if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = 1; + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; pInfo->aConstraintUsage[iCons].omit = 1; } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } /* Regardless of the strategy selected, FTS can deliver rows in rowid (or ** docid) order. Both ascending and descending are possible. @@ -113267,35 +131304,64 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + *ppStmt = pCsr->pStmt; + return rc; +} + /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return ** SQLITE_OK on success. */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - int rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK ){ - /* If no row was found and no error has occured, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. - */ - rc = SQLITE_CORRUPT_VTAB; + sqlite3_stmt *pStmt = 0; + + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } } - pCsr->isEof = 1; - if( pContext ){ - sqlite3_result_error_code(pContext, rc); - } - return rc; } - }else{ - return SQLITE_OK; } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; } /* @@ -113345,7 +131411,7 @@ static int fts3ScanInteriorNode( zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); if( zCsr>zEnd ){ - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } while( zCsrzEnd ){ - rc = SQLITE_CORRUPT_VTAB; + rc = FTS_CORRUPT_VTAB; goto finish_scan; } if( nPrefix+nSuffix>nAlloc ){ @@ -113376,6 +131442,7 @@ static int fts3ScanInteriorNode( } zBuffer = zNew; } + assert( zBuffer ); memcpy(&zBuffer[nPrefix], zCsr, nSuffix); nBuffer = nPrefix + nSuffix; zCsr += nSuffix; @@ -113447,7 +131514,7 @@ static int fts3SelectLeaf( assert( piLeaf || piLeaf2 ); - sqlite3Fts3GetVarint32(zNode, &iHeight); + fts3GetVarint32(zNode, &iHeight); rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); @@ -113649,11 +131716,11 @@ static void fts3PoslistMerge( int iCol1; /* The current column index in pp1 */ int iCol2; /* The current column index in pp2 */ - if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); + if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; else iCol1 = 0; - if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); + if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; else iCol2 = 0; @@ -113704,8 +131771,6 @@ static void fts3PoslistMerge( } /* -** nToken==1 searches for adjacent positions. -** ** This function is used to merge two position lists into one. When it is ** called, *pp1 and *pp2 must both point to position lists. A position-list is ** the part of a doclist that follows each document id. For example, if a row @@ -113725,6 +131790,8 @@ static void fts3PoslistMerge( ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. ** when the *pp1 token appears before the *pp2 token, but not more than nToken ** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. */ static int fts3PoslistPhraseMerge( char **pp, /* IN/OUT: Preallocated output buffer */ @@ -113734,7 +131801,7 @@ static int fts3PoslistPhraseMerge( char **pp1, /* IN/OUT: Left input list */ char **pp2 /* IN/OUT: Right input list */ ){ - char *p = (pp ? *pp : 0); + char *p = *pp; char *p1 = *pp1; char *p2 = *pp2; int iCol1 = 0; @@ -113743,14 +131810,14 @@ static int fts3PoslistPhraseMerge( /* Never set both isSaveLeft and isExact for the same invocation. */ assert( isSaveLeft==0 || isExact==0 ); - assert( *p1!=0 && *p2!=0 ); + assert( p!=0 && *p1!=0 && *p2!=0 ); if( *p1==POS_COLUMN ){ p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); } if( *p2==POS_COLUMN ){ p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } while( 1 ){ @@ -113760,7 +131827,7 @@ static int fts3PoslistPhraseMerge( sqlite3_int64 iPos1 = 0; sqlite3_int64 iPos2 = 0; - if( pp && iCol1 ){ + if( iCol1 ){ *p++ = POS_COLUMN; p += sqlite3Fts3PutVarint(p, iCol1); } @@ -113775,16 +131842,10 @@ static int fts3PoslistPhraseMerge( || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) ){ sqlite3_int64 iSave; - if( !pp ){ - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - return 1; - } iSave = isSaveLeft ? iPos1 : iPos2; fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; pSave = 0; + assert( p ); } if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ if( (*p2&0xFE)==0 ) break; @@ -113806,9 +131867,9 @@ static int fts3PoslistPhraseMerge( if( 0==*p1 || 0==*p2 ) break; p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } /* Advance pointer p1 or p2 (whichever corresponds to the smaller of @@ -113820,12 +131881,12 @@ static int fts3PoslistPhraseMerge( fts3ColumnlistCopy(0, &p1); if( 0==*p1 ) break; p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); }else{ fts3ColumnlistCopy(0, &p2); if( 0==*p2 ) break; p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } } @@ -113833,7 +131894,7 @@ static int fts3PoslistPhraseMerge( fts3PoslistCopy(0, &p1); *pp1 = p1; *pp2 = p2; - if( !pp || *pp==p ){ + if( *pp==p ){ return 0; } *p++ = 0x00; @@ -113891,22 +131952,34 @@ static int fts3PoslistNearMerge( } /* -** A pointer to an instance of this structure is used as the context -** argument to sqlite3Fts3SegReaderIterate() +** An instance of this function is used to merge together the (potentially +** large number of) doclists for each term that matches a prefix query. +** See function fts3TermSelectMerge() for details. */ typedef struct TermSelect TermSelect; struct TermSelect { - int isReqPos; - char *aaOutput[16]; /* Malloc'd output buffer */ - int anOutput[16]; /* Size of output in bytes */ + char *aaOutput[16]; /* Malloc'd output buffers */ + int anOutput[16]; /* Size each output buffer in bytes */ }; - +/* +** This function is used to read a single varint from a buffer. Parameter +** pEnd points 1 byte past the end of the buffer. When this function is +** called, if *pp points to pEnd or greater, then the end of the buffer +** has been reached. In this case *pp is set to 0 and the function returns. +** +** If *pp does not point to or past pEnd, then a single varint is read +** from *pp. *pp is then set to point 1 byte past the end of the read varint. +** +** If bDescIdx is false, the value read is added to *pVal before returning. +** If it is true, the value read is subtracted from *pVal before this +** function returns. +*/ static void fts3GetDeltaVarint3( - char **pp, - char *pEnd, - int bDescIdx, - sqlite3_int64 *pVal + char **pp, /* IN/OUT: Point to read varint from */ + char *pEnd, /* End of buffer */ + int bDescIdx, /* True if docids are descending */ + sqlite3_int64 *pVal /* IN/OUT: Integer value */ ){ if( *pp>=pEnd ){ *pp = 0; @@ -113921,6 +131994,21 @@ static void fts3GetDeltaVarint3( } } +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ static void fts3PutDeltaVarint3( char **pp, /* IN/OUT: Output pointer */ int bDescIdx, /* True for descending docids */ @@ -113941,10 +132029,34 @@ static void fts3PutDeltaVarint3( *pbFirst = 1; } -#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ static int fts3DoclistOrMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ char *a1, int n1, /* First doclist */ char *a2, int n2, /* Second doclist */ char **paOut, int *pnOut /* OUT: Malloc'd doclist */ @@ -113962,38 +132074,81 @@ static int fts3DoclistOrMerge( *paOut = 0; *pnOut = 0; - aOut = sqlite3_malloc(n1+n2); + + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); if( !aOut ) return SQLITE_NOMEM; p = aOut; fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 || p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } *paOut = aOut; - *pnOut = (p-aOut); + *pnOut = (int)(p-aOut); + assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); return SQLITE_OK; } +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ static void fts3DoclistPhraseMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ @@ -114016,30 +132171,80 @@ static void fts3DoclistPhraseMerge( fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 && p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( iDiff==0 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; int bFirstOutSave = bFirstOut; - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ p = pSave; iPrev = iPrevSave; bFirstOut = bFirstOutSave; } - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( iDiff<0 ){ fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } - *pnRight = p - aOut; + *pnRight = (int)(p - aOut); +} + +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +SQLITE_PRIVATE int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( paaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output ** buffer using memcpy(). */ @@ -114181,19 +132387,27 @@ static int fts3SegReaderCursorAppend( return SQLITE_OK; } +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - int rc = SQLITE_OK; - int rc2; - sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being @@ -114211,7 +132425,7 @@ static int fts3SegReaderCursor( if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ @@ -114234,7 +132448,9 @@ static int fts3SegReaderCursor( } rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, - iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; rc = fts3SegReaderCursorAppend(pCsr, pSeg); @@ -114254,6 +132470,7 @@ static int fts3SegReaderCursor( */ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -114271,35 +132488,51 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); assert( isPrefix==0 || isScan==0 ); - /* "isScan" is only set to true by the ft4aux module, an ordinary - ** full-text tables. */ - assert( isScan==0 || p->aIndex==0 ); - memset(pCsr, 0, sizeof(Fts3MultiSegReader)); - return fts3SegReaderCursor( - p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); } +/* +** In addition to its current configuration, have the Fts3MultiSegReader +** passed as the 4th argument also scan the doclist for term zTerm/nTerm. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3SegReaderCursorAddZero( - Fts3Table *p, - const char *zTerm, - int nTerm, - Fts3MultiSegReader *pCsr + Fts3Table *p, /* FTS virtual table handle */ + int iLangid, + const char *zTerm, /* Term to scan doclist of */ + int nTerm, /* Number of bytes in zTerm */ + Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ ){ - return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); + return fts3SegReaderCursor(p, + iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr + ); } - -SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( +/* +** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, +** if isPrefix is true, to scan the doclist for all terms for which +** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write +** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return +** an SQLite error code. +** +** It is the responsibility of the caller to free this object by eventually +** passing it to fts3SegReaderCursorFree() +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** Output parameter *ppSegcsr is set to 0 if an error occurs. +*/ +static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); @@ -114312,8 +132545,9 @@ SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor( - p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); pSegcsr->bLookup = 1; } } @@ -114321,19 +132555,21 @@ SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm+1 ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor( - p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr ); if( rc==SQLITE_OK ){ - rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); } } } } if( bFound==0 ){ - rc = sqlite3Fts3SegReaderCursor( - p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr ); pSegcsr->bLookup = !isPrefix; } @@ -114343,42 +132579,36 @@ SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( return rc; } +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } /* -** This function retreives the doclist for the specified term (or term -** prefix) from the database. -** -** The returned doclist may be in one of two formats, depending on the -** value of parameter isReqPos. If isReqPos is zero, then the doclist is -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos -** is non-zero, then the returned list is in the same format as is stored -** in the database without the found length specifier at the start of on-disk -** doclists. +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ - int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ Fts3SegFilter filter; /* Segment term filter configuration */ pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); - tsc.isReqPos = isReqPos; - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; @@ -114388,13 +132618,11 @@ static int fts3TermSelect( while( SQLITE_OK==rc && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ - rc = fts3TermSelectCb(p, (void *)&tsc, - pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist - ); + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(p, &tsc); + rc = fts3TermSelectFinishMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -114420,24 +132648,15 @@ static int fts3TermSelect( ** that the doclist is simply a list of docids stored as delta encoded ** varints. */ -static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ +static int fts3DoclistCountDocids(char *aList, int nList){ int nDoc = 0; /* Return value */ if( aList ){ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ char *p = aList; /* Cursor */ - if( !isPoslist ){ - /* The number of docids in the list is the same as the number of - ** varints. In FTS3 a varint consists of a single byte with the 0x80 - ** bit cleared and zero or more bytes with the 0x80 bit set. So to - ** count the varints in the buffer, just count the number of bytes - ** with the 0x80 bit clear. */ - while( pbase.pVtab)->pSegments==0 ); return rc; } +/* +** The following are copied from sqliteInt.h. +** +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#ifndef SQLITE_AMALGAMATION +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -114498,54 +132744,70 @@ static int fts3FilterMethod( ){ int rc; char *zSql; /* SQL statement used to access %_content */ + int eSearch; Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(nVal); - assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( nVal==0 || nVal==1 ); - assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( p->pSegments==0 ); + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr->aMatchinfo); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + if( idxStr ){ pCsr->bDesc = (idxStr[0]=='D'); }else{ pCsr->bDesc = p->bDescIdx; } - pCsr->eSearch = (i16)idxNum; + pCsr->eSearch = (i16)eSearch; - if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ - int iCol = idxNum-FTS3_FULLTEXT_SEARCH; - const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); - if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ return SQLITE_NOMEM; } - rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, - iCol, zQuery, -1, &pCsr->pExpr + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg ); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_ERROR ){ - static const char *zErr = "malformed MATCH expression: [%s]"; - p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); - } return rc; } - rc = sqlite3Fts3ReadLock(p); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); - + rc = fts3EvalStart(pCsr); sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; @@ -114557,24 +132819,25 @@ static int fts3FilterMethod( ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); - }else{ - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); + if( eSearch==FTS3_FULLSCAN_SEARCH ){ + zSql = sqlite3_mprintf( + "SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + if( zSql ){ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + } } - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); if( rc!=SQLITE_OK ) return rc; - if( idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - if( rc!=SQLITE_OK ) return rc; - } - return fts3NextMethod(pCursor); } @@ -114601,10 +132864,17 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ /* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column */ static int fts3ColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ int rc = SQLITE_OK; /* Return Code */ @@ -114612,22 +132882,34 @@ static int fts3ColumnMethod( Fts3Table *p = (Fts3Table *)pCursor->pVtab; /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+1 ); + assert( iCol>=0 && iCol<=p->nColumn+2 ); if( iCol==p->nColumn+1 ){ /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_result_int64(pContext, pCsr->iPrevId); + sqlite3_result_int64(pCtx, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. - */ - sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + ** Return a blob which is a pointer to the cursor. */ + sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); }else{ + /* The requested column is either a user column (one that contains + ** indexed data), or the language-id column. */ rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ - sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); + if( iCol==p->nColumn+2 ){ + int iLangid = 0; + if( p->zLanguageid ){ + iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); + } + sqlite3_result_int(pCtx, iLangid); + }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } } } @@ -114654,23 +132936,88 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); - sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); return rc; } /* -** Implementation of xBegin() method. This is a no-op. +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(pVtab); assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; + p->nLeafAdd = 0; + return fts3SetHasStat(p); } /* @@ -114710,7 +133057,7 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ */ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ char *p = &(*ppPoslist)[-2]; - char c; + char c = 0; while( p>pStart && (c=*p--)==0 ); while( p>pStart && (*p & 0x80) | c ){ @@ -114919,15 +133266,28 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ - rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ){ - return rc; + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); if( p->bHasDocsize ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", @@ -114951,13 +133311,28 @@ static int fts3RenameMethod( return rc; } +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - return fts3SyncMethod(pVtab); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; } + +/* +** The xRelease() method. +** +** This is a no-op. +*/ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(iSavepoint); @@ -114967,6 +133342,12 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( p->mxSavepoint = iSavepoint-1 ); return SQLITE_OK; } + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); @@ -115026,12 +133407,15 @@ static void hashDestroy(void *p){ */ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#endif #ifdef SQLITE_ENABLE_ICU SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); #endif /* -** Initialise the fts3 extension. If this extension is built as part +** Initialize the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. @@ -115041,12 +133425,19 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE + const sqlite3_tokenizer_module *pUnicode = 0; +#endif #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif +#ifndef SQLITE_DISABLE_FTS3_UNICODE + sqlite3Fts3UnicodeTokenizer(&pUnicode); +#endif + #ifdef SQLITE_TEST rc = sqlite3Fts3InitTerm(db); if( rc!=SQLITE_OK ) return rc; @@ -115058,7 +133449,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); - /* Allocate and initialise the hash-table used to store tokenizers. */ + /* Allocate and initialize the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; @@ -115070,6 +133461,10 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ if( rc==SQLITE_OK ){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) +#endif #ifdef SQLITE_ENABLE_ICU || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) #endif @@ -115104,9 +133499,13 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ db, "fts4", &fts3Module, (void *)pHash, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3InitTok(db, (void *)pHash); + } return rc; } + /* An error has occurred. Delete the hash table and return the error code. */ assert( rc!=SQLITE_OK ); if( pHash ){ @@ -115116,18 +133515,6 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ return rc; } -#if !SQLITE_CORE -SQLITE_API int sqlite3_extension_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); -} -#endif - - /* ** Allocate an Fts3MultiSegReader for each token in the expression headed ** by pExpr. @@ -115144,11 +133531,11 @@ SQLITE_API int sqlite3_extension_init( ** doclist and then traversed. */ static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ int *pnToken, /* OUT: Total number of tokens in phrase. */ int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -115157,7 +133544,7 @@ static void fts3EvalAllocateReaders( *pnToken += nToken; for(i=0; ipPhrase->aToken[i]; - int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + int rc = fts3TermSegReaderCursor(pCsr, pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr ); if( rc!=SQLITE_OK ){ @@ -115175,12 +133562,20 @@ static void fts3EvalAllocateReaders( } } +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +*/ static void fts3EvalPhraseMergeToken( - Fts3Table *pTab, - Fts3Phrase *p, - int iToken, - char *pList, - int nList + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ ){ assert( iToken!=p->iDoclistToken ); @@ -115229,9 +133624,15 @@ static void fts3EvalPhraseMergeToken( if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; } +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, - Fts3Phrase *p + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iToken; @@ -115244,7 +133645,7 @@ static int fts3EvalPhraseLoad( if( pToken->pSegcsr ){ int nThis = 0; char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } @@ -115255,25 +133656,32 @@ static int fts3EvalPhraseLoad( return rc; } +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; - int rc = SQLITE_OK; - - int nMaxUndeferred = pPhrase->iDoclistToken; - char *aPoslist = 0; - int nPoslist = 0; - int iPrev = -1; + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; if( pDeferred ){ char *pList; int nList; - rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); if( rc!=SQLITE_OK ) return rc; if( pList==0 ){ @@ -115295,7 +133703,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); sqlite3_free(aPoslist); aPoslist = pList; - nPoslist = aOut - aPoslist; + nPoslist = (int)(aOut - aPoslist); if( nPoslist==0 ){ sqlite3_free(aPoslist); pPhrase->doclist.pList = 0; @@ -115308,6 +133716,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; if( nMaxUndeferred<0 ){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; @@ -115338,7 +133747,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ pPhrase->doclist.pList = aOut; if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ pPhrase->doclist.bFreeList = 1; - pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); }else{ sqlite3_free(aOut); pPhrase->doclist.pList = 0; @@ -115351,29 +133760,62 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ return SQLITE_OK; } +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + /* ** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this ** function has been called successfully on an Fts3Phrase, it may be ** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - int rc; - Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; - if( pCsr->bDesc==pTab->bDescIdx - && bOptOk==1 - && p->nToken==1 - && pFirst->pSegcsr - && pFirst->pSegcsr->bLookup - ){ + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#ifdef SQLITE_TEST + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ /* Use the incremental approach. */ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); - rc = sqlite3Fts3MsrIncrStart( - pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } p->bIncr = 1; - }else{ /* Load the full doclist for the phrase into memory. */ rc = fts3EvalPhraseLoad(pCsr, p); @@ -115386,7 +133828,13 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ /* ** This function is used to iterate backwards (from the end to start) -** through doclists. +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. */ SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ @@ -115394,7 +133842,7 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( int nDoclist, /* Length of aDoclist in bytes */ char **ppIter, /* IN/OUT: Iterator pointer */ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ - int *pnList, /* IN/OUT: List length pointer */ + int *pnList, /* OUT: List length pointer */ u8 *pbEof /* OUT: End-of-file flag */ ){ char *p = *ppIter; @@ -115421,7 +133869,7 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( iMul = (bDescIdx ? -1 : 1); } - *pnList = pEnd - pNext; + *pnList = (int)(pEnd - pNext); *ppIter = pNext; *piDocid = iDocid; }else{ @@ -115435,12 +133883,257 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( }else{ char *pSave = p; fts3ReversePoslist(aDoclist, &p); - *pnList = (pSave - p); + *pnList = (int)(pSave - p); } *ppIter = p; } } +/* +** Iterate forwards through a doclist. +*/ +SQLITE_PRIVATE void sqlite3Fts3DoclistNext( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; + + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 && p->bIncr ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+1); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } + } + } + + *pbEof = bEof; + return rc; +} + /* ** Attempt to move the phrase iterator to point to the next matching docid. ** If an error occurs, return an SQLite error code. Otherwise, return @@ -115451,74 +134144,48 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( ** successfully advanced, *pbEof is set to 0. */ static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, - Fts3Phrase *p, - u8 *pbEof + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ ){ int rc = SQLITE_OK; Fts3Doclist *pDL = &p->doclist; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; if( p->bIncr ){ - assert( p->nToken==1 ); - assert( pDL->pNextDocid==0 ); - rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, - &pDL->iDocid, &pDL->pList, &pDL->nList - ); - if( rc==SQLITE_OK && !pDL->pList ){ - *pbEof = 1; - } + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof ); pDL->pList = pDL->pNextDocid; }else{ - char *pIter; /* Used to iterate through aAll */ - char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ - if( pDL->pNextDocid ){ - pIter = pDL->pNextDocid; - }else{ - pIter = pDL->aAll; - } - - if( pIter>=pEnd ){ - /* We have already reached the end of this doclist. EOF. */ - *pbEof = 1; - }else{ - sqlite3_int64 iDelta; - pIter += sqlite3Fts3GetVarint(pIter, &iDelta); - if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ - pDL->iDocid += iDelta; - }else{ - pDL->iDocid -= iDelta; - } - pDL->pList = pIter; - fts3PoslistCopy(0, &pIter); - pDL->nList = (pIter - pDL->pList); - - /* pIter now points just past the 0x00 that terminates the position- - ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim2(), then pIter may not actually - ** point to the start of the next docid value. The following line deals - ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim2(). */ - while( pIterpNextDocid = pIter; - assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); - *pbEof = 0; - } + fts3EvalDlPhraseNext(pTab, pDL, pbEof); } return rc; } +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ static void fts3EvalStartReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int bOptOk, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -115528,34 +134195,53 @@ static void fts3EvalStartReaders( if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break; } pExpr->bDeferred = (i==nToken); - *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); }else{ - fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); - fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); } } } +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { Fts3Phrase *pPhrase; /* The phrase the token belongs to */ int iToken; /* Position of token in phrase */ Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; - int nOvfl; + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ int iCol; /* The column the token must match */ }; +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3Expr *pExpr, - Fts3TokenAndCost **ppTC, - Fts3Expr ***ppOr, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK && pExpr ){ + if( *pRc==SQLITE_OK ){ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; @@ -115569,6 +134255,11 @@ static void fts3EvalTokenCosts( *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); } }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); if( pExpr->eType==FTSQUERY_OR ){ pRoot = pExpr->pLeft; **ppOr = pRoot; @@ -115585,19 +134276,30 @@ static void fts3EvalTokenCosts( } } +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ int rc; Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; sqlite3_stmt *pStmt; @@ -115618,7 +134320,7 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ } if( nDoc==0 || nByte==0 ){ sqlite3_reset(pStmt); - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } pCsr->nDoc = nDoc; @@ -115632,68 +134334,130 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ return SQLITE_OK; } +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3TokenAndCost *aTC, - int nTC + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ ){ - int nDocSize = 0; - int nDocEst = 0; - int rc = SQLITE_OK; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int ii; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ - int nOvfl = 0; - int nTerm = 0; + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ for(ii=0; ii0 ); - for(ii=0; iinOvfl) + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) ){ - pTC = &aTC[jj]; + pTC = &aTC[iTC]; } } assert( pTC ); - /* At this point pTC points to the cheapest remaining token. */ - if( ii==0 ){ - if( pTC->nOvfl ){ - nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; - }else{ + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ Fts3PhraseToken *pToken = pTC->pToken; int nList = 0; char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - nDocEst = fts3DoclistCountDocids(1, pList, nList); + int nCount; fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountnOvfl>=(nDocEst*nDocSize) ){ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - } - nDocEst = 1 + (nDocEst/4); } pTC->pToken = 0; } @@ -115701,36 +134465,30 @@ static int fts3EvalSelectDeferred( return rc; } -SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int nToken = 0; int nOr = 0; /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); - /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: - ** This call will eventually also be responsible for determining which - ** tokens are 'deferred' until the document text is loaded into memory. - ** - ** Each token in each phrase is dealt with using one of the following - ** three strategies: - ** - ** 1. Entire doclist loaded into memory as part of the - ** fts3EvalStartReaders() call. - ** - ** 2. Doclist loaded into memory incrementally, as part of each - ** sqlite3Fts3EvalNext() call. - ** - ** 3. Token doclist is never loaded. Instead, documents are loaded into - ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() - ** call. This is known as a "deferred" token. - */ - - /* If bOptOk is true, check if there are any tokens that should be deferred. - */ - if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( @@ -115746,9 +134504,9 @@ SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int b Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; - fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); - nToken = pTC-aTC; - nOr = ppOr-apOr; + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); if( rc==SQLITE_OK ){ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); @@ -115760,12 +134518,16 @@ SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int b sqlite3_free(aTC); } } +#endif - fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); + fts3EvalStartReaders(pCsr, pCsr->pExpr, &rc); return rc; } -static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); } @@ -115774,8 +134536,30 @@ static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ pPhrase->doclist.bFreeList = 0; } -static int fts3EvalNearTrim2( - int nNear, +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ char *aTmp, /* Temporary space to use */ char **paPoslist, /* IN/OUT: Position list */ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ @@ -115795,7 +134579,7 @@ static int fts3EvalNearTrim2( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 ); if( res ){ - nNew = (pOut - pPhrase->doclist.pList) - 1; + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; assert( pPhrase->doclist.pList[nNew]=='\0' ); assert( nNew<=pPhrase->doclist.nList && nNew>0 ); memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); @@ -115807,6 +134591,172 @@ static int fts3EvalNearTrim2( return res; } +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See fts3EvalTestDeferredAndNear() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ int res = 1; @@ -115828,7 +134778,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ ** ** The right-hand child of a NEAR node is always a phrase. The ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this. + ** no exceptions to this - it's the way the parser in fts3_expr.c works. */ if( *pRc==SQLITE_OK && pExpr->eType==FTSQUERY_NEAR @@ -115844,160 +134794,68 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ nTmp += p->pRight->pPhrase->doclist.nList; } nTmp += p->pPhrase->doclist.nList; - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; + if( nTmp==0 ){ res = 0; }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear = p->pParent->nNear; - Fts3Phrase *pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } } + + sqlite3_free(aTmp); } - - sqlite3_free(aTmp); } return res; } /* -** This macro is used by the fts3EvalNext() function. The two arguments are -** 64-bit docid values. If the current query is "ORDER BY docid ASC", then -** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then -** it returns (i2 - i1). This allows the same code to be used for merging -** doclists in ascending or descending order. +** This function is a helper function for fts3EvalTestDeferredAndNear(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. */ -#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) - -static void fts3EvalNext( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK ){ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; - - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); - if( pLeft->bDeferred ){ - fts3EvalNext(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - fts3EvalNext(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else{ - fts3EvalNext(pCsr, pRight, pRc); - } - } - - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); - } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ - fts3EvalNext(pCsr, pRight, pRc); - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - } - - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; - } - - break; - } - - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - - if( pRight->bStart==0 ){ - fts3EvalNext(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } - - fts3EvalNext(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNext(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; - } - - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalZeroPoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; - } - } - } -} - -static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - int bHit = 1; + int bHit = 1; /* Return value */ if( *pRc==SQLITE_OK ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) && fts3EvalNearTest(pExpr, pRc) ); @@ -116023,43 +134881,46 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pRight->pPhrase); + fts3EvalInvalidatePoslist(p->pRight->pPhrase); } } if( p->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pPhrase); + fts3EvalInvalidatePoslist(p->pPhrase); } } break; case FTSQUERY_OR: { - int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); bHit = bHit1 || bHit2; break; } case FTSQUERY_NOT: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) ); break; default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED if( pCsr->pDeferred && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) ){ Fts3Phrase *pPhrase = pExpr->pPhrase; assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; - }else{ + }else +#endif + { bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); } break; @@ -116070,27 +134931,49 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ } /* -** Return 1 if both of the following are true: +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: ** ** 1. *pRc is SQLITE_OK when this function returns, and ** ** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does not match the query. +** it is determined that the row does *not* match the query. ** ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ -static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ +static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ if( pCsr->pDeferred ){ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ rc = sqlite3Fts3CacheDeferredDoclists(pCsr); } } - bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ sqlite3Fts3FreeDeferredDoclists(pCsr); *pRc = rc; } @@ -116101,7 +134984,7 @@ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ ** Advance to the next document that matches the FTS expression in ** Fts3Cursor.pExpr. */ -SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ +static int fts3EvalNext(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return Code */ Fts3Expr *pExpr = pCsr->pExpr; assert( pCsr->isEof==0 ); @@ -116113,19 +134996,29 @@ SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ sqlite3_reset(pCsr->pStmt); } assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNext(pCsr, pExpr, &rc); + fts3EvalNextRow(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + return rc; } /* ** Restart interation for expression pExpr so that the next call to -** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** ** If *pRc is other than SQLITE_OK when this function is called, it is @@ -116141,14 +135034,18 @@ static void fts3EvalRestart( Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); if( pPhrase->bIncr ){ - assert( pPhrase->nToken==1 ); - assert( pPhrase->aToken[0].pSegcsr ); - sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } - pPhrase->doclist.pNextDocid = 0; pPhrase->doclist.iDocid = 0; } @@ -116193,7 +135090,7 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ pExpr->aMI[iCol*3 + 2] += (iCnt>0); if( *p==0x00 ) break; p++; - p += sqlite3Fts3GetVarint32(p, &iCol); + p += fts3GetVarint32(p, &iCol); } } @@ -116257,14 +135154,14 @@ static int fts3EvalGatherStats( assert( sqlite3_data_count(pCsr->pStmt)==0 ); /* Advance to the next document */ - fts3EvalNext(pCsr, pRoot, &rc); + fts3EvalNextRow(pCsr, pRoot, &rc); pCsr->isEof = pRoot->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR - && fts3EvalLoadDeferred(pCsr, &rc) + && fts3EvalTestDeferredAndNear(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ @@ -116286,10 +135183,10 @@ static int fts3EvalGatherStats( */ fts3EvalRestart(pCsr, pRoot, &rc); do { - fts3EvalNext(pCsr, pRoot, &rc); + fts3EvalNextRow(pCsr, pRoot, &rc); assert( pRoot->bEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - fts3EvalLoadDeferred(pCsr, &rc); + fts3EvalTestDeferredAndNear(pCsr, &rc); } } return rc; @@ -116363,7 +135260,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( ** of the current row. ** ** More specifically, the returned buffer contains 1 varint for each -** occurence of the phrase in the column, stored using the normal (delta+2) +** occurrence of the phrase in the column, stored using the normal (delta+2) ** compression and is terminated by either an 0x01 or 0x00 byte. For example, ** if the requested column contains "a b X c d X X" and the position-list ** for 'X' is requested, the buffer returned may contain: @@ -116373,29 +135270,128 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( ** This function works regardless of whether or not the phrase is deferred, ** incremental, or neither. */ -SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist( +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( Fts3Cursor *pCsr, /* FTS3 cursor object */ Fts3Expr *pExpr, /* Phrase to return doclist for */ - int iCol /* Column to return position list for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ ){ Fts3Phrase *pPhrase = pExpr->pPhrase; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - char *pIter = pPhrase->doclist.pList; + char *pIter; int iThis; + sqlite3_int64 iDocid; + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; assert( iCol>=0 && iColnColumn ); - if( !pIter - || pExpr->bEof - || pExpr->iDocid!=pCsr->iPrevId - || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) - ){ - return 0; + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; } - assert( pPhrase->doclist.nList>0 ); + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int iMul; /* +1 if csr dir matches index dir, else -1 */ + int bOr = 0; + u8 bEof = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int rc = SQLITE_OK; + int bEofSave = pExpr->bEof; + fts3EvalRestart(pCsr, pExpr, &rc); + while( rc==SQLITE_OK && !pExpr->bEof ){ + fts3EvalNextRow(pCsr, pExpr, &rc); + if( bEofSave==0 && pExpr->iDocid==iDocid ) break; + } + pIter = pPhrase->doclist.pList; + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + if( rc!=SQLITE_OK ) return rc; + } + + iMul = ((pCsr->bDesc==bDescDoclist) ? 1 : -1); + while( bTreeEof==1 + && pNear->bEof==0 + && (DOCID_CMP(pNear->iDocid, pCsr->iPrevId) * iMul)<0 + ){ + int rc = SQLITE_OK; + fts3EvalNextRow(pCsr, pExpr, &rc); + if( rc!=SQLITE_OK ) return rc; + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + } + + bEof = (pPhrase->doclist.nAll==0); + assert( bDescDoclist==0 || bDescDoclist==1 ); + assert( pCsr->bDesc==0 || pCsr->bDesc==1 ); + + if( bEof==0 ){ + if( pCsr->bDesc==bDescDoclist ){ + int dummy; + if( pNear->bEof ){ + /* This expression is already at EOF. So position it to point to the + ** last entry in the doclist at pPhrase->doclist.aAll[]. Variable + ** iDocid is already set for this entry, so all that is required is + ** to set pIter to point to the first byte of the last position-list + ** in the doclist. + ** + ** It would also be correct to set pIter and iDocid to zero. In + ** this case, the first call to sqltie3Fts4DoclistPrev() below + ** would also move the iterator to point to the last entry in the + ** doclist. However, this is expensive, as to do so it has to + ** iterate through the entire doclist from start to finish (since + ** it does not know the docid for the last entry). */ + pIter = &pPhrase->doclist.aAll[pPhrase->doclist.nAll-1]; + fts3ReversePoslist(pPhrase->doclist.aAll, &pIter); + } + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + sqlite3Fts3DoclistPrev( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + }else{ + if( pNear->bEof ){ + pIter = 0; + iDocid = 0; + } + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + } + } + + if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0; + } + if( pIter==0 ) return SQLITE_OK; + if( *pIter==0x01 ){ pIter++; - pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + pIter += fts3GetVarint32(pIter, &iThis); }else{ iThis = 0; } @@ -116403,10 +135399,11 @@ SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist( fts3ColumnlistCopy(0, &pIter); if( *pIter==0x00 ) return 0; pIter++; - pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + pIter += fts3GetVarint32(pIter, &iThis); } - return ((iCol==iThis)?pIter:0); + *ppOut = ((iCol==iThis)?pIter:0); + return SQLITE_OK; } /* @@ -116420,7 +135417,7 @@ SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; inToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); @@ -116429,6 +135426,33 @@ SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ } } + +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + #endif /************** End of fts3.c ************************************************/ @@ -116448,6 +135472,8 @@ SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ +/* #include */ typedef struct Fts3auxTable Fts3auxTable; typedef struct Fts3auxCursor Fts3auxCursor; @@ -116463,6 +135489,7 @@ struct Fts3auxCursor { Fts3SegFilter filter; char *zStop; int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ int isEof; /* True if cursor is at EOF */ sqlite3_int64 iRowid; /* Current rowid */ @@ -116477,7 +135504,8 @@ struct Fts3auxCursor { /* ** Schema of the terms table. */ -#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, col, documents, occurrences)" +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" /* ** This function does all the work for both the xConnect and xCreate methods. @@ -116502,20 +135530,29 @@ static int fts3auxConnectMethod( UNUSED_PARAMETER(pUnused); - /* The user should specify a single argument - the name of an fts3 table. */ - if( argc!=4 ){ - *pzErr = sqlite3_mprintf( - "wrong number of arguments to fts4aux constructor" - ); - return SQLITE_ERROR; - } + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; zDb = argv[1]; - nDb = strlen(zDb); - zFts3 = argv[3]; - nFts3 = strlen(zFts3); + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } + nFts3 = (int)strlen(zFts3); - rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); if( rc!=SQLITE_OK ) return rc; nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; @@ -116535,6 +135572,10 @@ static int fts3auxConnectMethod( *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; + + bad_args: + *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; } /* @@ -116571,6 +135612,8 @@ static int fts3auxBestIndexMethod( int iEq = -1; int iGe = -1; int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ UNUSED_PARAMETER(pVTab); @@ -116582,36 +135625,48 @@ static int fts3auxBestIndexMethod( pInfo->orderByConsumed = 1; } - /* Search for equality and range constraints on the "term" column. */ + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable && pInfo->aConstraint[i].iColumn==0 ){ + if( pInfo->aConstraint[i].usable ){ int op = pInfo->aConstraint[i].op; - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; - if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } } } if( iEq>=0 ){ pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; - pInfo->aConstraintUsage[iEq].argvIndex = 1; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; pInfo->estimatedCost = 5; }else{ pInfo->idxNum = 0; pInfo->estimatedCost = 20000; if( iGe>=0 ){ pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; - pInfo->aConstraintUsage[iGe].argvIndex = 1; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; pInfo->estimatedCost /= 2; } if( iLe>=0 ){ pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; - pInfo->aConstraintUsage[iLe].argvIndex = 1 + (iGe>=0); + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; pInfo->estimatedCost /= 2; } } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } return SQLITE_OK; } @@ -116771,7 +135826,14 @@ static int fts3auxFilterMethod( Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; int rc; - int isScan; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; UNUSED_PARAMETER(nVal); UNUSED_PARAMETER(idxStr); @@ -116781,7 +135843,21 @@ static int fts3auxFilterMethod( || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ); - isScan = (idxNum!=FTS4AUX_EQ_CONSTRAINT); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); @@ -116793,22 +135869,35 @@ static int fts3auxFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - if( idxNum&(FTS4AUX_EQ_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ){ + if( iEq>=0 || iGe>=0 ){ const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); if( zStr ){ pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; } } - if( idxNum&FTS4AUX_LE_CONSTRAINT ){ - int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0; - pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx])); - pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]); + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); if( pCsr->zStop==0 ) return SQLITE_NOMEM; } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); - rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -116832,24 +135921,37 @@ static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3auxColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts3auxCursor *p = (Fts3auxCursor *)pCursor; assert( p->isEof==0 ); - if( iCol==0 ){ /* Column "term" */ - sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); - }else if( iCol==1 ){ /* Column "col" */ - if( p->iCol ){ - sqlite3_result_int(pContext, p->iCol-1); - }else{ - sqlite3_result_text(pContext, "*", -1, SQLITE_STATIC); - } - }else if( iCol==2 ){ /* Column "documents" */ - sqlite3_result_int64(pContext, p->aStat[p->iCol].nDoc); - }else{ /* Column "occurrences" */ - sqlite3_result_int64(pContext, p->aStat[p->iCol].nOcc); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; } return SQLITE_OK; @@ -116986,6 +136088,8 @@ SQLITE_API int sqlite3_fts3_enable_parentheses = 0; */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 +/* #include */ +/* #include */ /* ** isNot: @@ -116998,7 +136102,9 @@ SQLITE_API int sqlite3_fts3_enable_parentheses = 0; typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ int isNot; /* True if getNextNode() sees a unary - */ @@ -117010,7 +136116,7 @@ struct ParseContext { ** This function is equivalent to the standard isspace() function. ** ** The standard isspace() can be awkward to use safely, because although it -** is defined to accept an argument of type int, its behaviour when passed +** is defined to accept an argument of type int, its behavior when passed ** an integer that falls outside of the range of the unsigned char type ** is undefined (and sometimes, "undefined" means segfault). This wrapper ** is defined to accept an argument of type char, and always returns 0 for @@ -117032,6 +136138,38 @@ static void *fts3MallocZero(int nByte){ return pRet; } +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); /* ** Extract the next token from buffer z (length n) using the tokenizer @@ -117057,17 +136195,22 @@ static int getNextToken( int rc; sqlite3_tokenizer_cursor *pCursor; Fts3Expr *pRet = 0; - int nConsumed = 0; + int i = 0; - rc = pModule->xOpen(pTokenizer, z, n, &pCursor); + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; - int nToken, iStart, iEnd, iPosition; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; int nByte; /* total space to allocate */ - pCursor->pTokenizer = pTokenizer; rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)fts3MallocZero(nByte); @@ -117086,17 +136229,30 @@ static int getNextToken( pRet->pPhrase->aToken[0].isPrefix = 1; iEnd++; } - if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pParse->isNot = 1; + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } } + } - nConsumed = iEnd; + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; } pModule->xClose(pCursor); } - *pnConsumed = nConsumed; *ppExpr = pRet; return rc; } @@ -117161,13 +136317,13 @@ static int getNextString( ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase ** structures. */ - rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; - pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ const char *zByte; - int nByte, iBegin, iEnd, iPos; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ Fts3PhraseToken *pToken; @@ -117187,6 +136343,7 @@ static int getNextString( pToken->n = nByte; pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); nToken = ii+1; } } @@ -117208,8 +136365,12 @@ static int getNextString( p->pPhrase->nToken = nToken; zBuf = (char *)&p->pPhrase->aToken[nToken]; - memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + }else{ + assert( nTemp==0 ); + } for(jj=0; jjpPhrase->nToken; jj++){ p->pPhrase->aToken[jj].z = zBuf; @@ -117231,12 +136392,6 @@ no_mem: return SQLITE_NOMEM; } -/* -** Function getNextNode(), which is called by fts3ExprParse(), may itself -** call fts3ExprParse(). So this forward declaration is required. -*/ -static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); - /* ** The output variable *ppExpr is populated with an allocated Fts3Expr ** structure, or set to 0 if the end of the input buffer is reached. @@ -117333,27 +136488,6 @@ static int getNextNode( } } - /* Check for an open bracket. */ - if( sqlite3_fts3_enable_parentheses ){ - if( *zInput=='(' ){ - int nConsumed; - pParse->nNest++; - rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed); - if( rc==SQLITE_OK && !*ppExpr ){ - rc = SQLITE_DONE; - } - *pnConsumed = (int)((zInput - z) + 1 + nConsumed); - return rc; - } - - /* Check for a close bracket. */ - if( *zInput==')' ){ - pParse->nNest--; - *pnConsumed = (int)((zInput - z) + 1); - return SQLITE_DONE; - } - } - /* See if we are dealing with a quoted phrase. If this is the case, then ** search for the closing quote and pass the whole string to getNextString() ** for processing. This is easy to do, as fts3 has no syntax for escaping @@ -117368,6 +136502,21 @@ static int getNextNode( return getNextString(pParse, &zInput[1], ii-1, ppExpr); } + if( sqlite3_fts3_enable_parentheses ){ + if( *zInput=='(' ){ + int nConsumed = 0; + pParse->nNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } /* If control flows to this point, this must be a regular token, or ** the end of the input. Read a regular token using the sqlite3_tokenizer @@ -117486,94 +136635,100 @@ static int fts3ExprParse( while( rc==SQLITE_OK ){ Fts3Expr *p = 0; int nByte = 0; + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); if( rc==SQLITE_OK ){ - int isPhrase; + if( p ){ + int isPhrase; - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; - }else{ - pRet = p; + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; } - }else{ - insertBinaryOperator(&pRet, pPrev, p); + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; } - isRequirePhrase = !isPhrase; + pPrev = p; } assert( nByte>0 ); } assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); nIn -= nByte; zIn += nByte; - pPrev = p; } if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ @@ -117591,6 +136746,7 @@ static int fts3ExprParse( pIter = pIter->pLeft; } pIter->pLeft = pRet; + pRet->pParent = pIter; pRet = pNotBranch; } } @@ -117607,6 +136763,223 @@ exprparse_out: return rc; } +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + /* ** Parameters z and n contain a pointer to and length of a buffer containing ** an fts3 query expression, respectively. This function attempts to parse the @@ -117633,50 +137006,80 @@ exprparse_out: */ SQLITE_PRIVATE int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr /* OUT: Parsed query structure */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ - int nParsed; - int rc; - ParseContext sParse; - sParse.pTokenizer = pTokenizer; - sParse.azCol = (const char **)azCol; - sParse.nCol = nCol; - sParse.iDefaultCol = iDefaultCol; - sParse.nNest = 0; - if( z==0 ){ - *ppExpr = 0; - return SQLITE_OK; + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } } - if( n<0 ){ - n = (int)strlen(z); - } - rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); - /* Check for mismatched parenthesis */ - if( rc==SQLITE_OK && sParse.nNest ){ - rc = SQLITE_ERROR; + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + *pzErr = sqlite3_mprintf( + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z); + } } return rc; } /* -** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** Free a single node of an expression tree. */ -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){ - if( p ){ - assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); - sqlite3Fts3ExprFree(p->pLeft); - sqlite3Fts3ExprFree(p->pRight); - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->aMI); - sqlite3_free(p); +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } } } @@ -117687,6 +137090,7 @@ SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){ #ifdef SQLITE_TEST +/* #include */ /* ** Function to query the hash-table of tokenizers (see README.tokenizers). @@ -117727,6 +137131,9 @@ static int queryTestTokenizer( ** the returned expression text and then freed using sqlite3_free(). */ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } switch( pExpr->eType ){ case FTSQUERY_PHRASE: { Fts3Phrase *pPhrase = pExpr->pPhrase; @@ -117834,10 +137241,21 @@ static void fts3ExprTest( azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); } - rc = sqlite3Fts3ExprParse( - pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr - ); + if( sqlite3_user_data(context) ){ + char *zDummy = 0; + rc = sqlite3Fts3ExprParse( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy + ); + assert( rc==SQLITE_OK || pExpr==0 ); + sqlite3_free(zDummy); + }else{ + rc = fts3ExprParseUnbalanced( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + ); + } + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ + sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); @@ -117860,9 +137278,15 @@ exprtest_out: ** with database connection db. */ SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ - return sqlite3_create_function( + int rc = sqlite3_create_function( db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + ); + } + return rc; } #endif @@ -117897,6 +137321,9 @@ SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ +/* #include */ +/* #include */ /* @@ -117963,13 +137390,13 @@ SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ */ static int fts3StrHash(const void *pKey, int nKey){ const char *z = (const char *)pKey; - int h = 0; + unsigned h = 0; if( nKey<=0 ) nKey = (int) strlen(z); while( nKey > 0 ){ h = (h<<3) ^ h ^ *z++; nKey--; } - return h & 0x7fffffff; + return (int)(h & 0x7fffffff); } static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ if( n1!=n2 ) return 1; @@ -118277,6 +137704,10 @@ SQLITE_PRIVATE void *sqlite3Fts3HashInsert( */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ +/* #include */ +/* #include */ +/* #include */ /* @@ -118287,7 +137718,7 @@ typedef struct porter_tokenizer { } porter_tokenizer; /* -** Class derived from sqlit3_tokenizer_cursor +** Class derived from sqlite3_tokenizer_cursor */ typedef struct porter_tokenizer_cursor { sqlite3_tokenizer_cursor base; @@ -118650,12 +138081,14 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ /* Step 2 */ switch( z[1] ){ case 'a': - stem(&z, "lanoita", "ate", m_gt_0) || - stem(&z, "lanoit", "tion", m_gt_0); + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } break; case 'c': - stem(&z, "icne", "ence", m_gt_0) || - stem(&z, "icna", "ance", m_gt_0); + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } break; case 'e': stem(&z, "rezi", "ize", m_gt_0); @@ -118664,43 +138097,54 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ stem(&z, "igol", "log", m_gt_0); break; case 'l': - stem(&z, "ilb", "ble", m_gt_0) || - stem(&z, "illa", "al", m_gt_0) || - stem(&z, "iltne", "ent", m_gt_0) || - stem(&z, "ile", "e", m_gt_0) || - stem(&z, "ilsuo", "ous", m_gt_0); + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } break; case 'o': - stem(&z, "noitazi", "ize", m_gt_0) || - stem(&z, "noita", "ate", m_gt_0) || - stem(&z, "rota", "ate", m_gt_0); + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } break; case 's': - stem(&z, "msila", "al", m_gt_0) || - stem(&z, "ssenevi", "ive", m_gt_0) || - stem(&z, "ssenluf", "ful", m_gt_0) || - stem(&z, "ssensuo", "ous", m_gt_0); + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } break; case 't': - stem(&z, "itila", "al", m_gt_0) || - stem(&z, "itivi", "ive", m_gt_0) || - stem(&z, "itilib", "ble", m_gt_0); + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } break; } /* Step 3 */ switch( z[0] ){ case 'e': - stem(&z, "etaci", "ic", m_gt_0) || - stem(&z, "evita", "", m_gt_0) || - stem(&z, "ezila", "al", m_gt_0); + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } break; case 'i': stem(&z, "itici", "ic", m_gt_0); break; case 'l': - stem(&z, "laci", "ic", m_gt_0) || - stem(&z, "luf", "", m_gt_0); + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } break; case 's': stem(&z, "ssen", "", m_gt_0); @@ -118741,9 +138185,11 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ z += 3; } }else if( z[2]=='e' ){ - stem(&z, "tneme", "", m_gt_1) || - stem(&z, "tnem", "", m_gt_1) || - stem(&z, "tne", "", m_gt_1); + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } } } break; @@ -118762,8 +138208,9 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ } break; case 't': - stem(&z, "eta", "", m_gt_1) || - stem(&z, "iti", "", m_gt_1); + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } break; case 'u': if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ @@ -118877,6 +138324,7 @@ static const sqlite3_tokenizer_module porterTokenizerModule = { porterOpen, porterClose, porterNext, + 0 }; /* @@ -118918,12 +138366,10 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT1 -#endif - #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ +/* #include */ /* ** Implementation of the SQL scalar function for accessing the underlying @@ -119099,14 +138545,15 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST +#include +/* #include */ /* ** Implementation of a special SQL scalar function for testing tokenizers ** designed to be used in concert with the Tcl testing framework. This -** function must be called with two arguments: +** function must be called with two or more arguments: ** -** SELECT (, ); -** SELECT (, ); +** SELECT (, ..., ); ** ** where is the name passed as the second argument ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') @@ -119143,27 +138590,27 @@ static void testFunc( const char *zInput; int nInput; - const char *zArg = 0; + const char *azArg[64]; const char *zToken; - int nToken; - int iStart; - int iEnd; - int iPos; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int i; Tcl_Obj *pRet; - assert( argc==2 || argc==3 ); + if( argc<2 ){ + sqlite3_result_error(context, "insufficient arguments", -1); + return; + } nName = sqlite3_value_bytes(argv[0]); zName = (const char *)sqlite3_value_text(argv[0]); nInput = sqlite3_value_bytes(argv[argc-1]); zInput = (const char *)sqlite3_value_text(argv[argc-1]); - if( argc==3 ){ - zArg = (const char *)sqlite3_value_text(argv[1]); - } - pHash = (Fts3Hash *)sqlite3_user_data(context); p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); @@ -119177,16 +138624,19 @@ static void testFunc( pRet = Tcl_NewObj(); Tcl_IncrRefCount(pRet); - if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){ + for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ zErr = "error in xCreate()"; goto finish; } pTokenizer->pModule = p; - if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){ + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ zErr = "error in xOpen()"; goto finish; } - pCsr->pTokenizer = pTokenizer; while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); @@ -119320,7 +138770,7 @@ static void intTestFunc( /* ** Set up SQL objects in database db used to access the contents of ** the hash table pointed to by argument pHash. The hash table must -** been initialised to use string keys, and to take a private copy +** been initialized to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); @@ -119362,10 +138812,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( } #ifdef SQLITE_TEST if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0); - } - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0); + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); } if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); @@ -119410,6 +138857,10 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ +/* #include */ +/* #include */ +/* #include */ typedef struct simple_tokenizer { @@ -119598,6 +139049,7 @@ static const sqlite3_tokenizer_module simpleTokenizerModule = { simpleOpen, simpleClose, simpleNext, + 0, }; /* @@ -119613,6 +139065,462 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ /************** End of fts3_tokenizer1.c *************************************/ +/************** Begin file fts3_tokenize_vtab.c ******************************/ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenize_vtab.c **********************************/ /************** Begin file fts3_write.c **************************************/ /* ** 2009 Oct 23 @@ -119635,6 +139543,12 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ +/* #include */ +/* #include */ + + +#define FTS_MAX_APPENDABLE_HEIGHT 16 /* ** When full-text index nodes are loaded from disk, the buffer that they @@ -119675,6 +139589,29 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif +/* +** The two values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; @@ -119722,6 +139659,8 @@ struct Fts3DeferredToken { */ struct Fts3SegReader { int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ @@ -119755,7 +139694,7 @@ struct Fts3SegReader { }; #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) -#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) /* ** An instance of this structure is used to create a segment b-tree in the @@ -119777,6 +139716,7 @@ struct SegmentWriter { int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ }; /* @@ -119835,13 +139775,26 @@ struct SegmentNode { #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 -#define SQL_SELECT_DOCTOTAL 22 -#define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 #define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 - #define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 /* ** This function is used to obtain an SQLite prepared statement handle @@ -119868,11 +139821,11 @@ static int fts3SqlStmt( /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", +/* 7 */ "SELECT %s WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", -/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", -/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " @@ -119890,12 +139843,71 @@ static int fts3SqlStmt( /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", -/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", /* 24 */ "", /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" + " ORDER BY (level %% 1024) ASC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" }; int rc = SQLITE_OK; @@ -119910,7 +139922,7 @@ static int fts3SqlStmt( if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } @@ -119934,26 +139946,22 @@ static int fts3SqlStmt( return rc; } + static int fts3SelectDocsize( Fts3Table *pTab, /* FTS3 table handle */ - int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ int rc; /* Return code */ - assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL ); - - rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); + rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0); if( rc==SQLITE_OK ){ - if( eStmt==SQL_SELECT_DOCSIZE ){ - sqlite3_bind_int64(pStmt, 1, iDocid); - } + sqlite3_bind_int64(pStmt, 1, iDocid); rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; pStmt = 0; }else{ rc = SQLITE_OK; @@ -119968,7 +139976,21 @@ SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal( Fts3Table *pTab, /* Fts3 table handle */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt); + sqlite3_stmt *pStmt = 0; + int rc; + rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)!=SQLITE_ROW + || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB + ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; + pStmt = 0; + } + } + *ppStmt = pStmt; + return rc; } SQLITE_PRIVATE int sqlite3Fts3SelectDocsize( @@ -119976,7 +139998,7 @@ SQLITE_PRIVATE int sqlite3Fts3SelectDocsize( sqlite3_int64 iDocid, /* Docid to read size data for */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt); + return fts3SelectDocsize(pTab, iDocid, ppStmt); } /* @@ -120006,35 +140028,73 @@ static void fts3SqlExec( /* -** This function ensures that the caller has obtained a shared-cache -** table-lock on the %_content table. This is required before reading -** data from the fts3 table. If this lock is not acquired first, then -** the caller may end up holding read-locks on the %_segments and %_segdir -** tables, but no read-lock on the %_content table. If this happens -** a second connection will be able to write to the fts3 table, but -** attempting to commit those writes might return SQLITE_LOCKED or -** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain -** write-locks on the %_segments and %_segdir ** tables). +** This function ensures that the caller has obtained an exclusive +** shared-cache table-lock on the %_segdir table. This is required before +** writing data to the fts3 table. If this lock is not acquired first, then +** the caller may end up attempting to take this lock as part of committing +** a transaction, causing SQLite to return SQLITE_LOCKED or +** LOCKED_SHAREDCACHEto a COMMIT command. ** -** We try to avoid this because if FTS3 returns any error when committing -** a transaction, the whole transaction will be rolled back. And this is -** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can -** still happen if the user reads data directly from the %_segments or -** %_segdir tables instead of going through FTS3 though. +** It is best to avoid this because if FTS3 returns any error when +** committing a transaction, the whole transaction will be rolled back. +** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. +** It can still happen if the user locks the underlying tables directly +** instead of accessing them via FTS. */ -SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ - int rc; /* Return code */ - sqlite3_stmt *pStmt; /* Statement used to obtain lock */ - - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_null(pStmt, 1); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); +static int fts3Writelock(Fts3Table *p){ + int rc = SQLITE_OK; + + if( p->nPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } } + return rc; } +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ +){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; +} + /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -120054,8 +140114,9 @@ SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ */ SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ - int iLevel, /* Level to select */ + int iLevel, /* Level to select (relative level) */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ ){ int rc; @@ -120069,14 +140130,16 @@ SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } *ppStmt = pStmt; @@ -120242,18 +140305,19 @@ static int fts3PendingTermsAddOne( */ static int fts3PendingTermsAdd( Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ const char *zText, /* Text of document to be inserted */ int iCol, /* Column into which text is being inserted */ - u32 *pnWord /* OUT: Number of tokens inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ ){ int rc; - int iStart; - int iEnd; - int iPos; + int iStart = 0; + int iEnd = 0; + int iPos = 0; int nWord = 0; char const *zToken; - int nToken; + int nToken = 0; sqlite3_tokenizer *pTokenizer = p->pTokenizer; sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; @@ -120271,11 +140335,10 @@ static int fts3PendingTermsAdd( return SQLITE_OK; } - rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); if( rc!=SQLITE_OK ){ return rc; } - pCsr->pTokenizer = pTokenizer; xNext = pModule->xNext; while( SQLITE_OK==rc @@ -120309,7 +140372,7 @@ static int fts3PendingTermsAdd( } pModule->xClose(pCsr); - *pnWord = nWord; + *pnWord += nWord; return (rc==SQLITE_DONE ? SQLITE_OK : rc); } @@ -120318,18 +140381,28 @@ static int fts3PendingTermsAdd( ** fts3PendingTermsAdd() are to add term/position-list pairs for the ** contents of the document with docid iDocid. */ -static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + /* TODO(shess) Explore whether partially flushing the buffer on ** forced-flush would provide better performance. I suspect that if ** we ordered the doclists by size and flushed the largest until the ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ + if( iDocid<=p->iPrevDocid + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; return SQLITE_OK; } @@ -120358,15 +140431,23 @@ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ ** Argument apVal is the same as the similarly named argument passed to ** fts3InsertData(). Parameter iDocid is the docid of the new row. */ -static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ - const char *zText = (const char *)sqlite3_value_text(apVal[i]); - int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); - if( rc!=SQLITE_OK ){ - return rc; + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } - aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } return SQLITE_OK; } @@ -120383,6 +140464,7 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ ** apVal[p->nColumn+1] Right-most user-defined column ** apVal[p->nColumn+2] Hidden column with same name as table ** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column */ static int fts3InsertData( Fts3Table *p, /* Full-text table */ @@ -120392,6 +140474,18 @@ static int fts3InsertData( int rc; /* Return code */ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + /* Locate the statement handle used to insert data into the %_content ** table. The SQL for this statement is: ** @@ -120401,9 +140495,13 @@ static int fts3InsertData( ** defined columns in the FTS3 table, plus one for the docid field. */ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); - if( rc!=SQLITE_OK ){ - return rc; + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); } + if( rc!=SQLITE_OK ) return rc; /* There is a quirk here. The users INSERT statement may have specified ** a value for the "rowid" field, for the "docid" field, or for both. @@ -120442,14 +140540,16 @@ static int fts3InsertData( ** Remove all data from the FTS3 table. Clear the hash table containing ** pending terms. */ -static int fts3DeleteAll(Fts3Table *p){ +static int fts3DeleteAll(Fts3Table *p, int bContent){ int rc = SQLITE_OK; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); - /* Delete everything from the %_content, %_segments and %_segdir tables. */ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); if( p->bHasDocsize ){ @@ -120461,6 +140561,15 @@ static int fts3DeleteAll(Fts3Table *p){ return rc; } +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + /* ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the @@ -120470,26 +140579,34 @@ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value *pRowid, /* The docid to be deleted */ - u32 *aSz /* Sizes of deleted document written here */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ ){ int rc; sqlite3_stmt *pSelect; + assert( *pbFound==0 ); if( *pRC ) return; rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; - for(i=1; i<=p->nColumn; i++){ - const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); - if( rc!=SQLITE_OK ){ - sqlite3_reset(pSelect); - *pRC = rc; - return; + int iLangid = langidFromSelect(p, pSelect); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } - aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; } rc = sqlite3_reset(pSelect); }else{ @@ -120502,7 +140619,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int, int); +static int fts3SegmentMerge(Fts3Table *, int, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -120521,6 +140638,7 @@ static int fts3SegmentMerge(Fts3Table *, int, int); */ static int fts3AllocateSegdirIdx( Fts3Table *p, + int iLangid, /* Language id */ int iIndex, /* Index for p->aIndex */ int iLevel, int *piIdx @@ -120529,10 +140647,15 @@ static int fts3AllocateSegdirIdx( sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -120546,7 +140669,8 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iIndex, iLevel); + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -120593,7 +140717,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( int rc; /* Return code */ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ - assert( pnBlob); + assert( pnBlob ); if( p->pSegments ){ rc = sqlite3_blob_reopen(p->pSegments, iBlockid); @@ -120679,6 +140803,18 @@ static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ return rc; } +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; + } + pSeg->aNode = 0; +} + /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, @@ -120718,12 +140854,7 @@ static int fts3SegReaderNext( return SQLITE_OK; } - if( !fts3SegReaderIsRootOnly(pReader) ){ - sqlite3_free(pReader->aNode); - sqlite3_blob_close(pReader->pBlob); - pReader->pBlob = 0; - } - pReader->aNode = 0; + fts3SegReaderSetEof(pReader); /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf ** blocks have already been traversed. */ @@ -120752,12 +140883,12 @@ static int fts3SegReaderNext( /* Because of the FTS3_NODE_PADDING bytes of padding, the following is ** safe (no risk of overread) even if the node data is corrupted. */ - pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); - pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } if( nPrefix+nSuffix>pReader->nTermAlloc ){ @@ -120776,7 +140907,7 @@ static int fts3SegReaderNext( memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; - pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); pReader->aDoclist = pNext; pReader->pOffsetList = 0; @@ -120787,7 +140918,7 @@ static int fts3SegReaderNext( if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } return SQLITE_OK; } @@ -120869,7 +141000,7 @@ static int fts3SegReaderNextDocid( /* The following line of code (and the "p++" below the while() loop) is ** normally all that is required to move pointer p to the desired ** position. The exception is if this node is being loaded from disk - ** incrementally and pointer "p" now points to the first byte passed + ** incrementally and pointer "p" now points to the first byte past ** the populated part of pReader->aNode[]. */ while( *p | c ) c = *p++ & 0x80; @@ -120889,6 +141020,7 @@ static int fts3SegReaderNextDocid( *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } + /* List may have been edited in place by fts3EvalNearTrim() */ while( pnPgsz; - assert( p->bHasStat ); + assert( p->bFts4 ); assert( pgsz>0 ); for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ @@ -120970,6 +141102,7 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ */ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ sqlite3_int64 iStartLeaf, /* First leaf to traverse */ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ sqlite3_int64 iEndBlock, /* Final block of segment */ @@ -120977,7 +141110,6 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( int nRoot, /* Size of buffer containing root node */ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ ){ - int rc = SQLITE_OK; /* Return code */ Fts3SegReader *pReader; /* Newly allocated SegReader object */ int nExtra = 0; /* Bytes to allocate segment root node */ @@ -120992,6 +141124,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( } memset(pReader, 0, sizeof(Fts3SegReader)); pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; pReader->iStartBlock = iStartLeaf; pReader->iLeafEndBlock = iEndLeaf; pReader->iEndBlock = iEndBlock; @@ -120999,19 +141132,15 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( if( nExtra ){ /* The entire segment is stored in the root node. */ pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; pReader->nNode = nRoot; memcpy(pReader->aNode, zRoot, nRoot); memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); }else{ pReader->iCurrentBlock = iStartLeaf-1; } - - if( rc==SQLITE_OK ){ - *ppReader = pReader; - }else{ - sqlite3Fts3SegReaderFree(pReader); - } - return rc; + *ppReader = pReader; + return SQLITE_OK; } /* @@ -121061,6 +141190,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ @@ -121069,7 +141199,6 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( pHash = &p->aIndex[iIndex].hPending; if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ - Fts3HashElem *pE = 0; /* Iterator variable */ for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); @@ -121103,8 +141232,13 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( }else{ /* The query is a simple term lookup that matches at most one term in - ** the index. All that is required is a straight hash-lookup. */ - Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; @@ -121284,27 +141418,55 @@ static int fts3WriteSegment( return rc; } +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ - int iLevel, /* Value for "level" field */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iLevel); + sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - sqlite3_bind_int64(pStmt, 5, iEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); @@ -121600,6 +141762,7 @@ static int fts3SegWriterAdd( /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; /* Add the current term to the interior node tree. The term added to ** the interior tree must: @@ -121629,6 +141792,9 @@ static int fts3SegWriterAdd( nDoclist; /* Doclist data */ } + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ @@ -121683,7 +141849,7 @@ static int fts3SegWriterAdd( static int fts3SegWriterFlush( Fts3Table *p, /* Virtual table handle */ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ - int iLevel, /* Value for 'level' column of %_segdir */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ int iIdx /* Value for 'idx' column of %_segdir */ ){ int rc; /* Return code */ @@ -121700,14 +141866,15 @@ static int fts3SegWriterFlush( pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir( - p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir( - p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } + p->nLeafAdd++; return rc; } @@ -121737,12 +141904,18 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pisEmpty = sqlite3_column_int(pStmt, 0); + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); } - rc = sqlite3_reset(pStmt); } return rc; } @@ -121755,7 +141928,12 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ sqlite3_stmt *pStmt; int rc; assert( iIndex>=0 && iIndexnIndex ); @@ -121768,14 +141946,70 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnMax = sqlite3_column_int(pStmt, 0); + *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + /* ** This function is used after merging multiple segments into a single large ** segment to delete the old, now redundant, segment b-trees. Specifically, @@ -121792,24 +142026,18 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ int i; /* Iterator variable */ - sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); for(i=0; rc==SQLITE_OK && iiStartBlock ){ - sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); - sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } + rc = fts3DeleteSegment(p, apSegment[i]); } if( rc!=SQLITE_OK ){ return rc; @@ -121819,13 +142047,17 @@ static int fts3DeleteSegdir( if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); } } @@ -121845,9 +142077,13 @@ static int fts3DeleteSegdir( ** ** If there are no entries in the input position list for column iCol, then ** *pnList is set to zero before returning. +** +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. */ static void fts3ColumnFilter( int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ char **ppList, /* IN/OUT: Pointer to position list */ int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ ){ @@ -121873,9 +142109,12 @@ static void fts3ColumnFilter( break; } p = &pList[1]; - p += sqlite3Fts3GetVarint32(p, &iCurrent); + p += fts3GetVarint32(p, &iCurrent); } + if( bZero && &pList[nList]!=pEnd ){ + memset(&pList[nList], 0, pEnd - &pList[nList]); + } *ppList = pList; *pnList = nList; } @@ -121949,19 +142188,19 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( if( rc!=SQLITE_OK ) return rc; fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + if( pMsr->iColFilter>=0 ){ - fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); } if( nList>0 ){ - if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); - if( rc!=SQLITE_OK ) return rc; - *paPoslist = pMsr->aBuffer; - assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); - }else{ - *paPoslist = pList; - } + *paPoslist = pList; *piDocid = iDocid; *pnPoslist = nList; break; @@ -121988,11 +142227,16 @@ static int fts3SegReaderStart( ** b-tree leaf nodes contain more than one term. */ for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; Fts3SegReader *pSeg = pCsr->apSegment[i]; do { int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; - }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); + } } fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); @@ -122094,6 +142338,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; @@ -122112,7 +142357,12 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( ** forward. Then sort the list in order of current term again. */ for(i=0; inAdvance; i++){ - rc = fts3SegReaderNext(p, apSegment[i], 0); + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); @@ -122153,6 +142403,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty + && !isFirst && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ pCsr->nDoclist = apSegment[0]->nDoclist; @@ -122177,8 +142428,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp); while( apSegment[0]->pOffsetList ){ int j; /* Number of segments that share a docid */ - char *pList; - int nList; + char *pList = 0; + int nList = 0; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); @@ -122192,7 +142443,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } if( isColFilter ){ - fts3ColumnFilter(pFilter->iCol, &pList, &nList); + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ @@ -122218,12 +142469,24 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); - iPrev = iDocid; - if( isRequirePos ){ - memcpy(&pCsr->aBuffer[nDoclist], pList, nList); - nDoclist += nList; - pCsr->aBuffer[nDoclist++] = '\0'; + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } } } @@ -122259,6 +142522,140 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( } } +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = (iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a @@ -122270,14 +142667,20 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level/index to create new segment at */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING @@ -122286,36 +142689,41 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ assert( iLevel=0 && iIndexnIndex ); - rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single - ** segment. The level of the new segment is equal to the the numerically + ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); + iNewLevel = iMaxLevel; bIgnoreEmpty = 1; - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; - rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); - iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; + assert( FTS3_SEGCURSOR_PENDING==-1 ); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); - assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); - assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert( iNewLevelnLeafData); + } + } + } finished: fts3SegWriterFree(pWriter); @@ -122345,16 +142762,38 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ /* -** Flush the contents of pendingTerms to level 0 segments. +** Flush the contents of pendingTerms to level 0 segments. */ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } + rc = sqlite3_reset(pStmt); + } + } return rc; } @@ -122399,9 +142838,9 @@ static void fts3DecodeIntArray( ** a blob of varints. */ static void fts3InsertDocsize( - int *pRC, /* Result code */ - Fts3Table *p, /* Table into which to insert */ - u32 *aSz /* Sizes of each column */ + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ ){ char *pBlob; /* The BLOB encoding of the document size */ int nBlob; /* Number of bytes in the BLOB */ @@ -122465,12 +142904,13 @@ static void fts3UpdateDocTotals( return; } pBlob = (char*)&a[nStat]; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); if( sqlite3_step(pStmt)==SQLITE_ROW ){ fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), @@ -122478,7 +142918,12 @@ static void fts3UpdateDocTotals( }else{ memset(a, 0, sizeof(u32)*(nStat) ); } - sqlite3_reset(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } if( nChng<0 && a[0]<(u32)(-nChng) ){ a[0] = 0; }else{ @@ -122494,35 +142939,1845 @@ static void fts3UpdateDocTotals( a[i+1] = x; } fts3EncodeIntArray(nStat, a, pBlob, &nBlob); - rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } - sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); sqlite3_free(a); } +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ - int i; int bSeenDone = 0; - int rc = SQLITE_OK; - for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - bSeenDone = 1; - rc = SQLITE_OK; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; } + sqlite3Fts3SegmentsClose(p); sqlite3Fts3PendingTermsClear(p); return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + int nByte = sizeof(u32) * (p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + int nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert( p->iOff<=p->nNode ); + + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( p->aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return nodeReaderNext(p); +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + pWriter->nLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); + } + + return rc; +} + +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ + + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + res = memcmp(zLhs, zRhs, nCmp); + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + } + sqlite3_free(aBlock); + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; + } + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + } + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } + } + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return SQLITE_CORRUPT_VTAB; + + return SQLITE_OK; +} + + +/* +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. +*/ +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = nMin; + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) + ){ + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (FTS3_MERGE_COUNT / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); + } + return rc; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + i64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrnIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + } + + sqlite3_finalize(pStmt); + } + + *pbOk = (cksum1==cksum2); + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB; + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -122540,6 +144795,14 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -122547,6 +144810,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ p->nMaxPendingData = atoi(&zVal[11]); rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; @@ -122555,6 +144821,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ return rc; } +#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. @@ -122605,32 +144872,34 @@ SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ iDocid = sqlite3_column_int64(pCsr->pStmt, 0); for(i=0; inColumn && rc==SQLITE_OK; i++){ - const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); - sqlite3_tokenizer_cursor *pTC = 0; - - rc = pModule->xOpen(pT, zText, -1, &pTC); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken; /* Number of bytes in token */ - int iDum1, iDum2; /* Dummy variables */ - int iPos; /* Position of token in zText */ - - pTC->pTokenizer = pT; - rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); - for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - Fts3PhraseToken *pPT = pDef->pToken; - if( (pDef->iCol>=p->nColumn || pDef->iCol==i) - && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) - ){ - fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } } } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - if( pTC ) pModule->xClose(pTC); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ if( pDef->pList ){ rc = fts3PendingListAppendVarint(&pDef->pList, 0); @@ -122692,6 +144961,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( return SQLITE_OK; } +#endif /* ** SQLite value pRowid contains the rowid of a row that may or may not be @@ -122701,26 +144971,32 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( static int fts3DeleteByRowid( Fts3Table *p, sqlite3_value *pRowid, - int *pnDoc, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ u32 *aSzDel ){ - int isEmpty = 0; - int rc = fts3IsEmpty(p, pRowid, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p); - *pnDoc = *pnDoc - 1; - }else{ - sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); - rc = fts3PendingTermsDocid(p, iRemove); - fts3DeleteTerms(&rc, p, pRowid, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } } } } @@ -122730,7 +145006,16 @@ static int fts3DeleteByRowid( /* ** This function does the work for the xUpdate method of FTS3 virtual -** tables. +** tables. The schema of the virtual table being: +** +** CREATE TABLE
( +** , +**
HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** */ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ @@ -122741,13 +145026,20 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ - sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ u32 *aSzIns = 0; /* Sizes of inserted documents */ - u32 *aSzDel; /* Sizes of deleted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); /* Check for a "special" INSERT operation. One of the form: ** @@ -122761,14 +145053,22 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( goto update_out; } + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } + /* Allocate space to hold the change in document sizes */ - aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ){ + aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; } - aSzDel = &aSzIns[p->nColumn+1]; - memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); + + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; /* If this is an INSERT operation, or an UPDATE that modifies the rowid ** value, then this operation requires constraint handling. @@ -122779,7 +145079,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( ** detect the conflict and return SQLITE_CONSTRAINT before beginning to ** modify the database file. */ - if( nArg>1 ){ + if( nArg>1 && p->zContentTbl==0 ){ /* Find the value object that holds the new rowid value. */ sqlite3_value *pNewRowid = apVal[3+p->nColumn]; if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ @@ -122824,20 +145124,23 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); isRemove = 1; - iRemove = sqlite3_value_int64(apVal[0]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); if( bInsertDone==0 ){ rc = fts3InsertData(p, apVal, pRowid); - if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } } - if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ - rc = fts3PendingTermsDocid(p, *pRowid); + if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ + rc = fts3PendingTermsDocid(p, iLangid, *pRowid); } if( rc==SQLITE_OK ){ - rc = fts3InsertTerms(p, apVal, aSzIns); + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); } if( p->bHasDocsize ){ fts3InsertDocsize(&rc, p, aSzIns); @@ -122845,12 +145148,12 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( nChng++; } - if( p->bHasStat ){ + if( p->bFts4 ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } update_out: - sqlite3_free(aSzIns); + sqlite3_free(aSzDel); sqlite3Fts3SegmentsClose(p); return rc; } @@ -122896,6 +145199,8 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ +/* #include */ /* ** Characters that may appear in the second argument to matchinfo(). @@ -123008,7 +145313,7 @@ struct StrBuffer { */ static void fts3GetDeltaPosition(char **pp, int *piPos){ int iVal; - *pp += sqlite3Fts3GetVarint32(*pp, &iVal); + *pp += fts3GetVarint32(*pp, &iVal); *piPos += (iVal-2); } @@ -123240,23 +145545,27 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ SnippetIter *p = (SnippetIter *)ctx; SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; char *pCsr; + int rc; pPhrase->nToken = pExpr->pPhrase->nToken; - - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; fts3GetDeltaPosition(&pCsr, &iFirst); + assert( iFirst>=0 ); pPhrase->pHead = pCsr; pPhrase->pTail = pCsr; pPhrase->iHead = iFirst; pPhrase->iTail = iFirst; }else{ - assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 ); + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); } - return SQLITE_OK; + return rc; } /* @@ -123265,9 +145574,9 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ ** is the snippet with the highest score, where scores are calculated ** by adding: ** -** (a) +1 point for each occurence of a matchable phrase in the snippet. +** (a) +1 point for each occurrence of a matchable phrase in the snippet. ** -** (b) +1000 points for the first occurence of each matchable phrase in +** (b) +1000 points for the first occurrence of each matchable phrase in ** the snippet for which the corresponding mCovered bit is not set. ** ** The selected snippet parameters are stored in structure *pFragment before @@ -123380,6 +145689,7 @@ static int fts3StringAppend( pStr->z = zNew; pStr->nAlloc = nAlloc; } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); /* Append the data to the string buffer. */ memcpy(&pStr->z[pStr->n], zAppend, nAppend); @@ -123411,6 +145721,7 @@ static int fts3StringAppend( */ static int fts3SnippetShift( Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ int nSnippet, /* Number of tokens desired for snippet */ const char *zDoc, /* Document text to extract snippet from */ int nDoc, /* Size of buffer zDoc in bytes */ @@ -123446,13 +145757,12 @@ static int fts3SnippetShift( /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) ** or more tokens in zDoc/nDoc. */ - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); if( rc!=SQLITE_OK ){ return rc; } - pC->pTokenizer = pTab->pTokenizer; while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ - const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3; + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); } pMod->xClose(pC); @@ -123496,8 +145806,6 @@ static int fts3SnippetText( int iCol = pFragment->iCol+1; /* Query column to extract text from */ sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ - const char *ZDUMMY; /* Dummy argument used with tokenizer */ - int DUMMY1; /* Dummy argument used with tokenizer */ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); if( zDoc==0 ){ @@ -123510,17 +145818,29 @@ static int fts3SnippetText( /* Open a token cursor on the document. */ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); if( rc!=SQLITE_OK ){ return rc; } - pC->pTokenizer = pTab->pTokenizer; while( rc==SQLITE_OK ){ - int iBegin; /* Offset in zDoc of start of token */ - int iFin; /* Offset in zDoc of end of token */ - int isHighlight; /* True for highlighted terms */ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ){ @@ -123536,7 +145856,9 @@ static int fts3SnippetText( if( !isShiftDone ){ int n = nDoc - iBegin; - rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask); + rc = fts3SnippetShift( + pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); isShiftDone = 1; /* Now that the shift has been done, check if the initial "..." are @@ -123648,13 +145970,14 @@ static int fts3ExprLocalHitsCb( int iPhrase, /* Phrase number */ void *pCtx /* Pointer to MatchInfo structure */ ){ + int rc = SQLITE_OK; MatchInfo *p = (MatchInfo *)pCtx; int iStart = iPhrase * p->nCol * 3; int i; - for(i=0; inCol; i++){ + for(i=0; inCol && rc==SQLITE_OK; i++){ char *pCsr; - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); if( pCsr ){ p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); }else{ @@ -123662,7 +145985,7 @@ static int fts3ExprLocalHitsCb( } } - return SQLITE_OK; + return rc; } static int fts3MatchinfoCheck( @@ -123672,8 +145995,8 @@ static int fts3MatchinfoCheck( ){ if( (cArg==FTS3_MATCHINFO_NPHRASE) || (cArg==FTS3_MATCHINFO_NCOL) - || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat) - || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) || (cArg==FTS3_MATCHINFO_LCS) || (cArg==FTS3_MATCHINFO_HITS) @@ -123728,7 +146051,7 @@ static int fts3MatchinfoSelectDoctotal( a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); - if( nDoc==0 ) return SQLITE_CORRUPT_VTAB; + if( nDoc==0 ) return FTS_CORRUPT_VTAB; *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; @@ -123823,8 +146146,10 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int nLive = 0; /* Number of iterators in aIter not at EOF */ for(i=0; inPhrase; i++){ + int rc; LcsIterator *pIt = &aIter[i]; - pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) return rc; if( pIt->pRead ){ pIt->iPos = pIt->iPosOffset; fts3LcsIteratorAdvance(&aIter[i]); @@ -124176,9 +146501,10 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iTerm; /* For looping through nTerm phrase terms */ char *pList; /* Pointer to position list for phrase */ int iPos = 0; /* First position in position-list */ + int rc; UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); @@ -124192,7 +146518,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ pT->iPos = iPos; } - return SQLITE_OK; + return rc; } /* @@ -124204,8 +146530,6 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; - const char *ZDUMMY; /* Dummy argument used with xNext() */ - int NDUMMY; /* Dummy argument used with xNext() */ int rc; /* Return Code */ int nToken; /* Number of tokens in query */ int iCol; /* Column currently being processed */ @@ -124238,9 +146562,11 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( */ for(iCol=0; iColnColumn; iCol++){ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ - int iStart; - int iEnd; - int iCurrent; + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; const char *zDoc; int nDoc; @@ -124269,9 +146595,10 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( } /* Initialize a tokenizer iterator to iterate through column iCol. */ - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); if( rc!=SQLITE_OK ) goto offsets_out; - pC->pTokenizer = pTab->pTokenizer; rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); while( rc==SQLITE_OK ){ @@ -124289,7 +146616,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( if( !pTerm ){ /* All offsets for this column have been gathered. */ - break; + rc = SQLITE_DONE; }else{ assert( iCurrent<=iMinPos ); if( 0==(0xFE&*pTerm->pList) ){ @@ -124306,8 +146633,8 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart ); rc = fts3StringAppend(&res, aBuffer, -1); - }else if( rc==SQLITE_DONE ){ - rc = SQLITE_CORRUPT_VTAB; + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; } } } @@ -124379,6 +146706,768 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( #endif /************** End of fts3_snippet.c ****************************************/ +/************** Begin file fts3_unicode.c ************************************/ +/* +** 2012 May 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "unicode" full-text-search tokenizer. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; + +struct unicode_tokenizer { + sqlite3_tokenizer base; + int bRemoveDiacritic; + int nException; + int *aiException; +}; + +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; + + +/* +** Destroy a tokenizer allocated by unicodeCreate(). +*/ +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); + } + return SQLITE_OK; +} + +/* +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). +** +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. +** +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. +*/ +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + int iCode; + int nEntry = 0; + + assert( bAlnum==0 || bAlnum==1 ); + + while( zaiException, (p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; + + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = iCode; + nNew++; + } + } + p->aiException = aNew; + p->nException = nNew; + } + + return SQLITE_OK; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). +*/ +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); +} + +/* +** Create a new tokenizer instance. +*/ +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ +){ + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; + + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->bRemoveDiacritic = 1; + + for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; + } + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->bRemoveDiacritic = 0; + } + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); + } + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); + } + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; + } + } + + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; + return rc; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +){ + unicode_cursor *pCsr; + + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(unicode_cursor)); + + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; + } + + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ +){ + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; + + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( z=zTerm ) return SQLITE_DONE; + + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; + } + + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); + } + + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, iCode) + || sqlite3FtsUnicodeIsdiacritic(iCode) + ); + + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; + return SQLITE_OK; +} + +/* +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. +*/ +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ + +/************** End of fts3_unicode.c ****************************************/ +/************** Begin file fts3_unicode2.c ***********************************/ +/* +** 2012 May 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + +/* #include */ + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( c<128 ){ + return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + }else if( c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1; +} + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int remove_diacritic(int c){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, + 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, + 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, + 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, + 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, + 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, + 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, + 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, + 62924, 63050, 63082, 63274, 63390, + }; + char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', + 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', + 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', + 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', + 'e', 'i', 'o', 'u', 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & (1 << (c-768))) : + (mask1 & (1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( c>=0 ); + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( iRes<0 || c>=aEntry[iRes].iCode ); + + if( iRes>=0 ){ + const struct TableEntry *p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + } + + if( bRemoveDiacritic ) ret = remove_diacritic(ret); + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ + +/************** End of fts3_unicode2.c ***************************************/ /************** Begin file rtree.c *******************************************/ /* ** 2001 September 15 @@ -124436,58 +147525,20 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) -/* -** This file contains an implementation of a couple of different variants -** of the r-tree algorithm. See the README file for further details. The -** same data-structure is used for all, but the algorithms for insert and -** delete operations vary. The variants used are selected at compile time -** by defining the following symbols: -*/ - -/* Either, both or none of the following may be set to activate -** r*tree variant algorithms. -*/ -#define VARIANT_RSTARTREE_CHOOSESUBTREE 0 -#define VARIANT_RSTARTREE_REINSERT 1 - -/* -** Exactly one of the following must be set to 1. -*/ -#define VARIANT_GUTTMAN_QUADRATIC_SPLIT 0 -#define VARIANT_GUTTMAN_LINEAR_SPLIT 0 -#define VARIANT_RSTARTREE_SPLIT 1 - -#define VARIANT_GUTTMAN_SPLIT \ - (VARIANT_GUTTMAN_LINEAR_SPLIT||VARIANT_GUTTMAN_QUADRATIC_SPLIT) - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT - #define PickNext QuadraticPickNext - #define PickSeeds QuadraticPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_GUTTMAN_LINEAR_SPLIT - #define PickNext LinearPickNext - #define PickSeeds LinearPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_RSTARTREE_SPLIT - #define AssignCells splitNodeStartree -#endif - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #else #endif +/* #include */ +/* #include */ +/* #include */ #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; +typedef unsigned short u16; typedef unsigned int u32; #endif @@ -124505,6 +147556,7 @@ typedef struct RtreeConstraint RtreeConstraint; typedef struct RtreeMatchArg RtreeMatchArg; typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ #define RTREE_MAX_DIMENSIONS 5 @@ -124513,22 +147565,33 @@ typedef union RtreeCoord RtreeCoord; ** ever contain very many entries, so a fixed number of buckets is ** used. */ -#define HASHSIZE 128 +#define HASHSIZE 97 + +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 /* ** An rtree virtual-table object. */ struct Rtree { - sqlite3_vtab base; + sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ - int nDim; /* Number of dimensions */ - int nBytesPerCell; /* Bytes consumed per cell */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ int nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ /* List of nodes removed during a CondenseTree operation. List is ** linked together via the pointer normally used for hash chains - @@ -124553,13 +147616,45 @@ struct Rtree { sqlite3_stmt *pWriteParent; sqlite3_stmt *pDeleteParent; - int eCoordType; + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ }; -/* Possible values for eCoordType: */ +/* Possible values for Rtree.eCoordType: */ #define RTREE_COORD_REAL32 0 #define RTREE_COORD_INT32 1 +/* +** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will +** only deal with integer coordinates. No floating point operations +** will be done. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ + typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 +#else + typedef double RtreeDValue; /* High accuracy coordinate */ + typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 +#endif + +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + /* ** The minimum number of cells allowed for a node is a third of the ** maximum. In Gutman's notation: @@ -124582,33 +147677,61 @@ struct Rtree { */ #define RTREE_MAX_DEPTH 40 + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + /* ** An rtree cursor object. */ struct RtreeCursor { - sqlite3_vtab_cursor base; - RtreeNode *pNode; /* Node cursor is currently pointing at */ - int iCell; /* Index of current cell in pNode */ + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ int iStrategy; /* Copy of idxNum search parameter */ int nConstraint; /* Number of entries in aConstraint */ RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ }; +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ union RtreeCoord { - float f; - int i; + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ }; /* ** The argument is an RtreeCoord. Return the value stored within the RtreeCoord -** formatted as a double. This macro assumes that local variable pRtree points -** to the Rtree structure associated with the RtreeCoord. +** formatted as a RtreeDValue (double or int64). This macro assumes that local +** variable pRtree points to the Rtree structure associated with the +** RtreeCoord. */ -#define DCOORD(coord) ( \ - (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ - ((double)coord.f) : \ - ((double)coord.i) \ -) +#ifdef SQLITE_RTREE_INT_ONLY +# define DCOORD(coord) ((RtreeDValue)coord.i) +#else +# define DCOORD(coord) ( \ + (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ + ((double)coord.f) : \ + ((double)coord.i) \ + ) +#endif /* ** A search constraint. @@ -124616,38 +147739,67 @@ union RtreeCoord { struct RtreeConstraint { int iCoord; /* Index of constrained coordinate */ int op; /* Constraining operation */ - double rValue; /* Constraint value. */ - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); - sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 -#define RTREE_MATCH 0x46 +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + /* ** An rtree structure node. */ struct RtreeNode { - RtreeNode *pParent; /* Parent node */ - i64 iNode; - int nRef; - int isDirty; - u8 *zData; - RtreeNode *pNext; /* Next node in this hash chain */ + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ }; + +/* Return the number of cells in a node */ #define NCELL(pNode) readInt16(&(pNode)->zData[2]) /* -** Structure to store a deserialized rtree record. +** A single cell from a node, deserialized */ struct RtreeCell { - i64 iRowid; - RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; }; @@ -124659,29 +147811,16 @@ struct RtreeCell { #define RTREE_GEOMETRY_MAGIC 0x891245AB /* -** An instance of this structure must be supplied as a blob argument to -** the right-hand-side of an SQL MATCH operator used to constrain an -** r-tree query. +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. */ struct RtreeMatchArg { - u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); - void *pContext; - int nParam; - double aParam[1]; -}; - -/* -** When a geometry callback is created (see sqlite3_rtree_geometry_callback), -** a single instance of the following structure is allocated. It is used -** as the context for the user-function created by by s_r_g_c(). The object -** is eventually deleted by the destructor mechanism provided by -** sqlite3_create_function_v2() (which is called by s_r_g_c() to create -** the geometry callback function). -*/ -struct RtreeGeomCallback { - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); - void *pContext; + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ }; #ifndef MAX @@ -124775,10 +147914,7 @@ static void nodeZero(Rtree *pRtree, RtreeNode *p){ ** in the Rtree.aHash table. */ static int nodeHash(i64 iNode){ - return ( - (iNode>>56) ^ (iNode>>48) ^ (iNode>>40) ^ (iNode>>32) ^ - (iNode>>24) ^ (iNode>>16) ^ (iNode>> 8) ^ (iNode>> 0) - ) % HASHSIZE; + return iNode % HASHSIZE; } /* @@ -124838,8 +147974,7 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ /* ** Obtain a reference to an r-tree node. */ -static int -nodeAcquire( +static int nodeAcquire( Rtree *pRtree, /* R-tree structure */ i64 iNode, /* Node number to load */ RtreeNode *pParent, /* Either the parent node or NULL */ @@ -124928,10 +148063,10 @@ nodeAcquire( ** Overwrite cell iCell of node pNode with the contents of pCell. */ static void nodeOverwriteCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ ){ int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -124943,7 +148078,7 @@ static void nodeOverwriteCell( } /* -** Remove cell the cell with index iCell from node pNode. +** Remove the cell with index iCell from node pNode. */ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -124960,11 +148095,10 @@ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ ** ** If there is not enough free space in pNode, return SQLITE_FULL. */ -static int -nodeInsertCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ ){ int nCell; /* Current number of cells in pNode */ int nMaxCell; /* Maximum number of cells for pNode */ @@ -124985,8 +148119,7 @@ nodeInsertCell( /* ** If the node is dirty, write it out to the database. */ -static int -nodeWrite(Rtree *pRtree, RtreeNode *pNode){ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode->isDirty ){ sqlite3_stmt *p = pRtree->pWriteNode; @@ -125011,8 +148144,7 @@ nodeWrite(Rtree *pRtree, RtreeNode *pNode){ ** Release a reference to a node. If the node is dirty and the reference ** count drops to zero, the node data is written to the database. */ -static int -nodeRelease(Rtree *pRtree, RtreeNode *pNode){ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode ){ assert( pNode->nRef>0 ); @@ -125040,9 +148172,9 @@ nodeRelease(Rtree *pRtree, RtreeNode *pNode){ ** an internal node, then the 64-bit integer is a child page number. */ static i64 nodeGetRowid( - Rtree *pRtree, - RtreeNode *pNode, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ ){ assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); @@ -125052,11 +148184,11 @@ static i64 nodeGetRowid( ** Return coordinate iCoord from cell iCell in node pNode. */ static void nodeGetCoord( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - int iCoord, - RtreeCoord *pCoord /* Space to write result to */ + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ ){ readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -125066,15 +148198,20 @@ static void nodeGetCoord( ** to by pCell with the results. */ static void nodeGetCell( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - RtreeCell *pCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ ){ - int ii; + u8 *pData; + u8 *pEnd; + RtreeCoord *pCoord; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); - for(ii=0; iinDim*2; ii++){ - nodeGetCoord(pRtree, pNode, iCell, ii, &pCell->aCoord[ii]); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pEnd = pData + pRtree->nDim*8; + pCoord = pCell->aCoord; + for(; pDataaConstraint ){ int i; /* Used to iterate through constraint array */ for(i=0; inConstraint; i++){ - sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; - if( pGeom ){ - if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); - sqlite3_free(pGeom); + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); } } sqlite3_free(pCsr->aConstraint); @@ -125216,12 +148353,13 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); - int rc; + int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; freeCursorConstraints(pCsr); - rc = nodeRelease(pRtree, pCsr->pNode); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); - return rc; + return SQLITE_OK; } /* @@ -125232,196 +148370,166 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ */ static int rtreeEof(sqlite3_vtab_cursor *cur){ RtreeCursor *pCsr = (RtreeCursor *)cur; - return (pCsr->pNode==0); + return pCsr->atEOF; } /* -** The r-tree constraint passed as the second argument to this function is -** guaranteed to be a MATCH constraint. -*/ -static int testRtreeGeom( - Rtree *pRtree, /* R-Tree object */ - RtreeConstraint *pConstraint, /* MATCH constraint to test */ - RtreeCell *pCell, /* Cell to test */ - int *pbRes /* OUT: Test result */ -){ - int i; - double aCoord[RTREE_MAX_DIMENSIONS*2]; - int nCoord = pRtree->nDim*2; - - assert( pConstraint->op==RTREE_MATCH ); - assert( pConstraint->pGeom ); - - for(i=0; iaCoord[i]); - } - return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); -} - -/* -** Cursor pCursor currently points to a cell in a non-leaf page. -** Set *pbEof to true if the sub-tree headed by the cell is filtered -** (excluded) by the constraints in the pCursor->aConstraint[] -** array, or false otherwise. +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". ** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. */ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - int bRes = 0; - int rc = SQLITE_OK; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; bRes==0 && iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - double cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); - double cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); - - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - - switch( p->op ){ - case RTREE_LE: case RTREE_LT: - bRes = p->rValuerValue>cell_max; - break; - - case RTREE_EQ: - bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &bRes); - bRes = !bRes; - break; - } - } - } - - *pbEof = bRes; - return rc; +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } - -/* -** Test if the cell that cursor pCursor currently points to -** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array. If so, set *pbEof to true before -** returning. If the cell is not filtered (excluded) by the constraints, -** set pbEof to zero. -** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. -** -** This function assumes that the cell is part of a leaf node. -*/ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - *pbEof = 0; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - double coord = DCOORD(cell.aCoord[p->iCoord]); - int res; - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - switch( p->op ){ - case RTREE_LE: res = (coord<=p->rValue); break; - case RTREE_LT: res = (coordrValue); break; - case RTREE_GE: res = (coord>=p->rValue); break; - case RTREE_GT: res = (coord>p->rValue); break; - case RTREE_EQ: res = (coord==p->rValue); break; - default: { - int rc; - assert( p->op==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &res); - if( rc!=SQLITE_OK ){ - return rc; - } - break; - } - } - - if( !res ){ - *pbEof = 1; - return SQLITE_OK; - } - } - - return SQLITE_OK; +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif /* -** Cursor pCursor currently points at a node that heads a sub-tree of -** height iHeight (if iHeight==0, then the node is a leaf). Descend -** to point to the left-most cell of the sub-tree that matches the -** configured constraints. +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. */ -static int descendToCell( - Rtree *pRtree, - RtreeCursor *pCursor, - int iHeight, - int *pEof /* OUT: Set to true if cannot descend */ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ ){ - int isEof; - int rc; - int ii; - RtreeNode *pChild; - sqlite3_int64 iRowid; + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ - RtreeNode *pSavedNode = pCursor->pNode; - int iSavedCell = pCursor->iCell; + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); - assert( iHeight>=0 ); - - if( iHeight==0 ){ - rc = testRtreeEntry(pRtree, pCursor, &isEof); + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; + for(i=0; iop==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; }else{ - rc = testRtreeCell(pRtree, pCursor, &isEof); - } - if( rc!=SQLITE_OK || isEof || iHeight==0 ){ - goto descend_to_cell_out; - } - - iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); - rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; - } - - nodeRelease(pRtree, pCursor->pNode); - pCursor->pNode = pChild; - isEof = 1; - for(ii=0; isEof && iiiCell = ii; - rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; } } - - if( isEof ){ - assert( pCursor->pNode==pChild ); - nodeReference(pSavedNode); - nodeRelease(pRtree, pChild); - pCursor->pNode = pSavedNode; - pCursor->iCell = iSavedCell; - } - -descend_to_cell_out: - *pEof = isEof; return rc; } +/* +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. +*/ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ + + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ + + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + *peWithin = NOT_WITHIN; +} + +/* +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. +*/ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; + } + *peWithin = NOT_WITHIN; +} + /* ** One of the cells in node pNode is guaranteed to have a 64-bit ** integer value equal to iRowid. Return the index of this cell. @@ -125434,6 +148542,7 @@ static int nodeRowidIndex( ){ int ii; int nCell = NCELL(pNode); + assert( nCell<200 ); for(ii=0; iirScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} + +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} + +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} + +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel>=0 && iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; + } + } + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ) break; + } + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); + } + } + pCur->atEOF = p==0; + return SQLITE_OK; +} + /* ** Rtree virtual table module xNext method. */ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ - Rtree *pRtree = (Rtree *)(pVtabCursor->pVtab); RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; int rc = SQLITE_OK; - /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is - ** already at EOF. It is against the rules to call the xNext() method of - ** a cursor that has already reached EOF. - */ - assert( pCsr->pNode ); - - if( pCsr->iStrategy==1 ){ - /* This "scan" is a direct lookup by rowid. There is no next entry. */ - nodeRelease(pRtree, pCsr->pNode); - pCsr->pNode = 0; - }else{ - /* Move to the next entry that matches the configured constraints. */ - int iHeight = 0; - while( pCsr->pNode ){ - RtreeNode *pNode = pCsr->pNode; - int nCell = NCELL(pNode); - for(pCsr->iCell++; pCsr->iCelliCell++){ - int isEof; - rc = descendToCell(pRtree, pCsr, iHeight, &isEof); - if( rc!=SQLITE_OK || !isEof ){ - return rc; - } - } - pCsr->pNode = pNode->pParent; - rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); - if( rc!=SQLITE_OK ){ - return rc; - } - nodeReference(pCsr->pNode); - nodeRelease(pRtree, pNode); - iHeight++; - } - } - + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); return rc; } @@ -125505,13 +148868,14 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ ** Rtree virtual table module xRowid method. */ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ - Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - - assert(pCsr->pNode); - *pRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - - return SQLITE_OK; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; } /* @@ -125520,21 +148884,28 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ Rtree *pRtree = (Rtree *)cur->pVtab; RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; if( i==0 ){ - i64 iRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - sqlite3_result_int64(ctx, iRowid); + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - RtreeCoord c; - nodeGetCoord(pRtree, pCsr->pNode, pCsr->iCell, i-1, &c); + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); +#ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ sqlite3_result_double(ctx, c.f); - }else{ + }else +#endif + { assert( pRtree->eCoordType==RTREE_COORD_INT32 ); sqlite3_result_int(ctx, c.i); } } - return SQLITE_OK; } @@ -125545,12 +148916,18 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ ** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf ** to zero and return an SQLite error code. */ -static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ int rc; *ppLeaf = 0; sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); sqlite3_reset(pRtree->pReadRowid); }else{ @@ -125566,42 +148943,45 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ ** operator. */ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *p; - sqlite3_rtree_geometry *pGeom; - int nBlob; + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ /* Check that value is actually a blob. */ - if( !sqlite3_value_type(pValue)==SQLITE_BLOB ) return SQLITE_ERROR; + if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; /* Check that the blob is roughly the right size. */ nBlob = sqlite3_value_bytes(pValue); if( nBlob<(int)sizeof(RtreeMatchArg) - || ((nBlob-sizeof(RtreeMatchArg))%sizeof(double))!=0 + || ((nBlob-sizeof(RtreeMatchArg))%sizeof(RtreeDValue))!=0 ){ return SQLITE_ERROR; } - pGeom = (sqlite3_rtree_geometry *)sqlite3_malloc( - sizeof(sqlite3_rtree_geometry) + nBlob - ); - if( !pGeom ) return SQLITE_NOMEM; - memset(pGeom, 0, sizeof(sqlite3_rtree_geometry)); - p = (RtreeMatchArg *)&pGeom[1]; + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; - memcpy(p, sqlite3_value_blob(pValue), nBlob); - if( p->magic!=RTREE_GEOMETRY_MAGIC - || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double)) - ){ - sqlite3_free(pGeom); + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); return SQLITE_ERROR; } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; - pGeom->pContext = p->pContext; - pGeom->nParam = p->nParam; - pGeom->aParam = p->aParam; - - pCons->xGeom = p->xGeom; - pCons->pGeom = pGeom; + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; return SQLITE_OK; } @@ -125615,43 +148995,59 @@ static int rtreeFilter( ){ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - RtreeNode *pRoot = 0; int ii; int rc = SQLITE_OK; + int iCell = 0; rtreeReference(pRtree); + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ freeCursorConstraints(pCsr); - pCsr->iStrategy = idxNum; + sqlite3_free(pCsr->aPoint); + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->iStrategy = idxNum; if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); - rc = findLeafNode(pRtree, iRowid, &pLeaf); - pCsr->pNode = pLeaf; - if( pLeaf ){ - assert( rc==SQLITE_OK ); - rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array ** with the configured constraints. */ - if( argc>0 ){ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); - assert( (idxStr==0 && argc==0) || (int)strlen(idxStr)==argc*2 ); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + assert( (idxStr==0 && argc==0) + || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; - p->iCoord = idxStr[ii*2+1]-'a'; - if( p->op==RTREE_MATCH ){ + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ /* A MATCH operator. The right-hand-side must be a blob that ** can be cast into an RtreeMatchArg object. One created using ** an sqlite3_rtree_geometry_callback() SQL user function. @@ -125660,41 +149056,52 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ - p->rValue = sqlite3_value_double(argv[ii]); +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = sqlite3_value_int64(argv[ii]); +#else + p->u.rValue = sqlite3_value_double(argv[ii]); +#endif } } } } - if( rc==SQLITE_OK ){ - pCsr->pNode = 0; - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - } - if( rc==SQLITE_OK ){ - int isEof = 1; - int nCell = NCELL(pRoot); - pCsr->pNode = pRoot; - for(pCsr->iCell=0; rc==SQLITE_OK && pCsr->iCelliCell++){ - assert( pCsr->pNode==pRoot ); - rc = descendToCell(pRtree, pCsr, pRtree->iDepth, &isEof); - if( !isEof ){ - break; - } - } - if( rc==SQLITE_OK && isEof ){ - assert( pCsr->pNode==pRoot ); - nodeRelease(pRtree, pRoot); - pCsr->pNode = 0; - } - assert( rc!=SQLITE_OK || !pCsr->pNode || pCsr->iCellpNode) ); + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); } } + nodeRelease(pRtree, pRoot); rtreeRelease(pRtree); return rc; } +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + /* ** Rtree virtual table module xBestIndex method. There are three ** table scan strategies to choose from (in order from most to @@ -125730,13 +149137,14 @@ static int rtreeFilter( ** is 'a', the second from the left 'b' etc. */ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; int rc = SQLITE_OK; int ii; + i64 nRow; /* Estimated rows returned by this scan */ int iIdx = 0; char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; memset(zIdxStr, 0, sizeof(zIdxStr)); - UNUSED_PARAMETER(tab); assert( pIdxInfo->idxStr==0 ); for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ @@ -125756,9 +149164,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ /* This strategy involves a two rowid lookups on an B-Tree structures ** and then a linear search of an R-Tree node. This should be ** considered almost as quick as a direct rowid lookup (for which - ** sqlite uses an internal cost of 0.0). + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. */ - pIdxInfo->estimatedCost = 10.0; + pIdxInfo->estimatedCost = 30.0; + setEstimatedRows(pIdxInfo, 1); return SQLITE_OK; } @@ -125776,7 +149186,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + 'a'; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -125787,19 +149197,22 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ return SQLITE_NOMEM; } - assert( iIdx>=0 ); - pIdxInfo->estimatedCost = (2000000.0 / (double)(iIdx + 1)); + + nRow = pRtree->nRowEst / (iIdx + 1); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + setEstimatedRows(pIdxInfo, nRow); + return rc; } /* ** Return the N-dimensional volumn of the cell stored in *p. */ -static float cellArea(Rtree *pRtree, RtreeCell *p){ - float area = 1.0; +static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ + RtreeDValue area = (RtreeDValue)1; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = (float)(area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); } return area; } @@ -125808,11 +149221,11 @@ static float cellArea(Rtree *pRtree, RtreeCell *p){ ** Return the margin length of cell p. The margin length is the sum ** of the objects size in each dimension. */ -static float cellMargin(Rtree *pRtree, RtreeCell *p){ - float margin = 0.0; +static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ + RtreeDValue margin = (RtreeDValue)0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - margin += (float)(DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); } return margin; } @@ -125857,8 +149270,8 @@ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ /* ** Return the amount cell p would grow by if it were unioned with pCell. */ -static float cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ - float area; +static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ + RtreeDValue area; RtreeCell cell; memcpy(&cell, p, sizeof(RtreeCell)); area = cellArea(pRtree, &cell); @@ -125866,64 +149279,32 @@ static float cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ return (cellArea(pRtree, &cell)-area); } -#if VARIANT_RSTARTREE_CHOOSESUBTREE || VARIANT_RSTARTREE_SPLIT -static float cellOverlap( +static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, RtreeCell *aCell, - int nCell, - int iExclude + int nCell ){ int ii; - float overlap = 0.0; + RtreeDValue overlap = RTREE_ZERO; for(ii=0; iinDim*2); jj+=2){ - double x1; - double x2; - - x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); - x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); - - if( x2nDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-1) ){ - int jj; - aCell = sqlite3_malloc(sizeof(RtreeCell)*nCell); - if( !aCell ){ - rc = SQLITE_NOMEM; - nodeRelease(pRtree, pNode); - pNode = 0; - continue; - } - for(jj=0; jjiDepth-1) ){ - overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); - } - if( (iCell==0) - || (overlapnDim; i++){ - float x1 = DCOORD(aCell[0].aCoord[i*2]); - float x2 = DCOORD(aCell[0].aCoord[i*2+1]); - float x3 = x1; - float x4 = x2; - int jj; - - int iCellLeft = 0; - int iCellRight = 0; - - for(jj=1; jjx4 ) x4 = right; - if( left>x3 ){ - x3 = left; - iCellRight = jj; - } - if( rightmaxNormalInnerWidth ){ - iLeftSeed = iCellLeft; - iRightSeed = iCellRight; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_LINEAR_SPLIT */ - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT -/* -** Implementation of the quadratic variant of the PickNext() function from -** Guttman[84]. -*/ -static RtreeCell *QuadraticPickNext( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - RtreeCell *pLeftBox, - RtreeCell *pRightBox, - int *aiUsed -){ - #define FABS(a) ((a)<0.0?-1.0*(a):(a)) - - int iSelect = -1; - float fDiff; - int ii; - for(ii=0; iifDiff ){ - fDiff = diff; - iSelect = ii; - } - } - } - aiUsed[iSelect] = 1; - return &aCell[iSelect]; -} - -/* -** Implementation of the quadratic variant of the PickSeeds() function from -** Guttman[84]. -*/ -static void QuadraticPickSeeds( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - int *piLeftSeed, - int *piRightSeed -){ - int ii; - int jj; - - int iLeftSeed = 0; - int iRightSeed = 1; - float fWaste = 0.0; - - for(ii=0; iifWaste ){ - iLeftSeed = ii; - iRightSeed = jj; - fWaste = waste; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_QUADRATIC_SPLIT */ /* ** Arguments aIdx, aDistance and aSpare all point to arrays of size @@ -126241,7 +149440,7 @@ static void QuadraticPickSeeds( static void SortByDistance( int *aIdx, int nIdx, - float *aDistance, + RtreeDValue *aDistance, int *aSpare ){ if( nIdx>1 ){ @@ -126267,8 +149466,8 @@ static void SortByDistance( aIdx[iLeft+iRight] = aLeft[iLeft]; iLeft++; }else{ - float fLeft = aDistance[aLeft[iLeft]]; - float fRight = aDistance[aRight[iRight]]; + RtreeDValue fLeft = aDistance[aLeft[iLeft]]; + RtreeDValue fRight = aDistance[aRight[iRight]]; if( fLeftnDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -126400,9 +149598,9 @@ static int splitNodeStartree( } for(ii=0; iinDim; ii++){ - float margin = 0.0; - float fBestOverlap = 0.0; - float fBestArea = 0.0; + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; int iBestLeft = 0; int nLeft; @@ -126414,8 +149612,8 @@ static int splitNodeStartree( RtreeCell left; RtreeCell right; int kk; - float overlap; - float area; + RtreeDValue overlap; + RtreeDValue area; memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); @@ -126428,7 +149626,7 @@ static int splitNodeStartree( } margin += cellMargin(pRtree, &left); margin += cellMargin(pRtree, &right); - overlap = cellOverlap(pRtree, &left, &right, 1, -1); + overlap = cellOverlap(pRtree, &left, &right, 1); area = cellArea(pRtree, &left) + cellArea(pRtree, &right); if( (nLeft==RTREE_MINCELLS(pRtree)) || (overlap0; i--){ - RtreeCell *pNext; - pNext = PickNext(pRtree, aCell, nCell, pBboxLeft, pBboxRight, aiUsed); - float diff = - cellGrowth(pRtree, pBboxLeft, pNext) - - cellGrowth(pRtree, pBboxRight, pNext) - ; - if( (RTREE_MINCELLS(pRtree)-NCELL(pRight)==i) - || (diff>0.0 && (RTREE_MINCELLS(pRtree)-NCELL(pLeft)!=i)) - ){ - nodeInsertCell(pRtree, pRight, pNext); - cellUnion(pRtree, pBboxRight, pNext); - }else{ - nodeInsertCell(pRtree, pLeft, pNext); - cellUnion(pRtree, pBboxLeft, pNext); - } - } - - sqlite3_free(aiUsed); - return SQLITE_OK; -} -#endif static int updateMapping( Rtree *pRtree, @@ -126594,7 +149736,8 @@ static int SplitNode( memset(pLeft->zData, 0, pRtree->iNodeSize); memset(pRight->zData, 0, pRtree->iNodeSize); - rc = AssignCells(pRtree, aCell, nCell, pLeft, pRight, &leftbbox, &rightbbox); + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); if( rc!=SQLITE_OK ){ goto splitnode_out; } @@ -126831,32 +149974,34 @@ static int Reinsert( int *aOrder; int *aSpare; RtreeCell *aCell; - float *aDistance; + RtreeDValue *aDistance; int nCell; - float aCenterCoord[RTREE_MAX_DIMENSIONS]; + RtreeDValue aCenterCoord[RTREE_MAX_DIMENSIONS]; int iDim; int ii; int rc = SQLITE_OK; + int n; - memset(aCenterCoord, 0, sizeof(float)*RTREE_MAX_DIMENSIONS); + memset(aCenterCoord, 0, sizeof(RtreeDValue)*RTREE_MAX_DIMENSIONS); nCell = NCELL(pNode)+1; + n = (nCell+1)&(~1); /* Allocate the buffers used by this operation. The allocation is ** relinquished before this function returns. */ - aCell = (RtreeCell *)sqlite3_malloc(nCell * ( - sizeof(RtreeCell) + /* aCell array */ - sizeof(int) + /* aOrder array */ - sizeof(int) + /* aSpare array */ - sizeof(float) /* aDistance array */ + aCell = (RtreeCell *)sqlite3_malloc(n * ( + sizeof(RtreeCell) + /* aCell array */ + sizeof(int) + /* aOrder array */ + sizeof(int) + /* aSpare array */ + sizeof(RtreeDValue) /* aDistance array */ )); if( !aCell ){ return SQLITE_NOMEM; } - aOrder = (int *)&aCell[nCell]; - aSpare = (int *)&aOrder[nCell]; - aDistance = (float *)&aSpare[nCell]; + aOrder = (int *)&aCell[n]; + aSpare = (int *)&aOrder[n]; + aDistance = (RtreeDValue *)&aSpare[n]; for(ii=0; iinDim; iDim++){ - aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2+1]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); } } for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = (float)(aCenterCoord[iDim]/((float)nCell*2.0)); + aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); } for(ii=0; iinDim; iDim++){ - float coord = (float)(DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2])); + RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); } } @@ -126941,16 +150086,12 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ -#if VARIANT_RSTARTREE_REINSERT if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ pRtree->iReinsertHeight = iHeight; rc = Reinsert(pRtree, pNode, pCell, iHeight); } -#else - rc = SplitNode(pRtree, pNode, pCell, iHeight); -#endif }else{ rc = AdjustTree(pRtree, pNode, pCell); if( rc==SQLITE_OK ){ @@ -127008,19 +150149,19 @@ static int newRowid(Rtree *pRtree, i64 *piRowid){ */ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ int rc; /* Return code */ - RtreeNode *pLeaf; /* Leaf node containing record iDelete */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ int iCell; /* Index of iDelete cell in pLeaf */ RtreeNode *pRoot; /* Root node of rtree structure */ - /* Obtain a reference to the root node to initialise Rtree.iDepth */ + /* Obtain a reference to the root node to initialize Rtree.iDepth */ rc = nodeAcquire(pRtree, 1, 0, &pRoot); /* Obtain a reference to the leaf node that contains the entry ** about to be deleted. */ if( rc==SQLITE_OK ){ - rc = findLeafNode(pRtree, iDelete, &pLeaf); + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } /* Delete the cell in question from the leaf node. */ @@ -127087,6 +150228,36 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ return rc; } +/* +** Rounding constants for float->double conversion. +*/ +#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ +#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ + +#if !defined(SQLITE_RTREE_INT_ONLY) +/* +** Convert an sqlite3_value into an RtreeValue (presumably a float) +** while taking care to round toward negative or positive, respectively. +*/ +static RtreeValue rtreeValueDown(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( f>d ){ + f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); + } + return f; +} +static RtreeValue rtreeValueUp(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( fnDim*2 + 3) ); +#ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - cell.aCoord[ii].f = (float)sqlite3_value_double(azData[ii+3]); - cell.aCoord[ii+1].f = (float)sqlite3_value_double(azData[ii+4]); + cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]); + cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]); if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){ rc = SQLITE_CONSTRAINT; goto constraint; } } - }else{ + }else +#endif + { for(ii=0; ii<(pRtree->nDim*2); ii+=2){ cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]); cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]); @@ -127178,7 +150352,7 @@ static int rtreeUpdate( */ if( rc==SQLITE_OK && nData>1 ){ /* Insert the new record into the r-tree */ - RtreeNode *pLeaf; + RtreeNode *pLeaf = 0; /* Figure out the rowid of the new row. */ if( bHaveRowid==0 ){ @@ -127226,6 +150400,43 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ return rc; } +/* +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. +*/ +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; + sqlite3_stmt *p; + int rc; + i64 nRow = 0; + + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; + } + + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } + } + sqlite3_free(zSql); + } + + return rc; +} + static sqlite3_module rtreeModule = { 0, /* iVersion */ rtreeCreate, /* xCreate - create a table */ @@ -127287,7 +150498,8 @@ static int rtreeSqlInit( char *zCreate = sqlite3_mprintf( "CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" -"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY, parentnode INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" "INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize ); @@ -127311,6 +150523,7 @@ static int rtreeSqlInit( appStmt[7] = &pRtree->pWriteParent; appStmt[8] = &pRtree->pDeleteParent; + rc = rtreeQueryStat1(db, pRtree); for(i=0; inBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } }else{ zSql = sqlite3_mprintf( @@ -127384,6 +150600,9 @@ static int getNodeSize( pRtree->zDb, pRtree->zName ); rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } } sqlite3_free(zSql); @@ -127429,8 +150648,8 @@ static int rtreeInit( sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Allocate the sqlite3_vtab structure */ - nDb = strlen(argv[1]); - nName = strlen(argv[2]); + nDb = (int)strlen(argv[1]); + nName = (int)strlen(argv[2]); pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); if( !pRtree ){ return SQLITE_NOMEM; @@ -127447,7 +150666,7 @@ static int rtreeInit( memcpy(pRtree->zName, argv[2], nName); /* Figure out the node size to use. */ - rc = getNodeSize(db, pRtree, isCreate); + rc = getNodeSize(db, pRtree, isCreate, pzErr); /* Create/Connect to the underlying relational database schema. If ** that is successful, call sqlite3_declare_vtab() to configure @@ -127482,6 +150701,8 @@ static int rtreeInit( if( rc==SQLITE_OK ){ *ppVtab = (sqlite3_vtab *)pRtree; }else{ + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); rtreeRelease(pRtree); } return rc; @@ -127492,10 +150713,10 @@ static int rtreeInit( ** Implementation of a scalar function that decodes r-tree nodes to ** human readable strings. This can be used for debugging and analysis. ** -** The scalar function takes two arguments, a blob of data containing -** an r-tree node, and the number of dimensions the r-tree indexes. -** For a two-dimensional r-tree structure called "rt", to deserialize -** all nodes, a statement like: +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: ** ** SELECT rtreenode(2, data) FROM rt_node; ** @@ -127525,10 +150746,16 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nodeGetCell(&tree, &node, ii, &cell); sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); - nCell = strlen(zCell); + nCell = (int)strlen(zCell); for(jj=0; jjxDestructor ) pInfo->xDestructor(pInfo->pContext); sqlite3_free(p); } /* -** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite -** scalar user function. This C function is the callback used for all such -** registered SQL functions. +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. ** -** The scalar user functions return a blob that is interpreted by r-tree -** table MATCH operators. +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. */ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); RtreeMatchArg *pBlob; int nBlob; - nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(double); + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue); pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); if( !pBlob ){ sqlite3_result_error_nomem(ctx); }else{ int i; pBlob->magic = RTREE_GEOMETRY_MAGIC; - pBlob->xGeom = pGeomCtx->xGeom; - pBlob->pContext = pGeomCtx->pContext; + pBlob->cb = pGeomCtx[0]; pBlob->nParam = nArg; for(i=0; iaParam[i] = sqlite3_value_int64(aArg[i]); +#else pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +#endif } - sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free); } } @@ -127624,10 +150876,10 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ ** Register a new geometry function for use with the r-tree MATCH operator. */ SQLITE_API int sqlite3_rtree_geometry_callback( - sqlite3 *db, - const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *), - void *pContext + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ ){ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ @@ -127635,17 +150887,44 @@ SQLITE_API int sqlite3_rtree_geometry_callback( pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); if( !pGeomCtx ) return SQLITE_NOMEM; pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; pGeomCtx->pContext = pContext; - - /* Create the new user-function. Register a destructor function to delete - ** the context object when it is no longer required. */ return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, - (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); } #if !SQLITE_CORE -SQLITE_API int sqlite3_extension_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_rtree_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -127683,7 +150962,7 @@ SQLITE_API int sqlite3_extension_init( ** * Implementations of the SQL scalar upper() and lower() functions ** for case mapping. ** -** * Integration of ICU and SQLite collation seqences. +** * Integration of ICU and SQLite collation sequences. ** ** * An implementation of the LIKE operator that uses ICU to ** provide case-independent matching. @@ -127697,6 +150976,7 @@ SQLITE_API int sqlite3_extension_init( #include #include +/* #include */ #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 @@ -128146,7 +151426,10 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ } #if !SQLITE_CORE -SQLITE_API int sqlite3_extension_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_icu_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -128176,8 +151459,12 @@ SQLITE_API int sqlite3_extension_init( #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU +/* #include */ +/* #include */ #include +/* #include */ +/* #include */ #include typedef struct IcuTokenizer IcuTokenizer; @@ -128266,13 +151553,16 @@ static int icuOpen( *ppCursor = 0; - if( nInput<0 ){ + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ nInput = strlen(zInput); } nChar = nInput+1; pCsr = (IcuCursor *)sqlite3_malloc( sizeof(IcuCursor) + /* IcuCursor */ - nChar * sizeof(UChar) + /* IcuCursor.aChar[] */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ ); if( !pCsr ){ @@ -128280,7 +151570,7 @@ static int icuOpen( } memset(pCsr, 0, sizeof(IcuCursor)); pCsr->aChar = (UChar *)&pCsr[1]; - pCsr->aOffset = (int *)&pCsr->aChar[nChar]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; pCsr->aOffset[iOut] = iInput; U8_NEXT(zInput, iInput, nInput, c); @@ -128352,7 +151642,7 @@ static int icuNext( while( iStartaChar, iWhite, pCsr->nChar, c); + U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c); if( u_isspace(c) ){ iStart = iWhite; }else{ diff --git a/src/sqlite/sqlite3.h b/src/sqlite/sqlite3.h index ed9edbd20..c31f126db 100644 --- a/src/sqlite/sqlite3.h +++ b/src/sqlite/sqlite3.h @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.7.1" -#define SQLITE_VERSION_NUMBER 3007007 -#define SQLITE_SOURCE_ID "2011-06-28 17:39:05 af0d91adf497f5f36ec3813f04235a6e195a605f" +#define SQLITE_VERSION "3.8.7.4" +#define SQLITE_VERSION_NUMBER 3008007 +#define SQLITE_SOURCE_ID "2014-12-09 01:34:36 f66f7a17b78ba617acde90fc810107f34f1a1f2e" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -177,7 +177,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N); ** CAPI3REF: Test To See If The Library Is Threadsafe ** ** ^The sqlite3_threadsafe() function returns zero if and only if -** SQLite was compiled mutexing code omitted due to the +** SQLite was compiled with mutexing code omitted due to the ** [SQLITE_THREADSAFE] compile-time option being set to 0. ** ** SQLite can be compiled with or without mutexes. When @@ -219,7 +219,8 @@ SQLITE_API int sqlite3_threadsafe(void); ** the opaque structure named "sqlite3". It is useful to think of an sqlite3 ** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and ** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] -** is its destructor. There are many other interfaces (such as +** and [sqlite3_close_v2()] are its destructors. There are many other +** interfaces (such as ** [sqlite3_prepare_v2()], [sqlite3_create_function()], and ** [sqlite3_busy_timeout()] to name but three) that are methods on an ** sqlite3 object. @@ -266,28 +267,46 @@ typedef sqlite_uint64 sqlite3_uint64; /* ** CAPI3REF: Closing A Database Connection ** -** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. -** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is -** successfully destroyed and all associated resources are deallocated. +** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors +** for the [sqlite3] object. +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if +** the [sqlite3] object is successfully destroyed and all associated +** resources are deallocated. ** -** Applications must [sqlite3_finalize | finalize] all [prepared statements] -** and [sqlite3_blob_close | close] all [BLOB handles] associated with -** the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close() is called on a [database connection] that still has -** outstanding [prepared statements] or [BLOB handles], then it returns -** SQLITE_BUSY. +** ^If the database connection is associated with unfinalized prepared +** statements or unfinished sqlite3_backup objects then sqlite3_close() +** will leave the database connection open and return [SQLITE_BUSY]. +** ^If sqlite3_close_v2() is called with unfinalized prepared statements +** and/or unfinished sqlite3_backups, then the database connection becomes +** an unusable "zombie" which will automatically be deallocated when the +** last prepared statement is finalized or the last sqlite3_backup is +** finished. The sqlite3_close_v2() interface is intended for use with +** host languages that are garbage collected, and where the order in which +** destructors are called is arbitrary. ** -** ^If [sqlite3_close()] is invoked while a transaction is open, +** Applications should [sqlite3_finalize | finalize] all [prepared statements], +** [sqlite3_blob_close | close] all [BLOB handles], and +** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +** with the [sqlite3] object prior to attempting to close the object. ^If +** sqlite3_close_v2() is called on a [database connection] that still has +** outstanding [prepared statements], [BLOB handles], and/or +** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation +** of resources is deferred until all [prepared statements], [BLOB handles], +** and [sqlite3_backup] objects are also destroyed. +** +** ^If an [sqlite3] object is destroyed while a transaction is open, ** the transaction is automatically rolled back. ** -** The C parameter to [sqlite3_close(C)] must be either a NULL +** The C parameter to [sqlite3_close(C)] and [sqlite3_close_v2(C)] +** must be either a NULL ** pointer or an [sqlite3] object pointer obtained ** from [sqlite3_open()], [sqlite3_open16()], or ** [sqlite3_open_v2()], and not previously closed. -** ^Calling sqlite3_close() with a NULL pointer argument is a -** harmless no-op. +** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer +** argument is a harmless no-op. */ -SQLITE_API int sqlite3_close(sqlite3 *); +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); /* ** The type for a callback function. @@ -351,7 +370,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); **
    **
  • The application must insure that the 1st parameter to sqlite3_exec() ** is a valid and open [database connection]. -**
  • The application must not close [database connection] specified by +**
  • The application must not close the [database connection] specified by ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. @@ -367,16 +386,14 @@ SQLITE_API int sqlite3_exec( /* ** CAPI3REF: Result Codes -** KEYWORDS: SQLITE_OK {error code} {error codes} -** KEYWORDS: {result code} {result codes} +** KEYWORDS: {result code definitions} ** ** Many SQLite functions return an integer result code from the set shown -** here in order to indicates success or failure. +** here in order to indicate success or failure. ** ** New error codes may be added in future versions of SQLite. ** -** See also: [SQLITE_IOERR_READ | extended result codes], -** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes]. +** See also: [extended result code definitions] */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ @@ -406,32 +423,27 @@ SQLITE_API int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ /* ** CAPI3REF: Extended Result Codes -** KEYWORDS: {extended error code} {extended error codes} -** KEYWORDS: {extended result code} {extended result codes} +** KEYWORDS: {extended result code definitions} ** -** In its default configuration, SQLite API routines return one of 26 integer -** [SQLITE_OK | result codes]. However, experience has shown that many of +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of ** these result codes are too coarse-grained. They do not provide as ** much information about problems as programmers might like. In an effort to ** address this, newer versions of SQLite (version 3.3.8 and later) include ** support for additional result codes that provide more detailed information -** about errors. The extended result codes are enabled or disabled +** about errors. These [extended result codes] are enabled or disabled ** on a per database connection basis using the -** [sqlite3_extended_result_codes()] API. -** -** Some of the available extended result codes are listed here. -** One may expect the number of extended result codes will be expand -** over time. Software that uses extended result codes should expect -** to see new result codes in future releases of SQLite. -** -** The SQLITE_OK result code will never be extended. It will always -** be exactly zero. +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. */ #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) @@ -455,12 +467,37 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) +#define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) +#define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) +#define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +#define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -476,6 +513,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ #define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ #define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MEMORY 0x00000080 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ #define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ #define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ @@ -495,7 +533,7 @@ SQLITE_API int sqlite3_exec( ** CAPI3REF: Device Characteristics ** ** The xDeviceCharacteristics method of the [sqlite3_io_methods] -** object returns an integer which is a vector of the these +** object returns an integer which is a vector of these ** bit values expressing I/O characteristics of the mass storage ** device that holds the file that the [sqlite3_io_methods] ** refers to. @@ -509,7 +547,15 @@ SQLITE_API int sqlite3_exec( ** first then the size of the file is extended, never the other ** way around. The SQLITE_IOCAP_SEQUENTIAL property means that ** information is written to disk in the same order as calls -** to xWrite(). +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -523,6 +569,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOCAP_SAFE_APPEND 0x00000200 #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 /* ** CAPI3REF: File Locking Levels @@ -629,7 +677,7 @@ struct sqlite3_file { ** locking strategy (for example to use dot-file locks), to inquire ** about the status of a lock, or to break stale locks. The SQLite ** core reserves all opcodes less than 100 for its own use. -** A [SQLITE_FCNTL_LOCKSTATE | list of opcodes] less than 100 is available. +** A [file control opcodes | list of opcodes] less than 100 is available. ** Applications that define a custom xFileControl method should use opcodes ** greater than 100 to avoid conflicts. VFS implementations should ** return [SQLITE_NOTFOUND] for file control opcodes that they do not @@ -694,11 +742,15 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; /* ** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} ** ** These integer constants are opcodes for the xFileControl method ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] @@ -711,7 +763,8 @@ struct sqlite3_io_methods { ** into an integer that the pArg argument points to. This capability ** is used during testing and only needs to be supported when SQLITE_TEST ** is defined. -** +**
      +**
    • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS ** layer a hint of how large the database file will grow to be during the ** current transaction. This hint is not guaranteed to be accurate but it @@ -719,6 +772,7 @@ struct sqlite3_io_methods { ** file space based on this hint in order to help writes to the database ** file run faster. ** +**
    • [[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified ** by the user. The fourth argument to [sqlite3_file_control()] should @@ -727,30 +781,195 @@ struct sqlite3_io_methods { ** chunks (say 1MB at a time), may reduce file-system fragmentation and ** improve performance on some systems. ** +**
    • [[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database ** connection. See the [sqlite3_file_control()] documentation for ** additional information. ** -** ^(The [SQLITE_FCNTL_SYNC_OMITTED] opcode is generated internally by -** SQLite and sent to all VFSes in place of a call to the xSync method -** when the database connection has [PRAGMA synchronous] set to OFF.)^ -** Some specialized VFSes need this signal in order to operate correctly -** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most -** VFSes do not need this signal and should silently ignore this opcode. -** Applications should not call [sqlite3_file_control()] with this -** opcode as doing so may disrupt the operation of the specialized VFSes -** that do require it. +**
    • [[SQLITE_FCNTL_SYNC_OMITTED]] +** No longer in use. +** +**
    • [[SQLITE_FCNTL_SYNC]] +** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and +** sent to the VFS immediately before the xSync method is invoked on a +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** of the xSync method. In most cases, the pointer argument passed with +** this file-control is NULL. However, if the database file is being synced +** as part of a multi-database commit, the argument points to a nul-terminated +** string containing the transactions master-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. +** +**
    • [[SQLITE_FCNTL_COMMIT_PHASETWO]] +** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite +** and sent to the VFS after a transaction has been committed immediately +** but before the database is unlocked. VFSes that do not need this signal +** should silently ignore this opcode. Applications should not call +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. +** +**
    • [[SQLITE_FCNTL_WIN32_AV_RETRY]] +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to provide robustness in the presence of +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete operations up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows these two values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer i the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** +**
    • [[SQLITE_FCNTL_PERSIST_WAL]] +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary +** write ahead log and shared memory files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +**
    • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +**
    • [[SQLITE_FCNTL_OVERWRITE]] +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. +** +**
    • [[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +**
    • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** +**
    • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
    • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
    • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** +**
    • [[SQLITE_FCNTL_TRACE]] +** The [SQLITE_FCNTL_TRACE] file control provides advisory information +** to the VFS about what the higher layers of the SQLite stack are doing. +** This file control is used by some VFS activity tracing [shims]. +** The argument is a zero-terminated string. Higher layers in the +** SQLite stack may generate instances of this file control if +** the [SQLITE_USE_FCNTL_TRACE] compile-time option is enabled. +** +**
    • [[SQLITE_FCNTL_HAS_MOVED]] +** The [SQLITE_FCNTL_HAS_MOVED] file control interprets its argument as a +** pointer to an integer and it writes a boolean into that integer depending +** on whether or not the file has been renamed, moved, or deleted since it +** was first opened. +** +**
    • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** +**
    */ -#define SQLITE_FCNTL_LOCKSTATE 1 -#define SQLITE_GET_LOCKPROXYFILE 2 -#define SQLITE_SET_LOCKPROXYFILE 3 -#define SQLITE_LAST_ERRNO 4 -#define SQLITE_FCNTL_SIZE_HINT 5 -#define SQLITE_FCNTL_CHUNK_SIZE 6 -#define SQLITE_FCNTL_FILE_POINTER 7 -#define SQLITE_FCNTL_SYNC_OMITTED 8 - +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_GET_LOCKPROXYFILE 2 +#define SQLITE_SET_LOCKPROXYFILE 3 +#define SQLITE_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_TRACE 19 +#define SQLITE_FCNTL_HAS_MOVED 20 +#define SQLITE_FCNTL_SYNC 21 +#define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 /* ** CAPI3REF: Mutex Handle @@ -805,7 +1024,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** from xFullPathname() with an optional suffix added. ** ^If a suffix is added to the zFilename parameter, it will ** consist of a single "-" character followed by no more than -** 10 alphanumeric and/or "-" characters. +** 11 alphanumeric and/or "-" characters. ** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, @@ -1178,16 +1397,10 @@ SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); ** order to verify that SQLite recovers gracefully from such ** conditions. ** -** The xMalloc and xFree methods must work like the -** malloc() and free() functions from the standard C library. -** The xRealloc method must work like realloc() from the standard C library -** with the exception that if the second argument to xRealloc is zero, -** xRealloc must be a no-op - it must not perform any allocation or -** deallocation. ^SQLite guarantees that the second argument to +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to ** xRealloc is always a value returned by a prior call to xRoundup. -** And so in cases where xRoundup always returns a positive number, -** xRealloc can perform exactly as the standard library realloc() and -** still be in compliance with this specification. ** ** xSize should return the allocated size of a memory allocation ** previously obtained from xMalloc or xRealloc. The allocated size @@ -1201,7 +1414,7 @@ SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); ** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, ** that causes the corresponding memory allocation to fail. ** -** The xInit method initializes the memory allocator. (For example, +** The xInit method initializes the memory allocator. For example, ** it might allocate any require mutexes or initialize internal data ** structures. The xShutdown method is invoked (indirectly) by ** [sqlite3_shutdown()] and should deallocate any resources acquired @@ -1342,7 +1555,7 @@ struct sqlite3_mem_methods { **
    ^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. +** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. ** There are three arguments to this option: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page @@ -1373,8 +1586,8 @@ struct sqlite3_mem_methods { ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. -** The minimum allocation size is capped at 2^12. Reasonable values -** for the minimum allocation size are 2^5 through 2^8.
    +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8. ** ** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    **
    ^(This option takes a single argument which is a pointer to an @@ -1411,19 +1624,21 @@ struct sqlite3_mem_methods { ** verb to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
    ** -** [[SQLITE_CONFIG_PCACHE]]
    SQLITE_CONFIG_PCACHE
    +** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    **
    ^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods] object. This object specifies the interface +** an [sqlite3_pcache_methods2] object. This object specifies the interface ** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.
    ** -** [[SQLITE_CONFIG_GETPCACHE]]
    SQLITE_CONFIG_GETPCACHE
    +** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    **
    ^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods] object. SQLite copies of the current +** [sqlite3_pcache_methods2] object. SQLite copies of the current ** page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    -**
    ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -1441,17 +1656,73 @@ struct sqlite3_mem_methods { ** function must be threadsafe.
    ** ** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI -**
    This option takes a single argument of type int. If non-zero, then +**
    ^(This option takes a single argument of type int. If non-zero, then ** URI handling is globally enabled. If the parameter is zero, then URI handling -** is globally disabled. If URI handling is globally enabled, all filenames +** is globally disabled.)^ ^If URI handling is globally enabled, all filenames ** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database -** connection is opened. If it is globally disabled, filenames are +** connection is opened. ^If it is globally disabled, filenames are ** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the -** database connection is opened. By default, URI handling is globally +** database connection is opened. ^(By default, URI handling is globally ** disabled. The default value may be changed by compiling with the -** [SQLITE_USE_URI] symbol defined. +** [SQLITE_USE_URI] symbol defined.)^ +** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN +**
    ^This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. ^The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +**
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +**
    These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. +**
    +** +** [[SQLITE_CONFIG_SQLLOG]] +**
    SQLITE_CONFIG_SQLLOG +**
    This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    ^SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** ^The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ +** ^If either argument to this option is negative, then that argument is +** changed to its compile-time default. +** +** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] +**
    SQLITE_CONFIG_WIN32_HEAPSIZE +**
    ^This option is only available if SQLite is compiled for Windows +** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. +** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** that specifies the maximum size of the created heap. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1467,10 +1738,16 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ /* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ #define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ -#define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ #define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ #define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1547,19 +1824,21 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); /* ** CAPI3REF: Last Insert Rowid ** -** ^Each entry in an SQLite table has a unique 64-bit signed +** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) +** has a unique 64-bit signed ** integer key called the [ROWID | "rowid"]. ^The rowid is always available ** as an undeclared column named ROWID, OID, or _ROWID_ as long as those ** names are not also used by explicitly declared columns. ^If ** the table has a column of type [INTEGER PRIMARY KEY] then that column ** is another alias for the rowid. ** -** ^This routine returns the [rowid] of the most recent -** successful [INSERT] into the database from the [database connection] -** in the first argument. ^As of SQLite version 3.7.7, this routines -** records the last insert rowid of both ordinary tables and [virtual tables]. -** ^If no successful [INSERT]s -** have ever occurred on that database connection, zero is returned. +** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the +** most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. +** ^Inserts into [WITHOUT ROWID] tables are not recorded. +** ^If no successful [INSERT]s into rowid tables +** have ever occurred on the database connection D, +** then sqlite3_last_insert_rowid(D) returns zero. ** ** ^(If an [INSERT] occurs within a trigger or within a [virtual table] ** method, then this routine will return the [rowid] of the inserted @@ -1751,27 +2030,33 @@ SQLITE_API int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors ** -** ^This routine sets a callback function that might be invoked whenever -** an attempt is made to open a database table that another thread -** or process has locked. +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. ** -** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] +** ^If the busy callback is NULL, then [SQLITE_BUSY] ** is returned immediately upon encountering the lock. ^If the busy callback ** is not NULL, then the callback might be invoked with two arguments. ** ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for this locking event. ^If the +** been invoked for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. +** access the database and [SQLITE_BUSY] is returned +** to the application. ** ^If the callback returns non-zero, then another attempt -** is made to open the database for reading and the cycle repeats. +** is made to access the database and the cycle repeats. ** ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. +** to the application instead of invoking the +** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -1785,28 +2070,15 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ** ^The default busy callback is NULL. ** -** ^The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] -** when SQLite is in the middle of a large transaction where all the -** changes will not fit into the in-memory cache. SQLite will -** already hold a RESERVED lock on the database file, but it needs -** to promote this lock to EXCLUSIVE so that it can spill cache -** pages into the database file without harm to concurrent -** readers. ^If it is unable to promote the lock, then the in-memory -** cache will be left in an inconsistent state and so the error -** code is promoted from the relatively benign [SQLITE_BUSY] to -** the more severe [SQLITE_IOERR_BLOCKED]. ^This error code promotion -** forces an automatic rollback of the changes. See the -** -** CorruptionFollowingBusyError wiki page for a discussion of why -** this is important. -** ** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any ** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] -** will also set or clear the busy handler. +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. ** ** The busy callback should not take any actions which modify the -** database connection that invoked the busy handler. Any such actions +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions ** result in undefined behavior. ** ** A busy handler must not close the database connection @@ -1822,15 +2094,17 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** will sleep multiple times until at least "ms" milliseconds of sleeping ** have accumulated. ^After at least "ms" milliseconds of sleeping, ** the handler returns 0 which causes [sqlite3_step()] to return -** [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. +** [SQLITE_BUSY]. ** ** ^Calling this routine with an argument less than or equal to zero ** turns off all busy handlers. ** ** ^(There can only be a single busy handler for a particular -** [database connection] any any given moment. If another busy handler +** [database connection] at any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); @@ -1955,7 +2229,7 @@ SQLITE_API void sqlite3_free_table(char **result); ** All of the usual printf() formatting options apply. In addition, there ** is are "%q", "%Q", and "%z" options. ** -** ^(The %q option works like %s in that it substitutes a null-terminated +** ^(The %q option works like %s in that it substitutes a nul-terminated ** string from the argument list. But %q also doubles every '\'' character. ** %q is designed for use inside a string literal.)^ By doubling each '\'' ** character it escapes that character and allows it to be inserted into @@ -2030,6 +2304,10 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns ** a NULL pointer. ** +** ^The sqlite3_malloc64(N) routine works just like +** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead +** of a signed 32-bit integer. +** ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is @@ -2041,24 +2319,38 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** might result if sqlite3_free() is called with a non-NULL pointer that ** was not obtained from sqlite3_malloc() or sqlite3_realloc(). ** -** ^(The sqlite3_realloc() interface attempts to resize a -** prior memory allocation to be at least N bytes, where N is the -** second parameter. The memory allocation to be resized is the first -** parameter.)^ ^ If the first parameter to sqlite3_realloc() +** ^The sqlite3_realloc(X,N) interface attempts to resize a +** prior memory allocation X to be at least N bytes. +** ^If the X parameter to sqlite3_realloc(X,N) ** is a NULL pointer then its behavior is identical to calling -** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc(). -** ^If the second parameter to sqlite3_realloc() is zero or +** sqlite3_malloc(N). +** ^If the N parameter to sqlite3_realloc(X,N) is zero or ** negative then the behavior is exactly the same as calling -** sqlite3_free(P) where P is the first parameter to sqlite3_realloc(). -** ^sqlite3_realloc() returns a pointer to a memory allocation -** of at least N bytes in size or NULL if sufficient memory is unavailable. +** sqlite3_free(X). +** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation +** of at least N bytes in size or NULL if insufficient memory is available. ** ^If M is the size of the prior allocation, then min(N,M) bytes ** of the prior allocation are copied into the beginning of buffer returned -** by sqlite3_realloc() and the prior allocation is freed. -** ^If sqlite3_realloc() returns NULL, then the prior allocation -** is not freed. +** by sqlite3_realloc(X,N) and the prior allocation is freed. +** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the +** prior allocation is not freed. ** -** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() +** ^The sqlite3_realloc64(X,N) interfaces works the same as +** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead +** of a 32-bit signed integer. +** +** ^If X is a memory allocation previously obtained from sqlite3_malloc(), +** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then +** sqlite3_msize(X) returns the size of that memory allocation in bytes. +** ^The value returned by sqlite3_msize(X) might be larger than the number +** of bytes requested when X was allocated. ^If X is a NULL pointer then +** sqlite3_msize(X) returns zero. If X points to something that is not +** the beginning of memory allocation, or if it points to a formerly +** valid memory allocation that has now been freed, then the behavior +** of sqlite3_msize(X) is undefined and possibly harmful. +** +** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(), +** sqlite3_malloc64(), and sqlite3_realloc64() ** is always aligned to at least an 8 byte boundary, or to a ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time ** option is used. @@ -2068,12 +2360,12 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** implementation of these routines to be omitted. That capability ** is no longer provided. Only built-in memory allocators can be used. ** -** The Windows OS interface layer calls +** Prior to SQLite version 3.7.10, the Windows OS interface layer called ** the system malloc() and free() directly when converting ** filenames between the UTF-8 encoding used by SQLite ** and whatever filename encoding is used by the particular Windows -** installation. Memory allocation errors are detected, but -** they are reported back as [SQLITE_CANTOPEN] or +** installation. Memory allocation errors were detected, but +** they were reported back as [SQLITE_CANTOPEN] or ** [SQLITE_IOERR] rather than [SQLITE_NOMEM]. ** ** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] @@ -2086,8 +2378,11 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** [sqlite3_free()] or [sqlite3_realloc()]. */ SQLITE_API void *sqlite3_malloc(int); +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); SQLITE_API void *sqlite3_realloc(void*, int); +SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); SQLITE_API void sqlite3_free(void*); +SQLITE_API sqlite3_uint64 sqlite3_msize(void*); /* ** CAPI3REF: Memory Allocator Statistics @@ -2125,11 +2420,13 @@ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. +** ^If N is less than one, then P can be a NULL pointer. ** -** ^The first time this routine is invoked (either internally or by -** the application) the PRNG is seeded using randomness obtained -** from the xRandomness method of the default [sqlite3_vfs] object. -** ^On all subsequent invocations, the pseudo-randomness is generated +** ^If this routine has not been previously called or if the previous +** call had N less than one, then the PRNG is seeded using randomness +** obtained from the xRandomness method of the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more then +** the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ @@ -2230,8 +2527,8 @@ SQLITE_API int sqlite3_set_authorizer( ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. ** -** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] -** from the [sqlite3_vtab_on_conflict()] interface. +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -2289,6 +2586,7 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_FUNCTION 31 /* NULL Function Name */ #define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ #define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ /* ** CAPI3REF: Tracing And Profiling Functions @@ -2304,6 +2602,9 @@ SQLITE_API int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -2329,9 +2630,10 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, ** interface is to keep a GUI updated during a large query. ** ** ^The parameter P is passed through as the only parameter to the -** callback function X. ^The parameter N is the number of +** callback function X. ^The parameter N is the approximate number of ** [virtual machine instructions] that are evaluated between successive -** invocations of the callback X. +** invocations of the callback X. ^If N is less than one then the progress +** handler is disabled. ** ** ^Only a single progress handler may be defined at one time per ** [database connection]; setting a new progress handler cancels the @@ -2367,9 +2669,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an English language description of the error following a failure of any ** of the sqlite3_open() routines. ** -** ^The default encoding for the database will be UTF-8 if -** sqlite3_open() or sqlite3_open_v2() is called and -** UTF-16 in the native byte order if sqlite3_open16() is used. +** ^The default encoding will be UTF-8 for databases created using +** sqlite3_open() or sqlite3_open_v2(). ^The default encoding for databases +** created using sqlite3_open16() will be UTF-16 in the native byte order. ** ** Whether or not an error occurs when it is opened, resources ** associated with the [database connection] handle should be released by @@ -2457,13 +2759,14 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** then it is interpreted as an absolute path. ^If the path does not begin ** with a '/' (meaning that the authority section is omitted from the URI) ** then the path is interpreted as a relative path. -** ^On windows, the first component of an absolute path -** is a drive specification (e.g. "C:"). +** ^(On windows, the first component of an absolute path +** is a drive specification (e.g. "C:").)^ ** ** [[core URI query parameters]] ** The query component of a URI may contain parameters that are interpreted ** either by SQLite itself, or by a [VFS | custom VFS implementation]. -** SQLite interprets the following three query parameters: +** SQLite and its built-in [VFSes] interpret the +** following query parameters: ** **
      **
    • vfs: ^The "vfs" parameter may be used to specify the name of @@ -2474,18 +2777,20 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** present, then the VFS specified by the option takes precedence over ** the value passed as the fourth parameter to sqlite3_open_v2(). ** -**
    • mode: ^(The mode parameter may be set to either "ro", "rw" or -** "rwc". Attempting to set it to any other value is an error)^. +**
    • mode: ^(The mode parameter may be set to either "ro", "rw", +** "rwc", or "memory". Attempting to set it to any other value is +** an error)^. ** ^If "ro" is specified, then the database is opened for read-only ** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the -** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** third argument to sqlite3_open_v2(). ^If the mode option is set to ** "rw", then the database is opened for read-write (but not create) ** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had ** been set. ^Value "rwc" is equivalent to setting both -** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If sqlite3_open_v2() is -** used, it is an error to specify a value for the mode parameter that is -** less restrictive than that specified by the flags passed as the third -** parameter. +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If the mode option is +** set to "memory" then a pure [in-memory database] that never reads +** or writes from disk is used. ^It is an error to specify a value for +** the mode parameter that is less restrictive than that specified by +** the flags passed in the third parameter to sqlite3_open_v2(). ** **
    • cache: ^The cache parameter may be set to either "shared" or ** "private". ^Setting it to "shared" is equivalent to setting the @@ -2493,8 +2798,30 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
    • psow: ^The psow parameter indicates whether or not the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. +** +**
    • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
    • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** **
    ** ** ^Specifying an unknown parameter in the query component of a URI is not an @@ -2524,8 +2851,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** Open file "data.db" in the current directory for read-only access. ** Regardless of whether or not shared-cache mode is enabled by ** default, use a private cache. -**
file:/home/fred/data.db?vfs=unix-nolock -** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +**
file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. **
file:data.db?mode=readonly ** An error. "readonly" is not a valid option for the "mode" parameter. **
@@ -2544,6 +2872,12 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** codepage is currently defined. Filenames containing international ** characters must be converted to UTF-8 prior to passing them into ** sqlite3_open() or sqlite3_open_v2(). +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling sqlite3_open() or sqlite3_open_v2(). Otherwise, various +** features that require the use of temporary files may fail. +** +** See also: [sqlite3_temp_directory] */ SQLITE_API int sqlite3_open( const char *filename, /* Database filename (UTF-8) */ @@ -2563,21 +2897,45 @@ SQLITE_API int sqlite3_open_v2( /* ** CAPI3REF: Obtain Values For URI Parameters ** -** This is a utility routine, useful to VFS implementations, that checks +** These are utility routines, useful to VFS implementations, that check ** to see if a database file was a URI that contained a specific query -** parameter, and if so obtains the value of the query parameter. +** parameter, and if so obtains the value of that query parameter. ** -** The zFilename argument is the filename pointer passed into the xOpen() -** method of a VFS implementation. The zParam argument is the name of the -** query parameter we seek. This routine returns the value of the zParam -** parameter if it exists. If the parameter does not exist, this routine -** returns a NULL pointer. +** If F is the database filename pointer passed into the xOpen() method of +** a VFS implementation when the flags parameter to xOpen() has one or +** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and +** P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. ** -** If the zFilename argument to this function is not a pointer that SQLite -** passed into the xOpen VFS method, then the behavior of this routine -** is undefined and probably undesirable. +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P is does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that SQLite passed into the xOpen +** VFS method, then the behavior of this routine is undefined and probably +** undesirable. */ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); /* @@ -2599,6 +2957,11 @@ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char * ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -2617,6 +2980,7 @@ SQLITE_API int sqlite3_errcode(sqlite3 *db); SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); SQLITE_API const char *sqlite3_errmsg(sqlite3*); SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); /* ** CAPI3REF: SQL Statement Object @@ -2735,6 +3099,10 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
SQLITE_LIMIT_TRIGGER_DEPTH
**
The maximum depth of recursion for triggers.
)^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
SQLITE_LIMIT_WORKER_THREADS
+**
The maximum number of auxiliary worker threads that a single +** [prepared statement] may start.
)^ ** */ #define SQLITE_LIMIT_LENGTH 0 @@ -2748,6 +3116,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 /* ** CAPI3REF: Compiling An SQL Statement @@ -2773,7 +3142,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** that the supplied string is nul-terminated, then there is a small ** performance advantage to be gained by passing an nByte parameter that ** is equal to the number of bytes in the input string including -** the nul-terminator bytes. +** the nul-terminator bytes as this saves SQLite from having to +** make a copy of the input string. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -2803,7 +3173,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
    • ** **
  • @@ -2824,8 +3195,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ^The specific value of WHERE-clause [parameter] might influence the ** choice of query plan if the parameter is the left-hand side of a [LIKE] ** or [GLOB] operator or if the parameter is compared to an indexed column -** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. -** the +** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. **
    • ** */ @@ -2898,6 +3268,25 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has not run to completion and/or has not +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); + /* ** CAPI3REF: Dynamically Typed Value Object ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} @@ -2988,18 +3377,31 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the ** number of bytes in the value, not the number of characters.)^ -** ^If the fourth parameter is negative, the length of the string is +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is ** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() or sqlite3_bind_text64() then +** that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occur at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. ** -** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and -** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or +** ^The fifth argument to the BLOB and string binding interfaces +** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), -** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** to dispose of the BLOB or string even if the call to bind API fails. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. @@ -3007,6 +3409,14 @@ typedef struct sqlite3_context sqlite3_context; ** SQLite makes its own private copy of the data immediately, before ** the sqlite3_bind_*() routine returns. ** +** ^The sixth argument to sqlite3_bind_text64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_text64() is not one of the +** allowed values shown above, or if the text encoding is different +** from the encoding specified by the sixth parameter, then the behavior +** is undefined. +** ** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that ** is filled with zeroes. ^A zeroblob uses a fixed amount of memory ** (just an integer to hold its size) while it is being processed. @@ -3027,6 +3437,9 @@ typedef struct sqlite3_context sqlite3_context; ** ** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an ** [error code] if anything goes wrong. +** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB +** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or +** [SQLITE_MAX_LENGTH]. ** ^[SQLITE_RANGE] is returned if the parameter ** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. ** @@ -3034,12 +3447,16 @@ typedef struct sqlite3_context sqlite3_context; ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, + void(*)(void*)); SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); -SQLITE_API int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, + void(*)(void*), unsigned char encoding); SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); @@ -3327,6 +3744,12 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of ** interfaces) then sqlite3_data_count(P) returns 0. ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. ** ** See also: [sqlite3_column_count()] */ @@ -3426,7 +3849,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), -** even empty strings, are always zero terminated. ^The return +** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an @@ -3449,19 +3872,19 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** **
    NULL INTEGER Result is 0 **
    NULL FLOAT Result is 0.0 -**
    NULL TEXT Result is NULL pointer -**
    NULL BLOB Result is NULL pointer +**
    NULL TEXT Result is a NULL pointer +**
    NULL BLOB Result is a NULL pointer **
    INTEGER FLOAT Convert from integer to float **
    INTEGER TEXT ASCII rendering of the integer **
    INTEGER BLOB Same as INTEGER->TEXT -**
    FLOAT INTEGER Convert from float to integer +**
    FLOAT INTEGER [CAST] to INTEGER **
    FLOAT TEXT ASCII rendering of the float -**
    FLOAT BLOB Same as FLOAT->TEXT -**
    TEXT INTEGER Use atoi() -**
    TEXT FLOAT Use atof() +**
    FLOAT BLOB [CAST] to BLOB +**
    TEXT INTEGER [CAST] to INTEGER +**
    TEXT FLOAT [CAST] to REAL **
    TEXT BLOB No change -**
    BLOB INTEGER Convert to TEXT then use atoi() -**
    BLOB FLOAT Convert to TEXT then use atof() +**
    BLOB INTEGER [CAST] to INTEGER +**
    BLOB FLOAT [CAST] to REAL **
    BLOB TEXT Add a zero terminator if needed **
    ** )^ @@ -3517,7 +3940,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings ** and BLOBs is freed automatically. Do not pass the pointers returned -** [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** ** ^(If a memory allocation error occurs during the evaluation of any @@ -3626,15 +4049,24 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Every SQL function implementation must be able to work -** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be -** more efficient with one encoding than another. ^An application may -** invoke sqlite3_create_function() or sqlite3_create_function16() multiple -** times with the same function but with different values of eTextRep. +** its parameters. The application should set this parameter to +** [SQLITE_UTF16LE] if the function implementation invokes +** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the +** implementation invokes [sqlite3_value_text16be()] on an input, or +** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] +** otherwise. ^The same SQL function may be registered multiple times using +** different preferred text encodings, with different implementations for +** each encoding. ** ^When multiple implementations of the same function are available, SQLite ** will pick the one that involves the least amount of data conversion. -** If there is only a single implementation which does not care what text -** encoding is used, then the fourth argument should be [SQLITE_ANY]. +** +** ^The fourth parameter may optionally be ORed with [SQLITE_DETERMINISTIC] +** to signal that the function will always return the same result given +** the same inputs within a single SQL statement. Most SQL functions are +** deterministic. The built-in [random()] SQL function is an example of a +** function that is not deterministic. The SQLite query planner is able to +** perform additional optimizations on deterministic functions, so use +** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. ** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ @@ -3720,9 +4152,19 @@ SQLITE_API int sqlite3_create_function_v2( #define SQLITE_UTF16LE 2 #define SQLITE_UTF16BE 3 #define SQLITE_UTF16 4 /* Use native byte order */ -#define SQLITE_ANY 5 /* sqlite3_create_function only */ +#define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ +/* +** CAPI3REF: Function Flags +** +** These constants may be ORed together with the +** [SQLITE_UTF8 | preferred text encoding] as the fourth argument +** to [sqlite3_create_function()], [sqlite3_create_function16()], or +** [sqlite3_create_function_v2()]. +*/ +#define SQLITE_DETERMINISTIC 0x800 + /* ** CAPI3REF: Deprecated Functions ** DEPRECATED @@ -3739,7 +4181,8 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -3762,7 +4205,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** object results in undefined behavior. ** ** ^These routines work just like the corresponding [column access functions] -** except that these routines take a single [protected sqlite3_value] object +** except that these routines take a single [protected sqlite3_value] object ** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. ** ** ^The sqlite3_value_text16() interface extracts a UTF-16 string @@ -3819,14 +4262,17 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. @@ -3869,41 +4315,49 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data ** -** The following two functions may be used by scalar SQL functions to +** These functions may be used by (non-aggregate) SQL functions to ** associate metadata with argument values. If the same value is passed to ** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated metadata may be preserved. This may -** be used, for example, to add a regular-expression matching scalar -** function. The compiled version of the regular expression is stored as -** metadata associated with the SQL value passed as the regular expression -** pattern. The compiled regular expression can be reused on multiple -** invocations of the same function so that the original pattern string -** does not need to be recompiled on each invocation. +** some circumstances the associated metadata may be preserved. An example +** of where this might be useful is in a regular-expression matching +** function. The compiled version of the regular expression can be stored as +** metadata associated with the pattern string. +** Then as long as the pattern string remains the same, +** the compiled regular expression can be reused on multiple +** invocations of the same function. ** ** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata ** associated by the sqlite3_set_auxdata() function with the Nth argument -** value to the application-defined function. ^If no metadata has been ever -** been set for the Nth argument of the function, or if the corresponding -** function parameter has changed since the meta-data was set, -** then sqlite3_get_auxdata() returns a NULL pointer. +** value to the application-defined function. ^If there is no metadata +** associated with the function argument, this sqlite3_get_auxdata() interface +** returns a NULL pointer. ** -** ^The sqlite3_set_auxdata() interface saves the metadata -** pointed to by its 3rd parameter as the metadata for the N-th -** argument of the application-defined function. Subsequent -** calls to sqlite3_get_auxdata() might return this data, if it has -** not been destroyed. -** ^If it is not NULL, SQLite will invoke the destructor -** function given by the 4th parameter to sqlite3_set_auxdata() on -** the metadata when the corresponding function parameter changes -** or when the SQL statement completes, whichever comes first. +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th +** argument of the application-defined function. ^Subsequent +** calls to sqlite3_get_auxdata(C,N) return P from the most recent +** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or +** NULL if the metadata has been discarded. +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, +** SQLite will invoke the destructor function X with parameter P exactly +** once, when the metadata is discarded. +** SQLite is free to discard the metadata at any time, including:
      +**
    • when the corresponding function parameter changes, or +**
    • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement, or +**
    • when sqlite3_set_auxdata() is invoked again on the same parameter, or +**
    • during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.
    )^ ** -** SQLite is free to call the destructor and drop metadata on any -** parameter of any function at any time. ^The only guarantee is that -** the destructor will be called before the metadata is dropped. +** Note the last bullet in particular. The destructor X in +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the +** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() +** should be called near the end of the function implementation and the +** function implementation should not make any use of P after +** sqlite3_set_auxdata() has been called. ** ** ^(In practice, metadata is preserved between function calls for -** expressions that are constant at compile time. This includes literal -** values and [parameters].)^ +** function parameters that are compile-time constants, including literal +** values and [parameters] and expressions composed from the same.)^ ** ** These routines must be called from the same thread in which ** the SQL function is running. @@ -3924,7 +4378,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -3977,11 +4431,11 @@ typedef void (*sqlite3_destructor_type)(void*); ** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() ** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. ** -** ^The sqlite3_result_toobig() interface causes SQLite to throw an error -** indicating that a string or BLOB is too long to represent. +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. ** -** ^The sqlite3_result_nomem() interface causes SQLite to throw an error -** indicating that a memory allocation failed. +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. ** ** ^The sqlite3_result_int() interface sets the return value ** of the application-defined function to be the 32-bit signed integer @@ -3998,6 +4452,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_text64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces @@ -4006,7 +4464,12 @@ typedef void (*sqlite3_destructor_type)(void*); ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined -** function result. +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. ** ^If the 4th parameter to the sqlite3_result_text* interfaces ** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that ** function as the destructor on the text or BLOB result when it has @@ -4036,6 +4499,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** the [sqlite3_context] pointer, the results are undefined. */ SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); SQLITE_API void sqlite3_result_double(sqlite3_context*, double); SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); @@ -4046,6 +4510,8 @@ SQLITE_API void sqlite3_result_int(sqlite3_context*, int); SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); SQLITE_API void sqlite3_result_null(sqlite3_context*); SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); @@ -4203,6 +4669,11 @@ SQLITE_API int sqlite3_key( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The key */ ); +SQLITE_API int sqlite3_key_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The key */ +); /* ** Change the key on an open database. If the current database is not @@ -4216,6 +4687,11 @@ SQLITE_API int sqlite3_rekey( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The new key */ ); +SQLITE_API int sqlite3_rekey_v2( + sqlite3 *db, /* Database to be rekeyed */ + const char *zDbName, /* Name of the database */ + const void *pKey, int nKey /* The new key */ +); /* ** Specify the activation key for a SEE database. Unless @@ -4265,6 +4741,13 @@ SQLITE_API int sqlite3_sleep(int); ** is a NULL pointer, then SQLite performs a search for an appropriate ** temporary file directory. ** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** ** It is not safe to read or modify this variable in more than one ** thread at a time. It is not safe to read or modify this variable ** if a [database connection] is being used at the same time in a separate @@ -4283,9 +4766,66 @@ SQLITE_API int sqlite3_sleep(int); ** Hence, if this variable is modified directly, either it should be ** made NULL or made to point to memory obtained from [sqlite3_malloc] ** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various +** features that require the use of temporary files may fail. Here is an +** example of how to do this using C++ with the Windows Runtime: +** +** 
    +** LPCWSTR zPath = Windows::Storage::ApplicationData::Current->
+**       TemporaryFolder->Path->Data();
+** char zPathBuf[MAX_PATH + 1];
+** memset(zPathBuf, 0, sizeof(zPathBuf));
+** WideCharToMultiByte(CP_UTF8, 0, zPath, -1, zPathBuf, sizeof(zPathBuf),
+**       NULL, NULL);
+** sqlite3_temp_directory = sqlite3_mprintf("%s", zPathBuf);
+**
    */ SQLITE_API SQLITE_EXTERN char *sqlite3_temp_directory; +/* +** CAPI3REF: Name Of The Folder Holding Database Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all database files +** specified with a relative pathname and created or accessed by +** SQLite when using a built-in windows [sqlite3_vfs | VFS] will be assumed +** to be relative to that directory.)^ ^If this variable is a NULL +** pointer, then SQLite assumes that all database files specified +** with a relative pathname are relative to the current directory +** for the process. Only the windows VFS makes use of this global +** variable; it is ignored by the unix VFS. +** +** Changing the value of this variable while a database connection is +** open can result in a corrupt database. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [data_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [data_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [data_store_directory pragma] should be avoided. +*/ +SQLITE_API SQLITE_EXTERN char *sqlite3_data_directory; + /* ** CAPI3REF: Test For Auto-Commit Mode ** KEYWORDS: {autocommit mode} @@ -4321,6 +4861,31 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); */ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +/* +** CAPI3REF: Return The Filename For A Database Connection +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename +** associated with database N of connection D. ^The main database file +** has the name "main". If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** a NULL pointer is returned. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + /* ** CAPI3REF: Find the next prepared statement ** @@ -4356,13 +4921,15 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); ** on the same [database connection] D, or NULL for ** the first call for each function on D. ** +** The commit and rollback hook callbacks are not reentrant. ** The callback implementation must not do anything that will modify ** the database connection that invoked the callback. Any actions ** to modify the database connection must be deferred until after the ** completion of the [sqlite3_step()] call that triggered the commit ** or rollback hook in the first place. -** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their -** database connections for the meaning of "modify" in this paragraph. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. ** ** ^Registering a NULL function disables the callback. ** @@ -4388,12 +4955,13 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument -** to be invoked whenever a row is updated, inserted or deleted. +** to be invoked whenever a row is updated, inserted or deleted in +** a rowid table. ** ^Any callback set by a previous call to this function ** for the same database connection is overridden. ** ** ^The second argument is a pointer to the function to invoke when a -** row is updated, inserted or deleted. +** row is updated, inserted or deleted in a rowid table. ** ^The first argument to the callback is a copy of the third argument ** to sqlite3_update_hook(). ** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], @@ -4406,6 +4974,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ** ^(The update hook is not invoked when internal system tables are ** modified (i.e. sqlite_master and sqlite_sequence).)^ +** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. ** ** ^In the current implementation, the update hook ** is not invoked when duplication rows are deleted because of an @@ -4437,7 +5006,6 @@ SQLITE_API void *sqlite3_update_hook( /* ** CAPI3REF: Enable Or Disable Shared Pager Cache -** KEYWORDS: {shared cache} ** ** ^(This routine enables or disables the sharing of the database cache ** and schema data structures between [database connection | connections] @@ -4460,6 +5028,9 @@ SQLITE_API void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** ** See Also: [SQLite Shared-Cache Mode] */ SQLITE_API int sqlite3_enable_shared_cache(int); @@ -4475,9 +5046,24 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** which might be more or less than the amount requested. ** ^The sqlite3_release_memory() routine is a no-op returning zero ** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** See also: [sqlite3_db_release_memory()] */ SQLITE_API int sqlite3_release_memory(int); +/* +** CAPI3REF: Free Memory Used By A Database Connection +** +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is in effect even +** when the [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. +** +** See also: [sqlite3_release_memory()] +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3*); + /* ** CAPI3REF: Impose A Limit On Heap Size ** @@ -4492,7 +5078,8 @@ SQLITE_API int sqlite3_release_memory(int); ** is advisory only. ** ** ^The return value from sqlite3_soft_heap_limit64() is the size of -** the soft heap limit prior to the call. ^If the argument N is negative +** the soft heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative ** then no change is made to the soft heap limit. Hence, the current ** size of the soft heap limit can be determined by invoking ** sqlite3_soft_heap_limit64() with a negative argument. @@ -4508,7 +5095,7 @@ SQLITE_API int sqlite3_release_memory(int); ** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and ** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. **
  • An alternative page cache implementation is specified using -** [sqlite3_config]([SQLITE_CONFIG_PCACHE],...). +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). **
  • The page cache allocates from its own memory pool supplied ** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than ** from the heap. @@ -4621,11 +5208,20 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -4651,11 +5247,11 @@ SQLITE_API int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -4667,7 +5263,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes @@ -4695,10 +5291,23 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** on the list of automatic extensions is a harmless no-op. ^No entry point ** will be called more than once for each database connection that is opened. ** -** See also: [sqlite3_reset_auto_extension()]. +** See also: [sqlite3_reset_auto_extension()] +** and [sqlite3_cancel_auto_extension()] */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); +/* +** CAPI3REF: Cancel Automatic Extension Loading +** +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the +** initialization routine X that was registered using a prior call to +** [sqlite3_auto_extension(X)]. ^The [sqlite3_cancel_auto_extension(X)] +** routine returns 1 if initialization routine X was successfully +** unregistered and it returns 0 if X was not on the list of initialization +** routines. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void)); + /* ** CAPI3REF: Reset Automatic Extension Loading ** @@ -4823,10 +5432,22 @@ struct sqlite3_module { ** the correct order to satisfy the ORDER BY clause so that no separate ** sorting step is required. ** -** ^The estimatedCost value is an estimate of the cost of doing the -** particular lookup. A full scan of a table with N entries should have -** a cost of N. A binary search of a table of N entries should have a -** cost of approximately log(N). +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite version 3.8.2. If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to included crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. */ struct sqlite3_index_info { /* Inputs */ @@ -4851,7 +5472,9 @@ struct sqlite3_index_info { char *idxStr; /* String, possibly obtained from sqlite3_malloc */ int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ int orderByConsumed; /* True if output is already ordered */ - double estimatedCost; /* Estimated cost of using this index */ + double estimatedCost; /* Estimated cost of using this index */ + /* Fields below are only available in SQLite 3.8.2 and later */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ }; /* @@ -5055,6 +5678,9 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** +** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] +** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. +** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces ** and the built-in [zeroblob] SQL function can be used, if desired, ** to create an empty, zero-filled blob in which to read or write using @@ -5249,17 +5875,16 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** implementations are available in the SQLite core: ** **
      -**
    • SQLITE_MUTEX_OS2 -**
    • SQLITE_MUTEX_PTHREAD +**
    • SQLITE_MUTEX_PTHREADS **
    • SQLITE_MUTEX_W32 **
    • SQLITE_MUTEX_NOOP **
    )^ ** ** ^The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_OS2, -** SQLITE_MUTEX_PTHREAD, and SQLITE_MUTEX_W32 implementations -** are appropriate for use on OS/2, Unix, and Windows. +** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix +** and Windows. ** ** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex @@ -5280,10 +5905,12 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU -**
  • SQLITE_MUTEX_STATIC_LRU2 +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** )^ ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -5448,7 +6075,7 @@ struct sqlite3_mutex_methods { ** ^These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provided versions of these +** ^The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. @@ -5487,6 +6114,9 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection @@ -5576,10 +6206,15 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 -#define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 -#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19 -#define SQLITE_TESTCTRL_LAST 19 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ +#define SQLITE_TESTCTRL_NEVER_CORRUPT 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_SORTER_MMAP 24 +#define SQLITE_TESTCTRL_LAST 24 /* ** CAPI3REF: SQLite Runtime Status @@ -5770,12 +6405,12 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** the current value is always zero.)^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
    SQLITE_DBSTATUS_CACHE_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ ** ^The full amount of memory used by the schemas is reported, even if the @@ -5784,11 +6419,40 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
    SQLITE_DBSTATUS_STMT_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. **
    +** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
    SQLITE_DBSTATUS_CACHE_HIT
    +**
    This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
    SQLITE_DBSTATUS_CACHE_MISS
    +**
    This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
    SQLITE_DBSTATUS_CACHE_WRITE
    +**
    This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
    +** +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
    SQLITE_DBSTATUS_DEFERRED_FKS
    +**
    This parameter returns zero for the current value if and only if +** all foreign key constraints (deferred or immediate) have been +** resolved.)^ ^The highwater mark is always 0. +**
    ** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -5798,7 +6462,11 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 -#define SQLITE_DBSTATUS_MAX 6 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_DEFERRED_FKS 10 +#define SQLITE_DBSTATUS_MAX 10 /* Largest defined DBSTATUS */ /* @@ -5853,11 +6521,20 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** improvement performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run. ** +** [[SQLITE_STMTSTATUS_VM_STEP]]
    SQLITE_STMTSTATUS_VM_STEP
    +**
    ^This is the number of virtual machine operations executed +** by the prepared statement if that number is less than or equal +** to 2147483647. The number of virtual machine operations can be +** used as a proxy for the total work done by the prepared statement. +** If the number of virtual machine operations exceeds 2147483647 +** then the value returned by this statement status code is undefined. +**
    ** */ #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 #define SQLITE_STMTSTATUS_SORT 2 #define SQLITE_STMTSTATUS_AUTOINDEX 3 +#define SQLITE_STMTSTATUS_VM_STEP 4 /* ** CAPI3REF: Custom Page Cache Object @@ -5868,17 +6545,33 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** sqlite3_pcache object except by holding and passing pointers ** to the object. ** -** See [sqlite3_pcache_methods] for additional information. +** See [sqlite3_pcache_methods2] for additional information. */ typedef struct sqlite3_pcache sqlite3_pcache; +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + /* ** CAPI3REF: Application Defined Page Cache. ** KEYWORDS: {page cache} ** -** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ +** instance of the sqlite3_pcache_methods2 structure.)^ ** In many applications, most of the heap memory allocated by ** SQLite is used for the page cache. ** By implementing a @@ -5892,7 +6585,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** extreme measure that is only needed by the most demanding applications. ** The built-in page cache is recommended for most uses. ** -** ^(The contents of the sqlite3_pcache_methods structure are copied to an +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ @@ -5901,7 +6594,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() -** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ ** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. ** ^(If the xInit() method is NULL, then the @@ -5928,17 +6621,15 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. ^szPage will not be a power of two. ^szPage -** will the page size of the database file that is to be cached plus an -** increment (here called "R") of less than 250. SQLite will use the -** extra R bytes on each page to store metadata about the underlying -** database page on disk. The value of R depends +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends ** on the SQLite version, the target platform, and how SQLite was compiled. -** ^(R is constant for a particular build of SQLite. Except, there are two -** distinct values of R when SQLite is compiled with the proprietary -** ZIPVFS extension.)^ ^The second argument to -** xCreate(), bPurgeable, is true if the cache being created will -** be used to cache database pages of a file stored on disk, or +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will @@ -5962,11 +6653,16 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ** [[the xFetch() page cache methods]] ** The xFetch() method locates a page in the cache and returns a pointer to -** the page, or a NULL pointer. -** A "page", in this context, means a buffer of szPage bytes aligned at an -** 8-byte boundary. The page to be fetched is determined by the key. ^The -** minimum key value is 1. After it has been retrieved using xFetch, the page -** is considered to be "pinned". +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". ** ** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content @@ -5975,7 +6671,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** parameter to help it determined what action to take: ** ** -**
    createFlag Behaviour when page is not already in cache +**
    createFlag Behavior when page is not already in cache **
    0 Do not allocate a new page. Return NULL. **
    1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. @@ -6019,8 +6715,37 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] -** handle invalid, and will not use it with any other sqlite3_pcache_methods +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 ** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. */ typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; struct sqlite3_pcache_methods { @@ -6037,6 +6762,7 @@ struct sqlite3_pcache_methods { void (*xDestroy)(sqlite3_pcache*); }; + /* ** CAPI3REF: Online Backup Object ** @@ -6366,17 +7092,33 @@ SQLITE_API int sqlite3_unlock_notify( /* ** CAPI3REF: String Comparison ** -** ^The [sqlite3_strnicmp()] API allows applications and extensions to -** compare the contents of two buffers containing UTF-8 strings in a -** case-independent fashion, using the same definition of case independence -** that SQLite uses internally when comparing identifiers. +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ +SQLITE_API int sqlite3_stricmp(const char *, const char *); SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + /* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. @@ -6456,6 +7198,9 @@ SQLITE_API void *sqlite3_wal_hook( ** ^The [wal_autocheckpoint pragma] can be used to invoke this interface ** from SQL. ** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] ** pages. The use of this interface @@ -6472,6 +7217,10 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** empty string, then a checkpoint is run on all databases of ** connection D. ^If the database connection D is not in ** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a +** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. +** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL +** or RESET checkpoint. ** ** ^The [wal_checkpoint pragma] can be used to invoke this interface ** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the @@ -6494,10 +7243,12 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** Checkpoint as many frames as possible without waiting for any database ** readers or writers to finish. Sync the db file if all frames in the log ** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] +** is never invoked. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (calls the busy-handler callback) until there is no +** This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database ** snapshot. It then checkpoints all frames in the log file and syncs the ** database file. This call blocks database writers while it is running, @@ -6505,7 +7256,8 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** **
    SQLITE_CHECKPOINT_RESTART
    ** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the busy-handler callback) +** checkpointing the log file it blocks (calls the +** [sqlite3_busy_handler|busy-handler callback]) ** until all readers are reading from the database file only. This ensures ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, @@ -6643,6 +7395,7 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); /* ** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to ** inform a [virtual table] implementation what the [ON CONFLICT] mode @@ -6671,7 +7424,7 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif -#endif +#endif /* _SQLITE3_H_ */ /* ** 2010 August 30 @@ -6695,6 +7448,16 @@ extern "C" { #endif typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif /* ** Register a geometry callback named zGeom that can be used as part of an @@ -6705,7 +7468,7 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; SQLITE_API int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), void *pContext ); @@ -6717,11 +7480,60 @@ SQLITE_API int sqlite3_rtree_geometry_callback( struct sqlite3_rtree_geometry { void *pContext; /* Copy of pContext passed to s_r_g_c() */ int nParam; /* Size of array aParam[] */ - double *aParam; /* Parameters passed to SQL geom function */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ void *pUser; /* Callback implementation user data */ void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ }; +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + #ifdef __cplusplus } /* end of the 'extern "C"' block */