diff options
author | Hans-Christoph Steiner <hans@eds.org> | 2013-08-13 15:42:54 -0400 |
---|---|---|
committer | Hans-Christoph Steiner <hans@eds.org> | 2013-08-13 15:42:54 -0400 |
commit | 08119c361d1181b3e8f1abb429236e488a664753 (patch) | |
tree | 77e5a8b6d411ca32c360c7e48df5c293b1e0baac /tool | |
parent | 1b5ba8e022836fa8ab93bc90df1b34a29ea6e134 (diff) |
Imported Upstream version 2.2.1
Diffstat (limited to 'tool')
-rwxr-xr-x | tool/build-all-msvc.bat | 320 | ||||
-rw-r--r-- | tool/build-shell.sh | 3 | ||||
-rw-r--r-- | tool/mksqlite3c.tcl | 8 | ||||
-rw-r--r-- | tool/mksqlite3h.tcl | 9 | ||||
-rw-r--r-- | tool/mkvsix.tcl | 368 | ||||
-rw-r--r-- | tool/showdb.c | 111 | ||||
-rw-r--r-- | tool/showwal.c | 293 | ||||
-rw-r--r-- | tool/spaceanal.tcl | 27 | ||||
-rw-r--r-- | tool/stack_usage.tcl | 98 | ||||
-rw-r--r-- | tool/vdbe-compress.tcl | 3 | ||||
-rw-r--r-- | tool/win/sqlite.vsix | bin | 32802 -> 32816 bytes |
11 files changed, 1005 insertions, 235 deletions
diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index a2d7dae..758036f 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -6,10 +6,54 @@ :: Multi-Platform Build Tool for MSVC
::
+REM
+REM This batch script is used to build the SQLite DLL for multiple platforms
+REM and configurations using MSVC. The built SQLite DLLs, their associated
+REM import libraries, and optionally their symbols files, are placed within
+REM the directory specified on the command line, in sub-directories named for
+REM their respective platforms and configurations. This batch script must be
+REM run from inside a Visual Studio Command Prompt for the desired version of
+REM Visual Studio ^(the initial platform configured for the command prompt does
+REM not really matter^). Exactly one command line argument is required, the
+REM name of an existing directory to be used as the final destination directory
+REM for the generated output files, which will be placed in sub-directories
+REM created therein. Ideally, the directory specified should be empty.
+REM
+REM Example:
+REM
+REM CD /D C:\dev\sqlite\core
+REM tool\build-all-msvc.bat C:\Temp
+REM
+REM In the example above, "C:\dev\sqlite\core" represents the root of the
+REM source tree for SQLite and "C:\Temp" represents the final destination
+REM directory for the generated output files.
+REM
+REM There are several environment variables that may be set to modify the
+REM behavior of this batch script and its associated Makefile. The list of
+REM platforms to build may be overriden by using the PLATFORMS environment
+REM variable, which should contain a list of platforms ^(e.g. x86 x86_amd64
+REM x86_arm^). All platforms must be supported by the version of Visual Studio
+REM being used. The list of configurations to build may be overridden by
+REM setting the CONFIGURATIONS environment variable, which should contain a
+REM list of configurations to build ^(e.g. Debug Retail^). Neither of these
+REM variable values may contain any double quotes, surrounding or embedded.
+REM Finally, the NCRTLIBPATH and NSDKLIBPATH environment variables may be set
+REM to specify the location of the CRT and SDK, respectively, needed to compile
+REM executables native to the architecture of the build machine during any
+REM cross-compilation that may be necessary, depending on the platforms to be
+REM built. These values in these two variables should be surrounded by double
+REM quotes if they contain spaces.
+REM
+REM Please note that the SQLite build process performed by the Makefile
+REM associated with this batch script requires both Gawk ^(gawk.exe^) and Tcl
+REM 8.5 ^(tclsh85.exe^) to be present in a directory contained in the PATH
+REM environment variable unless a pre-existing amalgamation file is used.
+REM
SETLOCAL
REM SET __ECHO=ECHO
REM SET __ECHO2=ECHO
+REM SET __ECHO3=ECHO
IF NOT DEFINED _AECHO (SET _AECHO=REM)
IF NOT DEFINED _CECHO (SET _CECHO=REM)
IF NOT DEFINED _VECHO (SET _VECHO=REM)
@@ -93,17 +137,35 @@ IF NOT DEFINED PLATFORMS ( %_VECHO% Platforms = '%PLATFORMS%'
REM
+REM NOTE: If the list of configurations is not already set, use the default
+REM list.
+REM
+IF NOT DEFINED CONFIGURATIONS (
+ SET CONFIGURATIONS=Debug Retail
+)
+
+%_VECHO% Configurations = '%CONFIGURATIONS%'
+
+REM
REM NOTE: Setup environment variables to translate between the MSVC platform
REM names and the names to be used for the platform-specific binary
REM directories.
REM
+SET amd64_NAME=x64
+SET arm_NAME=ARM
+SET x64_NAME=x64
SET x86_NAME=x86
SET x86_amd64_NAME=x64
SET x86_arm_NAME=ARM
+SET x86_x64_NAME=x64
+%_VECHO% amd64_Name = '%amd64_NAME%'
+%_VECHO% arm_Name = '%arm_NAME%'
+%_VECHO% x64_Name = '%x64_NAME%'
%_VECHO% x86_Name = '%x86_NAME%'
%_VECHO% x86_amd64_Name = '%x86_amd64_NAME%'
%_VECHO% x86_arm_Name = '%x86_arm_NAME%'
+%_VECHO% x86_x64_Name = '%x86_x64_NAME%'
REM
REM NOTE: Check for the external tools needed during the build process ^(i.e.
@@ -115,6 +177,24 @@ FOR %%T IN (gawk.exe tclsh85.exe) DO ( )
REM
+REM NOTE: The Gawk executable "gawk.exe" is required during the SQLite build
+REM process unless a pre-existing amalgamation file is used.
+REM
+IF NOT DEFINED gawk.exe_PATH (
+ ECHO The Gawk executable "gawk.exe" is required to be in the PATH.
+ GOTO errors
+)
+
+REM
+REM NOTE: The Tcl 8.5 executable "tclsh85.exe" is required during the SQLite
+REM build process unless a pre-existing amalgamation file is used.
+REM
+IF NOT DEFINED tclsh85.exe_PATH (
+ ECHO The Tcl 8.5 executable "tclsh85.exe" is required to be in the PATH.
+ GOTO errors
+)
+
+REM
REM NOTE: Set the TOOLPATH variable to contain all the directories where the
REM external tools were found in the search above.
REM
@@ -127,12 +207,31 @@ REM NOTE: Check for MSVC 2012 because the Windows SDK directory handling is REM slightly different for that version.
REM
IF "%VisualStudioVersion%" == "11.0" (
- SET SET_NSDKLIBPATH=1
+ REM
+ REM NOTE: If the Windows SDK library path has already been set, do not set
+ REM it to something else later on.
+ REM
+ IF NOT DEFINED NSDKLIBPATH (
+ SET SET_NSDKLIBPATH=1
+ )
) ELSE (
CALL :fn_UnsetVariable SET_NSDKLIBPATH
)
REM
+REM NOTE: Check if this is the Windows Phone SDK. If so, a different batch
+REM file is necessary to setup the build environment. Since the variable
+REM values involved here may contain parenthesis, using GOTO instead of
+REM an IF block is required.
+REM
+IF DEFINED WindowsPhoneKitDir GOTO set_vcvarsall_phone
+SET VCVARSALL=%VCINSTALLDIR%\vcvarsall.bat
+GOTO set_vcvarsall_done
+:set_vcvarsall_phone
+SET VCVARSALL=%VCINSTALLDIR%\WPSDK\WP80\vcvarsphoneall.bat
+:set_vcvarsall_done
+
+REM
REM NOTE: This is the outer loop. There should be exactly one iteration per
REM platform.
REM
@@ -142,7 +241,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM be used for the name of the platform-specific binary directory via
REM the environment variables setup earlier.
REM
- CALL :fn_SetVariable %%P_NAME PLATFORMNAME
+ CALL :fn_CopyVariable %%P_NAME PLATFORMNAME
REM
REM NOTE: This is the inner loop. There should be exactly one iteration.
@@ -172,6 +271,7 @@ FOR %%P IN (%PLATFORMS%) DO ( CALL :fn_UnsetVariable Platform
REM CALL :fn_UnsetVariable VCINSTALLDIR
CALL :fn_UnsetVariable VSINSTALLDIR
+ CALL :fn_UnsetVariable WindowsPhoneKitDir
CALL :fn_UnsetVariable WindowsSdkDir
CALL :fn_UnsetVariable WindowsSdkDir_35
CALL :fn_UnsetVariable WindowsSdkDir_old
@@ -181,129 +281,151 @@ FOR %%P IN (%PLATFORMS%) DO ( REM
SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot%
- REM
- REM NOTE: Launch a nested command shell to perform the following steps:
- REM
- REM 1. Setup the MSVC environment for this platform using the
- REM official batch file.
- REM
- REM 2. Make sure that no stale build output files are present.
- REM
- REM 3. Build the "sqlite3.dll" and "sqlite3.lib" binaries for this
- REM platform.
- REM
- REM 4. Copy the "sqlite3.dll" and "sqlite3.lib" binaries for this
- REM platform to the platform-specific directory beneath the
- REM binary directory.
- REM
- "%ComSpec%" /C (
+ FOR %%B IN (%CONFIGURATIONS%) DO (
REM
- REM NOTE: Attempt to setup the MSVC environment for this platform.
+ REM NOTE: When preparing the debug build, set the DEBUG and MEMDEBUG
+ REM environment variables to be picked up by the MSVC makefile
+ REM itself.
REM
- %__ECHO% CALL "%VCINSTALLDIR%\vcvarsall.bat" %%P
-
- IF ERRORLEVEL 1 (
- ECHO Failed to call "%VCINSTALLDIR%\vcvarsall.bat" for platform %%P.
- GOTO errors
+ IF /I "%%B" == "Debug" (
+ SET DEBUG=2
+ SET MEMDEBUG=1
+ ) ELSE (
+ CALL :fn_UnsetVariable DEBUG
+ CALL :fn_UnsetVariable MEMDEBUG
)
REM
- REM NOTE: If this batch file is not running in "what-if" mode, check to
- REM be sure we were actually able to setup the MSVC environment as
- REM current versions of their official batch file do not set the
- REM exit code upon failure.
+ REM NOTE: Launch a nested command shell to perform the following steps:
REM
- IF NOT DEFINED __ECHO (
- IF NOT DEFINED WindowsSdkDir (
- ECHO Cannot build, Windows SDK not found for platform %%P.
- GOTO errors
- )
- )
-
+ REM 1. Setup the MSVC environment for this platform using the
+ REM official batch file.
REM
- REM NOTE: When using MSVC 2012, the native SDK path cannot simply use
- REM the "lib" sub-directory beneath the location specified in the
- REM WindowsSdkDir environment variable because that location does
- REM not actually contain the necessary library files for x86.
- REM This must be done for each iteration because it relies upon
- REM the WindowsSdkDir environment variable being set by the batch
- REM file used to setup the MSVC environment.
+ REM 2. Make sure that no stale build output files are present.
REM
- IF DEFINED SET_NSDKLIBPATH (
- CALL :fn_SetVariable WindowsSdkDir NSDKLIBPATH
- CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86
- )
-
+ REM 3. Build the "sqlite3.dll" and "sqlite3.lib" binaries for this
+ REM platform.
REM
- REM NOTE: Unless prevented from doing so, invoke NMAKE with the MSVC
- REM makefile to clean any stale build output from previous
- REM iterations of this loop and/or previous runs of this batch
- REM file, etc.
+ REM 4. Copy the "sqlite3.dll" and "sqlite3.lib" binaries for this
+ REM platform to the platform-specific directory beneath the
+ REM binary directory.
REM
- IF NOT DEFINED NOCLEAN (
- %__ECHO% nmake -f Makefile.msc clean
+ "%ComSpec%" /C (
+ REM
+ REM NOTE: Attempt to setup the MSVC environment for this platform.
+ REM
+ %__ECHO3% CALL "%VCVARSALL%" %%P
IF ERRORLEVEL 1 (
- ECHO Failed to clean for platform %%P.
+ ECHO Failed to call "%VCVARSALL%" for platform %%P.
GOTO errors
)
- ) ELSE (
+
REM
- REM NOTE: Even when the cleaning step has been disabled, we still need
- REM to remove the build output for the files we are specifically
- REM wanting to build for each platform.
+ REM NOTE: If this batch file is not running in "what-if" mode, check to
+ REM be sure we were actually able to setup the MSVC environment
+ REM as current versions of their official batch file do not set
+ REM the exit code upon failure.
REM
- %__ECHO% DEL /Q sqlite3.dll sqlite3.lib sqlite3.pdb
- )
+ IF NOT DEFINED __ECHO3 (
+ IF NOT DEFINED WindowsPhoneKitDir (
+ IF NOT DEFINED WindowsSdkDir (
+ ECHO Cannot build, Windows SDK not found for platform %%P.
+ GOTO errors
+ )
+ )
+ )
- REM
- REM NOTE: Invoke NMAKE with the MSVC makefile to build the "sqlite3.dll"
- REM binary. The x86 compiler will be used to compile the native
- REM command line tools needed during the build process itself.
- REM Also, disable looking for and/or linking to the native Tcl
- REM runtime library.
- REM
- %__ECHO% nmake -f Makefile.msc sqlite3.dll "NCC=""%VCINSTALLDIR%\bin\cl.exe""" USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS%
+ REM
+ REM NOTE: When using MSVC 2012, the native SDK path cannot simply use
+ REM the "lib" sub-directory beneath the location specified in the
+ REM WindowsSdkDir environment variable because that location does
+ REM not actually contain the necessary library files for x86.
+ REM This must be done for each iteration because it relies upon
+ REM the WindowsSdkDir environment variable being set by the batch
+ REM file used to setup the MSVC environment.
+ REM
+ IF DEFINED SET_NSDKLIBPATH (
+ IF DEFINED WindowsPhoneKitDir (
+ CALL :fn_CopyVariable WindowsPhoneKitDir NSDKLIBPATH
+ CALL :fn_AppendVariable NSDKLIBPATH \lib\x86
+ ) ELSE IF DEFINED WindowsSdkDir (
+ CALL :fn_CopyVariable WindowsSdkDir NSDKLIBPATH
+ CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86
+ )
+ )
- IF ERRORLEVEL 1 (
- ECHO Failed to build "sqlite3.dll" for platform %%P.
- GOTO errors
- )
+ REM
+ REM NOTE: Unless prevented from doing so, invoke NMAKE with the MSVC
+ REM makefile to clean any stale build output from previous
+ REM iterations of this loop and/or previous runs of this batch
+ REM file, etc.
+ REM
+ IF NOT DEFINED NOCLEAN (
+ %__ECHO% nmake -f Makefile.msc clean
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to clean for platform %%P.
+ GOTO errors
+ )
+ ) ELSE (
+ REM
+ REM NOTE: Even when the cleaning step has been disabled, we still
+ REM need to remove the build output for the files we are
+ REM specifically wanting to build for each platform.
+ REM
+ %__ECHO% DEL /Q sqlite3.dll sqlite3.lib sqlite3.pdb
+ )
- REM
- REM NOTE: Copy the "sqlite3.dll" file to the platform-specific directory
- REM beneath the binary directory.
- REM
- %__ECHO% XCOPY sqlite3.dll "%BINARYDIRECTORY%\%%D\" %FFLAGS% %DFLAGS%
+ REM
+ REM NOTE: Call NMAKE with the MSVC makefile to build the "sqlite3.dll"
+ REM binary. The x86 compiler will be used to compile the native
+ REM command line tools needed during the build process itself.
+ REM Also, disable looking for and/or linking to the native Tcl
+ REM runtime library.
+ REM
+ %__ECHO% nmake -f Makefile.msc sqlite3.dll XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS%
- IF ERRORLEVEL 1 (
- ECHO Failed to copy "sqlite3.dll" to "%BINARYDIRECTORY%\%%D\".
- GOTO errors
- )
+ IF ERRORLEVEL 1 (
+ ECHO Failed to build %%B "sqlite3.dll" for platform %%P.
+ GOTO errors
+ )
- REM
- REM NOTE: Copy the "sqlite3.lib" file to the platform-specific directory
- REM beneath the binary directory.
- REM
- %__ECHO% XCOPY sqlite3.lib "%BINARYDIRECTORY%\%%D\" %FFLAGS% %DFLAGS%
+ REM
+ REM NOTE: Copy the "sqlite3.dll" file to the appropriate directory for
+ REM the build and platform beneath the binary directory.
+ REM
+ %__ECHO% XCOPY sqlite3.dll "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
- IF ERRORLEVEL 1 (
- ECHO Failed to copy "sqlite3.lib" to "%BINARYDIRECTORY%\%%D\".
- GOTO errors
- )
+ IF ERRORLEVEL 1 (
+ ECHO Failed to copy "sqlite3.dll" to "%BINARYDIRECTORY%\%%B\%%D\".
+ GOTO errors
+ )
- REM
- REM NOTE: Copy the "sqlite3.pdb" file to the platform-specific directory
- REM beneath the binary directory unless we are prevented from doing
- REM so.
- REM
- IF NOT DEFINED NOSYMBOLS (
- %__ECHO% XCOPY sqlite3.pdb "%BINARYDIRECTORY%\%%D\" %FFLAGS% %DFLAGS%
+ REM
+ REM NOTE: Copy the "sqlite3.lib" file to the appropriate directory for
+ REM the build and platform beneath the binary directory.
+ REM
+ %__ECHO% XCOPY sqlite3.lib "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
IF ERRORLEVEL 1 (
- ECHO Failed to copy "sqlite3.pdb" to "%BINARYDIRECTORY%\%%D\".
+ ECHO Failed to copy "sqlite3.lib" to "%BINARYDIRECTORY%\%%B\%%D\".
GOTO errors
)
+
+ REM
+ REM NOTE: Copy the "sqlite3.pdb" file to the appropriate directory for
+ REM the build and platform beneath the binary directory unless we
+ REM are prevented from doing so.
+ REM
+ IF NOT DEFINED NOSYMBOLS (
+ %__ECHO% XCOPY sqlite3.pdb "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to copy "sqlite3.pdb" to "%BINARYDIRECTORY%\%%B\%%D\".
+ GOTO errors
+ )
+ )
)
)
)
@@ -339,7 +461,7 @@ GOTO no_errors VERIFY MAYBE 2> NUL
GOTO :EOF
-:fn_SetVariable
+:fn_CopyVariable
SETLOCAL
IF NOT DEFINED %1 GOTO :EOF
IF "%2" == "" GOTO :EOF
diff --git a/tool/build-shell.sh b/tool/build-shell.sh index 8e62a71..6a48299 100644 --- a/tool/build-shell.sh +++ b/tool/build-shell.sh @@ -17,5 +17,6 @@ gcc -o sqlite3 -g -Os -I. \ -DSQLITE_ENABLE_RTREE \ -DHAVE_READLINE \ -DHAVE_USLEEP=1 \ - ../sqlite/src/shell.c ../sqlite/src/test_vfstrace.c \ + ../sqlite/src/shell.c \ + ../sqlite/src/test_vfstrace.c \ sqlite3.c -ldl -lreadline -lncurses diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index deb20a9..07c01ef 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -23,7 +23,7 @@ # # Begin by reading the "sqlite3.h" header file. Extract the version number -# from in this file. The versioon number is needed to generate the header +# from in this file. The version number is needed to generate the header # comment of the amalgamation. # if {[lsearch $argv --nostatic]>=0} { @@ -92,6 +92,7 @@ if {$addstatic} { # foreach hdr { crypto.h + sqlcipher.h btree.h btreeInt.h fts3.h @@ -221,11 +222,15 @@ proc copy_file {filename} { # used subroutines first in order to help the compiler find # inlining opportunities. # + foreach file { sqliteInt.h crypto.c crypto_impl.c + crypto_libtomcrypt.c + crypto_openssl.c + crypto_cc.c global.c ctime.c @@ -315,6 +320,7 @@ foreach file { fts3_porter.c fts3_tokenizer.c fts3_tokenizer1.c + fts3_tokenize_vtab.c fts3_write.c fts3_snippet.c fts3_unicode.c diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index f68f61a..a89b9f9 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -68,9 +68,14 @@ set declpattern {^ *[a-zA-Z][a-zA-Z_0-9 ]+ \**sqlite3_[_a-zA-Z0-9]+\(} # Force the output to use unix line endings, even on Windows. fconfigure stdout -translation lf -# Process the src/sqlite.h.in ext/rtree/sqlite3rtree.h files. +set filelist [subst { + $TOP/src/sqlite.h.in + $TOP/ext/rtree/sqlite3rtree.h +}] + +# Process the source files. # -foreach file [list $TOP/src/sqlite.h.in $TOP/ext/rtree/sqlite3rtree.h] { +foreach file $filelist { set in [open $file] while {![eof $in]} { diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index a751778..e9f1f81 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -2,7 +2,95 @@ # # This script is used to generate a VSIX (Visual Studio Extension) file for # SQLite usable by Visual Studio. - +# +# PREREQUISITES +# +# 1. Tcl 8.4 and later are supported, earlier versions have not been tested. +# +# 2. The "sqlite3.h" file is assumed to exist in the parent directory of the +# directory containing this script. The [optional] second command line +# argument to this script may be used to specify an alternate location. +# This script also assumes that the "sqlite3.h" file corresponds with the +# version of the binaries to be packaged. This assumption is not verified +# by this script. +# +# 3. The temporary directory specified in the TEMP or TMP environment variables +# must refer to an existing directory writable by the current user. +# +# 4. The "zip" and "unzip" command line tools must be located either in a +# directory contained in the PATH environment variable or specified as the +# exact file names to execute in the "ZipTool" and "UnZipTool" environment +# variables, respectively. +# +# 5. The template VSIX file (which is basically a zip file) must be located in +# a "win" directory inside the directory containing this script. It should +# not contain any executable binaries. It should only contain dynamic +# textual content files to be processed using [subst] and/or static content +# files to be copied verbatim. +# +# 6. The executable and other compiled binary files to be packaged into the +# final VSIX file (e.g. DLLs, LIBs, and PDBs) must be located in a single +# directory tree. The top-level directory of the tree must be specified as +# the first command line argument to this script. The second level +# sub-directory names must match those of the build configuration (e.g. +# "Debug" or "Retail"). The third level sub-directory names must match +# those of the platform (e.g. "x86", "x64", and "ARM"). For example, the +# binary files to be packaged would need to be organized as follows when +# packaging the "Debug" and "Retail" build configurations for the "x86" and +# "x64" platforms (in this example, "C:\temp" is the top-level directory as +# specified in the first command line argument): +# +# C:\Temp\Debug\x86\sqlite3.lib +# C:\Temp\Debug\x86\sqlite3.dll +# C:\Temp\Debug\x86\sqlite3.pdb +# C:\Temp\Debug\x64\sqlite3.lib +# C:\Temp\Debug\x64\sqlite3.dll +# C:\Temp\Debug\x64\sqlite3.pdb +# C:\Temp\Retail\x86\sqlite3.lib +# C:\Temp\Retail\x86\sqlite3.dll +# C:\Temp\Retail\x86\sqlite3.pdb +# C:\Temp\Retail\x64\sqlite3.lib +# C:\Temp\Retail\x64\sqlite3.dll +# C:\Temp\Retail\x64\sqlite3.pdb +# +# The above directory tree organization is performed automatically if the +# "tool\build-all-msvc.bat" batch script is used to build the binary files +# to be packaged. +# +# USAGE +# +# The first argument to this script is required and must be the name of the +# top-level directory containing the directories and files organized into a +# tree as described in item 6 of the PREREQUISITES section, above. The second +# argument is optional and if present must contain the name of the directory +# containing the root of the source tree for SQLite. The third argument is +# optional and if present must contain the flavor the VSIX package to build. +# Currently, the only supported package flavors are "WinRT" and "WP80". The +# fourth argument is optional and if present must be a string containing a list +# of platforms to include in the VSIX package. The format of the platform list +# string is "platform1,platform2,platform3". Typically, when on Windows, this +# script is executed using commands similar to the following from a normal +# Windows command prompt: +# +# CD /D C:\dev\sqlite\core +# tclsh85 tool\mkvsix.tcl C:\Temp +# +# In the example above, "C:\dev\sqlite\core" represents the root of the source +# tree for SQLite and "C:\Temp" represents the top-level directory containing +# the executable and other compiled binary files, organized into a directory +# tree as described in item 6 of the PREREQUISITES section, above. +# +# This script should work on non-Windows platforms as well, provided that all +# the requirements listed in the PREREQUISITES section are met. +# +# NOTES +# +# The temporary directory is used as a staging area for the final VSIX file. +# The template VSIX file is extracted, its contents processed, and then the +# resulting files are packaged into the final VSIX file. +# +package require Tcl 8.4 + proc fail { {error ""} {usage false} } { if {[string length $error] > 0} then { puts stdout $error @@ -11,7 +99,8 @@ proc fail { {error ""} {usage false} } { puts stdout "usage:\ [file tail [info nameofexecutable]]\ -[file tail [info script]] <binaryDirectory> \[sourceDirectory\]" +[file tail [info script]] <binaryDirectory> \[sourceDirectory\]\ +\[packageFlavor\] \[platformNames\]" exit 1 } @@ -84,20 +173,24 @@ proc writeFile { fileName data } { proc substFile { fileName } { # # NOTE: Performs all Tcl command, variable, and backslash substitutions in - # the specified file and then re-writes the contents of that same file + # the specified file and then rewrites the contents of that same file # with the substituted data. # return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] } -proc replacePlatform { fileName platformName } { +proc replaceFileNameTokens { fileName name buildName platformName } { # # NOTE: Returns the specified file name containing the platform name instead # of platform placeholder tokens. # - return [string map [list <platform> $platformName] $fileName] + return [string map [list <build> $buildName <platform> $platformName \ + <name> $name] $fileName] } +# +# NOTE: This is the entry point for this script. +# set script [file normalize [info script]] if {[string length $script] == 0} then { @@ -113,7 +206,7 @@ set rootName [file rootname [file tail $script]] # NOTE: Process and verify all the command line arguments. # set argc [llength $argv] -if {$argc != 1 && $argc != 2} then {fail} +if {$argc < 1 || $argc > 4} then {fail} set binaryDirectory [lindex $argv 0] @@ -126,7 +219,7 @@ if {![file exists $binaryDirectory] || \ fail "binary directory does not exist" } -if {$argc == 2} then { +if {$argc >= 2} then { set sourceDirectory [lindex $argv 1] } else { # @@ -145,6 +238,47 @@ if {![file exists $sourceDirectory] || \ fail "source directory does not exist" } +if {$argc >= 3} then { + set packageFlavor [lindex $argv 2] +} else { + # + # NOTE: Assume the package flavor is WinRT. + # + set packageFlavor WinRT +} + +if {[string length $packageFlavor] == 0} then { + fail "invalid package flavor" +} + +if {[string equal -nocase $packageFlavor WinRT]} then { + set shortName SQLite.WinRT + set displayName "SQLite for Windows Runtime" + set targetPlatformIdentifier Windows + set extraSdkPath "" + set extraFileListAttributes [appendArgs \ + "\r\n " {AppliesTo="WindowsAppContainer"} \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] +} elseif {[string equal -nocase $packageFlavor WP80]} then { + set shortName SQLite.WP80 + set displayName "SQLite for Windows Phone" + set targetPlatformIdentifier "Windows Phone" + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes "" +} else { + fail "unsupported package flavor, must be \"WinRT\" or \"WP80\"" +} + +if {$argc >= 4} then { + set platformNames [list] + + foreach platformName [split [lindex $argv 3] ", "] { + if {[string length $platformName] > 0} then { + lappend platformNames $platformName + } + } +} + ############################################################################### # @@ -168,7 +302,8 @@ if {![file exists $templateFile] || \ } set currentDirectory [pwd] -set outputFile [file join $currentDirectory sqlite-output.vsix] +set outputFile [file join $currentDirectory [appendArgs sqlite- \ + $packageFlavor -output.vsix]] if {[file exists $outputFile]} then { fail [appendArgs "output file \"" $outputFile "\" already exists"] @@ -244,60 +379,92 @@ if {![regexp -line -- $pattern $data dummy version]} then { ############################################################################### # -# NOTE: Setup the master file list data, including the necessary flags. +# NOTE: Setup all the master file list data. This includes the source file +# names, the destination file names, and the file processing flags. The +# possible file processing flags are: +# +# "buildNeutral" -- This flag indicates the file location and content do +# not depend on the build configuration. +# +# "platformNeutral" -- This flag indicates the file location and content +# do not depend on the build platform. +# +# "subst" -- This flag indicates that the file contains dynamic textual +# content that needs to be processed using [subst] prior to +# packaging the file into the final VSIX package. The primary +# use of this flag is to insert the name of the VSIX package, +# some package flavor-specific value, or the SQLite version +# into a file. +# +# "noDebug" -- This flag indicates that the file should be skipped when +# processing the debug build. +# +# "noRetail" -- This flag indicates that the file should be skipped when +# processing the retail build. +# +# "move" -- This flag indicates that the file should be moved from the +# source to the destination instead of being copied. +# +# This file metadata may be overridden, either in whole or in part, via +# the user-specific customizations file. # if {![info exists fileNames(source)]} then { - set fileNames(source) [list "" "" "" \ - [file join $sourceDirectory sqlite3.h] \ - [file join $binaryDirectory <platform> sqlite3.lib] \ - [file join $binaryDirectory <platform> sqlite3.dll]] + set fileNames(source) [list "" "" \ + [file join $stagingDirectory DesignTime <build> <platform> sqlite3.props] \ + [file join $sourceDirectory sqlite3.h] \ + [file join $binaryDirectory <build> <platform> sqlite3.lib] \ + [file join $binaryDirectory <build> <platform> sqlite3.dll]] if {![info exists no(symbols)]} then { lappend fileNames(source) \ - [file join $binaryDirectory <platform> sqlite3.pdb] + [file join $binaryDirectory <build> <platform> sqlite3.pdb] } } if {![info exists fileNames(destination)]} then { set fileNames(destination) [list \ - [file join $stagingDirectory extension.vsixmanifest] \ - [file join $stagingDirectory SDKManifest.xml] \ - [file join $stagingDirectory DesignTime CommonConfiguration \ - <platform> SQLite.WinRT.props] \ - [file join $stagingDirectory DesignTime CommonConfiguration \ - <platform> sqlite3.h] \ - [file join $stagingDirectory DesignTime CommonConfiguration \ - <platform> sqlite3.lib] \ - [file join $stagingDirectory Redist CommonConfiguration \ - <platform> sqlite3.dll]] + [file join $stagingDirectory extension.vsixmanifest] \ + [file join $stagingDirectory SDKManifest.xml] \ + [file join $stagingDirectory DesignTime <build> <platform> <name>.props] \ + [file join $stagingDirectory DesignTime <build> <platform> sqlite3.h] \ + [file join $stagingDirectory DesignTime <build> <platform> sqlite3.lib] \ + [file join $stagingDirectory Redist <build> <platform> sqlite3.dll]] if {![info exists no(symbols)]} then { lappend fileNames(destination) \ - [file join $stagingDirectory Redist Debug \ - <platform> sqlite3.pdb] + [file join $stagingDirectory Redist <build> <platform> sqlite3.pdb] } } -if {![info exists fileNames(neutral)]} then { - set fileNames(neutral) [list 1 1 1 1 0 0] +if {![info exists fileNames(flags)]} then { + set fileNames(flags) [list \ + [list buildNeutral platformNeutral subst] \ + [list buildNeutral platformNeutral subst] \ + [list buildNeutral platformNeutral subst move] \ + [list buildNeutral platformNeutral] \ + [list] [list] [list noRetail]] if {![info exists no(symbols)]} then { - lappend fileNames(neutral) 0 + lappend fileNames(flags) [list noRetail] } } -if {![info exists fileNames(subst)]} then { - set fileNames(subst) [list 1 1 1 0 0 0] +############################################################################### - if {![info exists no(symbols)]} then { - lappend fileNames(subst) 0 - } +# +# NOTE: Setup the list of builds supported by this script. These may be +# overridden via the user-specific customizations file. +# +if {![info exists buildNames]} then { + set buildNames [list Debug Retail] } ############################################################################### # -# NOTE: Setup the list of platforms supported by this script. +# NOTE: Setup the list of platforms supported by this script. These may be +# overridden via the command line or the user-specific customizations +# file. # if {![info exists platformNames]} then { set platformNames [list x86 x64 ARM] @@ -311,72 +478,117 @@ if {![info exists platformNames]} then { file mkdir $stagingDirectory # -# NOTE: Build the Tcl command used to extract the template package to the -# staging directory. +# NOTE: Build the Tcl command used to extract the template VSIX package to +# the staging directory. # set extractCommand [list exec -- $unzip $templateFile -d $stagingDirectory] # -# NOTE: Extract the template package to the staging directory. +# NOTE: Extract the template VSIX package to the staging directory. # eval $extractCommand ############################################################################### # -# NOTE: Process each file in the master file list. There are actually four -# parallel lists that contain the source file names, destination file -# names, the platform-neutral flags, and the use-subst flags. When the -# platform-neutral flag is non-zero, the file is not platform-specific. -# When the use-subst flag is non-zero, the file is considered to be a -# text file that may contain Tcl variable and/or command replacements, -# to be dynamically replaced during processing. If the source file name -# is an empty string, then the destination file name will be assumed to -# already exist in the staging directory and will not be copied; however, -# dynamic replacements may still be performed on the destination file -# prior to the package being re-zipped. -# -foreach sourceFileName $fileNames(source) \ - destinationFileName $fileNames(destination) \ - isNeutral $fileNames(neutral) useSubst $fileNames(subst) { +# NOTE: Process each file in the master file list. There are actually three +# parallel lists that contain the source file names, the destination file +# names, and the file processing flags. If the "buildNeutral" flag is +# present, the file location and content do not depend on the build +# configuration and "CommonConfiguration" will be used in place of the +# build configuration name. If the "platformNeutral" flag is present, +# the file location and content do not depend on the build platform and +# "neutral" will be used in place of the build platform name. If the +# "subst" flag is present, the file is assumed to be a text file that may +# contain Tcl variable, command, and backslash replacements, to be +# dynamically replaced during processing using the Tcl [subst] command. +# If the "noDebug" flag is present, the file will be skipped when +# processing for the debug build. If the "noRetail" flag is present, the +# file will be skipped when processing for the retail build. If the +# "move" flag is present, the source file will be deleted after it is +# copied to the destination file. If the source file name is an empty +# string, the destination file name will be assumed to already exist in +# the staging directory and will not be copied; however, Tcl variable, +# command, and backslash replacements may still be performed on the +# destination file prior to the final VSIX package being built if the +# "subst" flag is present. +# +foreach sourceFileName $fileNames(source) \ + destinationFileName $fileNames(destination) \ + fileFlags $fileNames(flags) { + # + # NOTE: Process the file flags into separate boolean variables that may be + # used within the loop. + # + set isBuildNeutral [expr {[lsearch $fileFlags buildNeutral] != -1}] + set isPlatformNeutral [expr {[lsearch $fileFlags platformNeutral] != -1}] + set isMove [expr {[lsearch $fileFlags move] != -1}] + set useSubst [expr {[lsearch $fileFlags subst] != -1}] + # - # NOTE: If the current file is platform-neutral, then only one platform will - # be processed for it, namely "neutral"; otherwise, each supported - # platform will be processed for it individually. + # NOTE: If the current file is build-neutral, then only one build will + # be processed for it, namely "CommonConfiguration"; otherwise, each + # supported build will be processed for it individually. # - foreach platformName [expr {$isNeutral ? [list neutral] : $platformNames}] { + foreach buildName \ + [expr {$isBuildNeutral ? [list CommonConfiguration] : $buildNames}] { # - # NOTE: Use the actual platform name in the destination file name. + # NOTE: Should the current file be skipped for this build? # - set newDestinationFileName [replacePlatform $destinationFileName \ - $platformName] + if {[lsearch $fileFlags no${buildName}] != -1} then { + continue + } # - # NOTE: Does the source file need to be copied to the destination file? + # NOTE: If the current file is platform-neutral, then only one platform + # will be processed for it, namely "neutral"; otherwise, each + # supported platform will be processed for it individually. # - if {[string length $sourceFileName] > 0} then { + foreach platformName \ + [expr {$isPlatformNeutral ? [list neutral] : $platformNames}] { # - # NOTE: First, make sure the destination directory exists. + # NOTE: Use the actual platform name in the destination file name. # - file mkdir [file dirname $newDestinationFileName] + set newDestinationFileName [replaceFileNameTokens $destinationFileName \ + $shortName $buildName $platformName] # - # NOTE: Then, copy the source file to the destination file verbatim. + # NOTE: Does the source file need to be copied to the destination file? # - file copy [replacePlatform $sourceFileName $platformName] \ - $newDestinationFileName - } + if {[string length $sourceFileName] > 0} then { + # + # NOTE: First, make sure the destination directory exists. + # + file mkdir [file dirname $newDestinationFileName] + + # + # NOTE: Then, copy the source file to the destination file verbatim. + # + set newSourceFileName [replaceFileNameTokens $sourceFileName \ + $shortName $buildName $platformName] + + file copy $newSourceFileName $newDestinationFileName + + # + # NOTE: If this is a move instead of a copy, delete the source file + # now. + # + if {$isMove} then { + file delete $newSourceFileName + } + } - # - # NOTE: Does the destination file contain dynamic replacements that must - # be processed now? - # - if {$useSubst} then { # - # NOTE: Perform any dynamic replacements contained in the destination - # file and then re-write it in-place. + # NOTE: Does the destination file contain dynamic replacements that must + # be processed now? # - substFile $newDestinationFileName + if {$useSubst} then { + # + # NOTE: Perform any dynamic replacements contained in the destination + # file and then re-write it in-place. + # + substFile $newDestinationFileName + } } } } @@ -391,13 +603,13 @@ foreach sourceFileName $fileNames(source) \ cd $stagingDirectory # -# NOTE: Build the Tcl command used to archive the final package in the +# NOTE: Build the Tcl command used to archive the final VSIX package in the # output directory. # set archiveCommand [list exec -- $zip -r $outputFile *] # -# NOTE: Build the final package archive in the output directory. +# NOTE: Build the final VSIX package archive in the output directory. # eval $archiveCommand diff --git a/tool/showdb.c b/tool/showdb.c index d378d05..27424e0 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -6,7 +6,11 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> + +#if !defined(_MSC_VER) #include <unistd.h> +#endif + #include <stdlib.h> #include <string.h> #include "sqlite3.h" @@ -172,7 +176,7 @@ static void print_db_header(void){ print_decode_line(aData, 56, 4, "Text encoding"); print_decode_line(aData, 60, 4, "User version"); print_decode_line(aData, 64, 4, "Incremental-vacuum mode"); - print_decode_line(aData, 68, 4, "meta[7]"); + print_decode_line(aData, 68, 4, "Application ID"); print_decode_line(aData, 72, 4, "meta[8]"); print_decode_line(aData, 76, 4, "meta[9]"); print_decode_line(aData, 80, 4, "meta[10]"); @@ -467,7 +471,7 @@ static void page_usage_msg(int pgno, const char *zFormat, ...){ zMsg = sqlite3_vmprintf(zFormat, ap); va_end(ap); if( pgno<=0 || pgno>mxPage ){ - printf("ERROR: page %d out of bounds. Range=1..%d. Msg: %s\n", + printf("ERROR: page %d out of range 1..%d: %s\n", pgno, mxPage, zMsg); sqlite3_free(zMsg); return; @@ -475,7 +479,7 @@ static void page_usage_msg(int pgno, const char *zFormat, ...){ if( zPageUse[pgno]!=0 ){ printf("ERROR: page %d used multiple times:\n", pgno); printf("ERROR: previous: %s\n", zPageUse[pgno]); - printf("ERROR: current: %s\n", zPageUse[pgno]); + printf("ERROR: current: %s\n", zMsg); sqlite3_free(zPageUse[pgno]); } zPageUse[pgno] = zMsg; @@ -612,14 +616,31 @@ static void page_usage_freelist(int pgno){ } /* +** Determine pages used as PTRMAP pages +*/ +static void page_usage_ptrmap(unsigned char *a){ + if( a[55] ){ + int usable = pagesize - a[20]; + int pgno = 2; + int perPage = usable/5; + while( pgno<=mxPage ){ + page_usage_msg(pgno, "PTRMAP page covering %d..%d", + pgno+1, pgno+perPage); + pgno += perPage + 1; + } + } +} + +/* ** Try to figure out how every page in the database file is being used. */ static void page_usage_report(const char *zDbName){ - int i; + int i, j; int rc; sqlite3 *db; sqlite3_stmt *pStmt; unsigned char *a; + char zQuery[200]; /* Avoid the pathological case */ if( mxPage<1 ){ @@ -644,20 +665,26 @@ static void page_usage_report(const char *zDbName){ /* Discover the usage of each page */ a = getContent(0, 100); page_usage_freelist(decodeInt32(a+32)); + page_usage_ptrmap(a); free(a); page_usage_btree(1, 0, 0, "sqlite_master"); - rc = sqlite3_prepare_v2(db, - "SELECT type, name, rootpage FROM SQLITE_MASTER WHERE rootpage", - -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - int pgno = sqlite3_column_int(pStmt, 2); - page_usage_btree(pgno, 0, 0, sqlite3_column_text(pStmt, 1)); + sqlite3_exec(db, "PRAGMA writable_schema=ON", 0, 0, 0); + for(j=0; j<2; j++){ + sqlite3_snprintf(sizeof(zQuery), zQuery, + "SELECT type, name, rootpage FROM SQLITE_MASTER WHERE rootpage" + " ORDER BY rowid %s", j?"DESC":""); + rc = sqlite3_prepare_v2(db, zQuery, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int pgno = sqlite3_column_int(pStmt, 2); + page_usage_btree(pgno, 0, 0, sqlite3_column_text(pStmt, 1)); + } + }else{ + printf("ERROR: cannot query database: %s\n", sqlite3_errmsg(db)); } - }else{ - printf("ERROR: cannot query database: %s\n", sqlite3_errmsg(db)); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) break; } - sqlite3_finalize(pStmt); sqlite3_close(db); /* Print the report and free memory used */ @@ -670,6 +697,53 @@ static void page_usage_report(const char *zDbName){ } /* +** Try to figure out how every page in the database file is being used. +*/ +static void ptrmap_coverage_report(const char *zDbName){ + unsigned int pgno; + unsigned char *aHdr; + unsigned char *a; + int usable; + int perPage; + unsigned int i; + + /* Avoid the pathological case */ + if( mxPage<1 ){ + printf("empty database\n"); + return; + } + + /* Make sure PTRMAPs are used in this database */ + aHdr = getContent(0, 100); + if( aHdr[55]==0 ){ + printf("database does not use PTRMAP pages\n"); + return; + } + usable = pagesize - aHdr[20]; + perPage = usable/5; + free(aHdr); + printf("%5d: root of sqlite_master\n", 1); + for(pgno=2; pgno<=mxPage; pgno += perPage+1){ + printf("%5d: PTRMAP page covering %d..%d\n", pgno, + pgno+1, pgno+perPage); + a = getContent((pgno-1)*pagesize, usable); + for(i=0; i+5<=usable && pgno+1+i/5<=mxPage; i+=5){ + const char *zType = "???"; + unsigned int iFrom = decodeInt32(&a[i+1]); + switch( a[i] ){ + case 1: zType = "b-tree root page"; break; + case 2: zType = "freelist page"; break; + case 3: zType = "first page of overflow"; break; + case 4: zType = "later page of overflow"; break; + case 5: zType = "b-tree non-root page"; break; + } + printf("%5d: %s, parent=%u\n", pgno+1+i/5, zType, iFrom); + } + free(a); + } +} + +/* ** Print a usage comment */ static void usage(const char *argv0){ @@ -678,6 +752,7 @@ static void usage(const char *argv0){ "args:\n" " dbheader Show database header\n" " pgidx Index of how each page is used\n" + " ptrmap Show all PTRMAP page content\n" " NNN..MMM Show hex of pages NNN through MMM\n" " NNN..end Show hex of pages NNN through end of file\n" " NNNb Decode btree page NNN\n" @@ -727,6 +802,14 @@ int main(int argc, char **argv){ page_usage_report(argv[1]); continue; } + if( strcmp(argv[i], "ptrmap")==0 ){ + ptrmap_coverage_report(argv[1]); + continue; + } + if( strcmp(argv[i], "help")==0 ){ + usage(argv[0]); + continue; + } if( !isdigit(argv[i][0]) ){ fprintf(stderr, "%s: unknown option: [%s]\n", argv[0], argv[i]); continue; diff --git a/tool/showwal.c b/tool/showwal.c index ae25a59..2888c10 100644 --- a/tool/showwal.c +++ b/tool/showwal.c @@ -18,6 +18,65 @@ static int perLine = 16; /* HEX elements to print per line */ typedef long long int i64; /* Datatype for 64-bit integers */ +/* Information for computing the checksum */ +typedef struct Cksum Cksum; +struct Cksum { + int bSwap; /* True to do byte swapping on 32-bit words */ + unsigned s0, s1; /* Current checksum value */ +}; + +/* +** extract a 32-bit big-endian integer +*/ +static unsigned int getInt32(const unsigned char *a){ + unsigned int x = (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; + return x; +} + +/* +** Swap bytes on a 32-bit unsigned integer +*/ +static unsigned int swab32(unsigned int x){ + return (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24); +} + +/* Extend the checksum. Reinitialize the checksum if bInit is true. +*/ +static void extendCksum( + Cksum *pCksum, + unsigned char *aData, + unsigned int nByte, + int bInit +){ + unsigned int *a32; + if( bInit ){ + int a = 0; + *((char*)&a) = 1; + if( a==1 ){ + /* Host is little-endian */ + pCksum->bSwap = getInt32(aData)!=0x377f0682; + }else{ + /* Host is big-endian */ + pCksum->bSwap = getInt32(aData)!=0x377f0683; + } + pCksum->s0 = 0; + pCksum->s1 = 0; + } + a32 = (unsigned int*)aData; + while( nByte>0 ){ + unsigned int x0 = a32[0]; + unsigned int x1 = a32[1]; + if( pCksum->bSwap ){ + x0 = swab32(x0); + x1 = swab32(x1); + } + pCksum->s0 += x0 + pCksum->s1; + pCksum->s1 += x1 + pCksum->s0; + nByte -= 8; + a32 += 2; + } +} /* ** Convert the var-int format into i64. Return the number of bytes @@ -152,39 +211,46 @@ static void print_frame(int iFrame){ } /* -** extract a 32-bit big-endian integer -*/ -static unsigned int getInt32(const unsigned char *a){ - unsigned int x = (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; - return x; -} - -/* -** Print an entire page of content as hex +** Summarize a single frame on a single line. */ -static void print_oneline_frame(int iFrame){ +static void print_oneline_frame(int iFrame, Cksum *pCksum){ int iStart; unsigned char *aData; + unsigned int s0, s1; iStart = 32 + (iFrame-1)*(pagesize+24); aData = getContent(iStart, 24); - fprintf(stdout, "Frame %4d: %6d %6d 0x%08x 0x%08x 0x%08x 0x%08x\n", + extendCksum(pCksum, aData, 8, 0); + extendCksum(pCksum, getContent(iStart+24, pagesize), pagesize, 0); + s0 = getInt32(aData+16); + s1 = getInt32(aData+20); + fprintf(stdout, "Frame %4d: %6d %6d 0x%08x,%08x 0x%08x,%08x %s\n", iFrame, getInt32(aData), getInt32(aData+4), getInt32(aData+8), getInt32(aData+12), - getInt32(aData+16), - getInt32(aData+20) + s0, + s1, + (s0==pCksum->s0 && s1==pCksum->s1) ? "" : "cksum-fail" ); + + /* Reset the checksum so that a single frame checksum failure will not + ** cause all subsequent frames to also show a failure. */ + pCksum->s0 = s0; + pCksum->s1 = s1; free(aData); } /* ** Decode the WAL header. */ -static void print_wal_header(void){ +static void print_wal_header(Cksum *pCksum){ unsigned char *aData; aData = getContent(0, 32); + if( pCksum ){ + extendCksum(pCksum, aData, 24, 1); + printf("Checksum byte order: %s\n", pCksum->bSwap ? "swapped" : "native"); + } printf("WAL Header:\n"); print_decode_line(aData, 0, 4,1,"Magic. 0x377f0682 (le) or 0x377f0683 (be)"); print_decode_line(aData, 4, 4, 0, "File format"); @@ -194,60 +260,199 @@ static void print_wal_header(void){ print_decode_line(aData, 20,4, 1, "Salt-2"); print_decode_line(aData, 24,4, 1, "Checksum-1"); print_decode_line(aData, 28,4, 1, "Checksum-2"); + if( pCksum ){ + if( pCksum->s0!=getInt32(aData+24) ){ + printf("**** cksum-1 mismatch: 0x%08x\n", pCksum->s0); + } + if( pCksum->s1!=getInt32(aData+28) ){ + printf("**** cksum-2 mismatch: 0x%08x\n", pCksum->s1); + } + } free(aData); } +/* +** Describe cell content. +*/ +static int describeContent( + unsigned char *a, /* Cell content */ + int nLocal, /* Bytes in a[] */ + char *zDesc /* Write description here */ +){ + int nDesc = 0; + int n, i, j; + i64 x, v; + const unsigned char *pData; + const unsigned char *pLimit; + char sep = ' '; + + pLimit = &a[nLocal]; + n = decodeVarint(a, &x); + pData = &a[x]; + a += n; + i = x - n; + while( i>0 && pData<=pLimit ){ + n = decodeVarint(a, &x); + a += n; + i -= n; + nLocal -= n; + zDesc[0] = sep; + sep = ','; + nDesc++; + zDesc++; + if( x==0 ){ + sprintf(zDesc, "*"); /* NULL is a "*" */ + }else if( x>=1 && x<=6 ){ + v = (signed char)pData[0]; + pData++; + switch( x ){ + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + sprintf(zDesc, "%lld", v); + }else if( x==7 ){ + sprintf(zDesc, "real"); + pData += 8; + }else if( x==8 ){ + sprintf(zDesc, "0"); + }else if( x==9 ){ + sprintf(zDesc, "1"); + }else if( x>=12 ){ + int size = (x-12)/2; + if( (x&1)==0 ){ + sprintf(zDesc, "blob(%d)", size); + }else{ + sprintf(zDesc, "txt(%d)", size); + } + pData += size; + } + j = strlen(zDesc); + zDesc += j; + nDesc += j; + } + return nDesc; +} + +/* +** Compute the local payload size given the total payload size and +** the page size. +*/ +static int localPayload(i64 nPayload, char cType){ + int maxLocal; + int minLocal; + int surplus; + int nLocal; + if( cType==13 ){ + /* Table leaf */ + maxLocal = pagesize-35; + minLocal = (pagesize-12)*32/255-23; + }else{ + maxLocal = (pagesize-12)*64/255-23; + minLocal = (pagesize-12)*32/255-23; + } + if( nPayload>maxLocal ){ + surplus = minLocal + (nPayload-minLocal)%(pagesize-4); + if( surplus<=maxLocal ){ + nLocal = surplus; + }else{ + nLocal = minLocal; + } + }else{ + nLocal = nPayload; + } + return nLocal; +} /* ** Create a description for a single cell. +** +** The return value is the local cell size. */ -static int describeCell(unsigned char cType, unsigned char *a, char **pzDesc){ +static int describeCell( + unsigned char cType, /* Page type */ + unsigned char *a, /* Cell content */ + int showCellContent, /* Show cell content if true */ + char **pzDesc /* Store description here */ +){ int i; int nDesc = 0; int n = 0; int leftChild; i64 nPayload; i64 rowid; - static char zDesc[100]; + int nLocal; + static char zDesc[1000]; i = 0; if( cType<=5 ){ leftChild = ((a[0]*256 + a[1])*256 + a[2])*256 + a[3]; a += 4; n += 4; - sprintf(zDesc, "left-child: %d ", leftChild); + sprintf(zDesc, "lx: %d ", leftChild); nDesc = strlen(zDesc); } if( cType!=5 ){ i = decodeVarint(a, &nPayload); a += i; n += i; - sprintf(&zDesc[nDesc], "sz: %lld ", nPayload); + sprintf(&zDesc[nDesc], "n: %lld ", nPayload); nDesc += strlen(&zDesc[nDesc]); + nLocal = localPayload(nPayload, cType); + }else{ + nPayload = nLocal = 0; } if( cType==5 || cType==13 ){ i = decodeVarint(a, &rowid); a += i; n += i; - sprintf(&zDesc[nDesc], "rowid: %lld ", rowid); + sprintf(&zDesc[nDesc], "r: %lld ", rowid); nDesc += strlen(&zDesc[nDesc]); } + if( nLocal<nPayload ){ + int ovfl; + unsigned char *b = &a[nLocal]; + ovfl = ((b[0]*256 + b[1])*256 + b[2])*256 + b[3]; + sprintf(&zDesc[nDesc], "ov: %d ", ovfl); + nDesc += strlen(&zDesc[nDesc]); + n += 4; + } + if( showCellContent && cType!=5 ){ + nDesc += describeContent(a, nLocal, &zDesc[nDesc-1]); + } *pzDesc = zDesc; - return n; + return nLocal+n; } /* ** Decode a btree page */ -static void decode_btree_page(unsigned char *a, int pgno, int hdrSize){ +static void decode_btree_page( + unsigned char *a, /* Content of the btree page to be decoded */ + int pgno, /* Page number */ + int hdrSize, /* Size of the page1-header in bytes */ + const char *zArgs /* Flags to control formatting */ +){ const char *zType = "unknown"; int nCell; - int i; + int i, j; int iCellPtr; + int showCellContent = 0; + int showMap = 0; + char *zMap = 0; switch( a[0] ){ case 2: zType = "index interior node"; break; case 5: zType = "table interior node"; break; case 10: zType = "index leaf"; break; case 13: zType = "table leaf"; break; } + while( zArgs[0] ){ + switch( zArgs[0] ){ + case 'c': showCellContent = 1; break; + case 'm': showMap = 1; break; + } + zArgs++; + } printf("Decode of btree page %d:\n", pgno); print_decode_line(a, 0, 1, 0, zType); print_decode_line(a, 1, 2, 0, "Offset to first freeblock"); @@ -261,13 +466,40 @@ static void decode_btree_page(unsigned char *a, int pgno, int hdrSize){ }else{ iCellPtr = 8; } + if( nCell>0 ){ + printf(" key: lx=left-child n=payload-size r=rowid\n"); + } + if( showMap ){ + zMap = malloc(pagesize); + memset(zMap, '.', pagesize); + memset(zMap, '1', hdrSize); + memset(&zMap[hdrSize], 'H', iCellPtr); + memset(&zMap[hdrSize+iCellPtr], 'P', 2*nCell); + } for(i=0; i<nCell; i++){ int cofst = iCellPtr + i*2; char *zDesc; + int n; + cofst = a[cofst]*256 + a[cofst+1]; - describeCell(a[0], &a[cofst-hdrSize], &zDesc); + n = describeCell(a[0], &a[cofst-hdrSize], showCellContent, &zDesc); + if( showMap ){ + char zBuf[30]; + memset(&zMap[cofst], '*', n); + zMap[cofst] = '['; + zMap[cofst+n-1] = ']'; + sprintf(zBuf, "%d", i); + j = strlen(zBuf); + if( j<=n-2 ) memcpy(&zMap[cofst+1], zBuf, j); + } printf(" %03x: cell[%d] %s\n", cofst, i, zDesc); } + if( showMap ){ + for(i=0; i<pagesize; i+=64){ + printf(" %03x: %.64s\n", i, &zMap[i]); + } + free(zMap); + } } int main(int argc, char **argv){ @@ -298,15 +530,18 @@ int main(int argc, char **argv){ printf("Available pages: 1..%d\n", mxFrame); if( argc==2 ){ int i; - print_wal_header(); - for(i=1; i<=mxFrame; i++) print_oneline_frame(i); + Cksum x; + print_wal_header(&x); + for(i=1; i<=mxFrame; i++){ + print_oneline_frame(i, &x); + } }else{ int i; for(i=2; i<argc; i++){ int iStart, iEnd; char *zLeft; if( strcmp(argv[i], "header")==0 ){ - print_wal_header(); + print_wal_header(0); continue; } if( !isdigit(argv[i][0]) ){ @@ -318,11 +553,11 @@ int main(int argc, char **argv){ iEnd = mxFrame; }else if( zLeft && zLeft[0]=='.' && zLeft[1]=='.' ){ iEnd = strtol(&zLeft[2], 0, 0); -#if 0 }else if( zLeft && zLeft[0]=='b' ){ int ofst, nByte, hdrSize; unsigned char *a; if( iStart==1 ){ + hdrSize = 100; ofst = hdrSize = 100; nByte = pagesize-100; }else{ @@ -330,11 +565,11 @@ int main(int argc, char **argv){ ofst = (iStart-1)*pagesize; nByte = pagesize; } + ofst = 32 + hdrSize + (iStart-1)*(pagesize+24) + 24; a = getContent(ofst, nByte); - decode_btree_page(a, iStart, hdrSize); + decode_btree_page(a, iStart, hdrSize, zLeft+1); free(a); continue; -#endif }else{ iEnd = iStart; } diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index fd59670..6988f6e 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -30,32 +30,34 @@ foreach arg $argv { } } if {$file_to_analyze==""} usage -if {![file exists $file_to_analyze]} { - puts stderr "No such file: $file_to_analyze" +set root_filename $file_to_analyze +regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename +if {![file exists $root_filename]} { + puts stderr "No such file: $root_filename" exit 1 } -if {![file readable $file_to_analyze]} { - puts stderr "File is not readable: $file_to_analyze" +if {![file readable $root_filename]} { + puts stderr "File is not readable: $root_filename" exit 1 } -set true_file_size [file size $file_to_analyze] +set true_file_size [file size $root_filename] if {$true_file_size<512} { - puts stderr "Empty or malformed database: $file_to_analyze" + puts stderr "Empty or malformed database: $root_filename" exit 1 } # Compute the total file size assuming test_multiplexor is being used. # Assume that SQLITE_ENABLE_8_3_NAMES might be enabled # -set extension [file extension $file_to_analyze] -set pattern $file_to_analyze +set extension [file extension $root_filename] +set pattern $root_filename append pattern {[0-3][0-9][0-9]} foreach f [glob -nocomplain $pattern] { incr true_file_size [file size $f] set extension {} } if {[string length $extension]>=2 && [string length $extension]<=4} { - set pattern [file rootname $file_to_analyze] + set pattern [file rootname $root_filename] append pattern {.[0-3][0-9][0-9]} foreach f [glob -nocomplain $pattern] { incr true_file_size [file size $f] @@ -64,7 +66,10 @@ if {[string length $extension]>=2 && [string length $extension]<=4} { # Open the database # -sqlite3 db $file_to_analyze +if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} { + puts stderr "error trying to open $file_to_analyze: $msg" + exit 1 +} register_dbstat_vtab db db eval {SELECT count(*) FROM sqlite_master} @@ -484,7 +489,7 @@ set user_percent [percent $user_payload $file_bytes] # Output the summary statistics calculated above. # -puts "/** Disk-Space Utilization Report For $file_to_analyze" +puts "/** Disk-Space Utilization Report For $root_filename" catch { puts "*** As of [clock format [clock seconds] -format {%Y-%b-%d %H:%M:%S}]" } diff --git a/tool/stack_usage.tcl b/tool/stack_usage.tcl new file mode 100644 index 0000000..b3574f0 --- /dev/null +++ b/tool/stack_usage.tcl @@ -0,0 +1,98 @@ +#!/usr/bin/tclsh +# +# Parse the output of +# +# objdump -d sqlite3.o +# +# for x64 and generate a report showing: +# +# (1) Stack used by each function +# (2) Recursion paths and their aggregate stack depth +# +set getStack 0 +while {![eof stdin]} { + set line [gets stdin] + if {[regexp {^[0-9a-f]+ <([^>]+)>:\s*$} $line all procname]} { + set curfunc $procname + set root($curfunc) 1 + set calls($curfunc) {} + set calledby($curfunc) {} + set recursive($curfunc) {} + set stkdepth($curfunc) 0 + set getStack 1 + continue + } + if {[regexp {callq? +[0-9a-z]+ <([^>]+)>} $line all other]} { + set key [list $curfunc $other] + set callpair($key) 1 + unset -nocomplain root($curfunc) + continue + } + if {[regexp {sub +\$(0x[0-9a-z]+),%[er]sp} $line all xdepth]} { + if {$getStack} { + scan $xdepth %x depth + set stkdepth($curfunc) $depth + set getStack 0 + } + continue + } +} + +puts "****************** Stack Usage By Function ********************" +set sdlist {} +foreach f [array names stkdepth] { + lappend sdlist [list $stkdepth($f) $f] +} +foreach sd [lsort -integer -decr -index 0 $sdlist] { + foreach {depth fname} $sd break + puts [format {%6d %s} $depth $fname] +} + +puts "****************** Stack Usage By Recursion *******************" +foreach key [array names callpair] { + foreach {from to} $key break + lappend calls($from) $to + # lappend calledby($to) $from +} +proc all_descendents {root} { + global calls recursive + set todo($root) $root + set go 1 + while {$go} { + set go 0 + foreach f [array names todo] { + set path $todo($f) + unset todo($f) + if {![info exists calls($f)]} continue + foreach x $calls($f) { + if {$x==$root} { + lappend recursive($root) [concat $path $root] + } elseif {![info exists d($x)]} { + set go 1 + set todo($x) [concat $path $x] + set d($x) 1 + } + } + } + } + return [array names d] +} +set pathlist {} +foreach f [array names recursive] { + all_descendents $f + foreach m $recursive($f) { + set depth 0 + foreach b [lrange $m 0 end-1] { + set depth [expr {$depth+$stkdepth($b)}] + } + lappend pathlist [list $depth $m] + } +} +foreach path [lsort -integer -decr -index 0 $pathlist] { + foreach {depth m} $path break + set first [lindex $m 0] + puts [format {%6d %s %d} $depth $first $stkdepth($first)] + foreach b [lrange $m 1 end] { + puts " $b $stkdepth($b)" + } +} diff --git a/tool/vdbe-compress.tcl b/tool/vdbe-compress.tcl index 3bcff9e..95cc1eb 100644 --- a/tool/vdbe-compress.tcl +++ b/tool/vdbe-compress.tcl @@ -79,6 +79,9 @@ while {![eof stdin]} { append unionDef " $line\n" append afterUnion $line\n lappend vlist $vname + } elseif {[regexp {^#(if|endif)} $line] && [llength $vlist]>0} { + append unionDef "$line\n" + append afterUnion $line\n } else { break } diff --git a/tool/win/sqlite.vsix b/tool/win/sqlite.vsix Binary files differindex 4bdfda5..93eefac 100644 --- a/tool/win/sqlite.vsix +++ b/tool/win/sqlite.vsix |