3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
23 #include <apr_pools.h>
25 #include "svn_types.h"
26 #include "svn_error.h"
27 #include "svn_pools.h"
29 #include "svn_dirent_uri.h"
30 #include "svn_checksum.h"
32 #include "internal_statements.h"
34 #include "private/svn_sqlite.h"
35 #include "svn_private_config.h"
36 #include "private/svn_dep_compat.h"
37 #include "private/svn_atomic.h"
38 #include "private/svn_skel.h"
39 #include "private/svn_token.h"
41 #include "private/svn_io_private.h"
42 #include "private/svn_utf_private.h"
45 #ifdef SVN_UNICODE_NORMALIZATION_FIXES
46 #include "private/svn_utf_private.h"
47 #include "private/svn_string_private.h"
48 #endif /* SVN_UNICODE_NORMALIZATION_FIXES */
51 #include "private/svn_debug.h"
54 #ifdef SVN_SQLITE_INLINE
55 /* Import the sqlite3 API vtable from sqlite3wrapper.c */
56 # define SQLITE_OMIT_DEPRECATED
57 # include <sqlite3ext.h>
58 extern const sqlite3_api_routines *const svn_sqlite3__api_funcs;
59 extern int (*const svn_sqlite3__api_initialize)(void);
60 extern int (*const svn_sqlite3__api_config)(int, ...);
61 # define sqlite3_api svn_sqlite3__api_funcs
62 # define sqlite3_initialize svn_sqlite3__api_initialize
63 # define sqlite3_config svn_sqlite3__api_config
68 #if !SQLITE_VERSION_AT_LEAST(3,7,12)
69 #error SQLite is too old -- version 3.7.12 is the minimum required version
72 #ifndef SQLITE_DETERMINISTIC
73 #define SQLITE_DETERMINISTIC 0
76 #ifdef SVN_UNICODE_NORMALIZATION_FIXES
77 /* Limit the length of a GLOB or LIKE pattern. */
78 #ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
79 # define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000
81 #endif /* SVN_UNICODE_NORMALIZATION_FIXES */
84 svn_sqlite__compiled_version(void)
86 static const char sqlite_version[] = SQLITE_VERSION;
87 return sqlite_version;
91 svn_sqlite__runtime_version(void)
93 return sqlite3_libversion();
97 INTERNAL_STATEMENTS_SQL_DECLARE_STATEMENTS(internal_statements);
101 /* An sqlite query execution callback. */
103 sqlite_tracer(void *data, const char *sql)
105 /* sqlite3 *db3 = data; */
106 SVN_DBG(("sql=\"%s\"\n", sql));
110 #ifdef SQLITE3_PROFILE
111 /* An sqlite execution timing callback. */
113 sqlite_profiler(void *data, const char *sql, sqlite3_uint64 duration)
115 /* sqlite3 *db3 = data; */
116 SVN_DBG(("[%.3f] sql=\"%s\"\n", 1e-9 * duration, sql));
120 #if defined(SVN_DEBUG) && defined(SQLITE_CONFIG_LOG)
122 sqlite_error_log(void* baton, int err, const char* msg)
124 fprintf(SVN_DBG_OUTPUT, "DBG: sqlite[S%d]: %s\n", err, msg);
129 svn_sqlite__dbg_enable_errorlog(void)
131 #if defined(SVN_DEBUG) && defined(SQLITE_CONFIG_LOG)
132 sqlite3_config(SQLITE_CONFIG_LOG, sqlite_error_log, (void*)NULL /* baton */);
137 struct svn_sqlite__db_t
140 const char * const *statement_strings;
142 svn_sqlite__stmt_t **prepared_stmts;
143 apr_pool_t *state_pool;
145 #ifdef SVN_UNICODE_NORMALIZATION_FIXES
146 /* Buffers for SQLite extensoins. */
147 svn_membuf_t sqlext_buf1;
148 svn_membuf_t sqlext_buf2;
149 svn_membuf_t sqlext_buf3;
150 #endif /* SVN_UNICODE_NORMALIZATION_FIXES */
153 struct svn_sqlite__stmt_t
155 sqlite3_stmt *s3stmt;
156 svn_sqlite__db_t *db;
157 svn_boolean_t needs_reset;
160 struct svn_sqlite__context_t
162 sqlite3_context *context;
165 struct svn_sqlite__value_t
167 sqlite3_value *value;
171 /* Convert SQLite error codes to SVN. Evaluates X multiple times */
172 #define SQLITE_ERROR_CODE(x) ((x) == SQLITE_READONLY \
173 ? SVN_ERR_SQLITE_READONLY \
174 : ((x) == SQLITE_BUSY \
175 ? SVN_ERR_SQLITE_BUSY \
176 : ((x) == SQLITE_CONSTRAINT \
177 ? SVN_ERR_SQLITE_CONSTRAINT \
178 : SVN_ERR_SQLITE_ERROR)))
181 /* SQLITE->SVN quick error wrap, much like SVN_ERR. */
182 #define SQLITE_ERR(x, db) do \
184 int sqlite_err__temp = (x); \
185 if (sqlite_err__temp != SQLITE_OK) \
186 return svn_error_createf(SQLITE_ERROR_CODE(sqlite_err__temp), \
187 NULL, "sqlite[S%d]: %s", \
189 sqlite3_errmsg((db)->db3)); \
192 #define SQLITE_ERR_CLOSE(x, db, pool) do \
194 int sqlite_err__temp = (x); \
195 if (sqlite_err__temp != SQLITE_OK) \
197 const char *sqlite_err__msg \
198 = apr_pstrdup(pool, sqlite3_errmsg((db)->db3)); \
199 return svn_error_compose_create( \
200 svn_error_createf(SQLITE_ERROR_CODE(sqlite_err__temp), \
201 NULL, "sqlite[S%d]: %s", \
202 sqlite_err__temp, sqlite_err__msg), \
203 svn_sqlite__close(db)); \
207 #define SQLITE_ERR_MSG(x, msg) do \
209 int sqlite_err__temp = (x); \
210 if (sqlite_err__temp != SQLITE_OK) \
211 return svn_error_createf(SQLITE_ERROR_CODE(sqlite_err__temp), \
212 NULL, "sqlite[S%d]: %s", \
213 sqlite_err__temp, msg); \
216 #define SVN_ERR_CLOSE(x, db) do \
218 svn_error_t *svn__err = (x); \
220 return svn_error_compose_create(svn__err, svn_sqlite__close(db)); \
224 /* Time (in milliseconds) to wait for sqlite locks before giving up. */
225 #define BUSY_TIMEOUT 10000
228 /* Convenience wrapper around exec_sql2(). */
229 #define exec_sql(db, sql) exec_sql2((db), (sql), SQLITE_OK)
231 /* Run the statement SQL on DB, ignoring SQLITE_OK and IGNORED_ERR.
232 (Note: the IGNORED_ERR parameter itself is not ignored.) */
234 exec_sql2(svn_sqlite__db_t *db, const char *sql, int ignored_err)
237 int sqlite_err = sqlite3_exec(db->db3, sql, NULL, NULL, &err_msg);
239 if (sqlite_err != SQLITE_OK && sqlite_err != ignored_err)
241 svn_error_t *err = svn_error_createf(SQLITE_ERROR_CODE(sqlite_err), NULL,
243 " executing statement '%s'"),
244 sqlite_err, err_msg, sql);
245 sqlite3_free(err_msg);
254 prepare_statement(svn_sqlite__stmt_t **stmt, svn_sqlite__db_t *db,
255 const char *text, apr_pool_t *result_pool)
257 *stmt = apr_palloc(result_pool, sizeof(**stmt));
259 (*stmt)->needs_reset = FALSE;
261 SQLITE_ERR(sqlite3_prepare_v2(db->db3, text, -1, &(*stmt)->s3stmt, NULL), db);
268 svn_sqlite__exec_statements(svn_sqlite__db_t *db, int stmt_idx)
270 SVN_ERR_ASSERT(stmt_idx < db->nbr_statements);
272 return svn_error_trace(exec_sql(db, db->statement_strings[stmt_idx]));
277 svn_sqlite__get_statement(svn_sqlite__stmt_t **stmt, svn_sqlite__db_t *db,
280 SVN_ERR_ASSERT(stmt_idx < db->nbr_statements);
282 if (db->prepared_stmts[stmt_idx] == NULL)
283 SVN_ERR(prepare_statement(&db->prepared_stmts[stmt_idx], db,
284 db->statement_strings[stmt_idx],
287 *stmt = db->prepared_stmts[stmt_idx];
289 if ((*stmt)->needs_reset)
290 return svn_error_trace(svn_sqlite__reset(*stmt));
295 /* Like svn_sqlite__get_statement but gets an internal statement.
297 All internal statements that use this api are executed with step_done(),
298 so we don't need the fallback reset handling here or in the pool cleanup */
300 get_internal_statement(svn_sqlite__stmt_t **stmt, svn_sqlite__db_t *db,
303 /* The internal statements are stored after the registered statements */
304 int prep_idx = db->nbr_statements + stmt_idx;
305 SVN_ERR_ASSERT(stmt_idx < STMT_INTERNAL_LAST);
307 if (db->prepared_stmts[prep_idx] == NULL)
308 SVN_ERR(prepare_statement(&db->prepared_stmts[prep_idx], db,
309 internal_statements[stmt_idx],
312 *stmt = db->prepared_stmts[prep_idx];
319 step_with_expectation(svn_sqlite__stmt_t* stmt,
320 svn_boolean_t expecting_row)
322 svn_boolean_t got_row;
324 SVN_ERR(svn_sqlite__step(&got_row, stmt));
325 if ((got_row && !expecting_row)
327 (!got_row && expecting_row))
328 return svn_error_create(SVN_ERR_SQLITE_ERROR,
329 svn_sqlite__reset(stmt),
331 ? _("sqlite: Expected database row missing")
332 : _("sqlite: Extra database row found"));
338 svn_sqlite__step_done(svn_sqlite__stmt_t *stmt)
340 SVN_ERR(step_with_expectation(stmt, FALSE));
341 return svn_error_trace(svn_sqlite__reset(stmt));
345 svn_sqlite__step_row(svn_sqlite__stmt_t *stmt)
347 return svn_error_trace(step_with_expectation(stmt, TRUE));
352 svn_sqlite__step(svn_boolean_t *got_row, svn_sqlite__stmt_t *stmt)
354 int sqlite_result = sqlite3_step(stmt->s3stmt);
356 if (sqlite_result != SQLITE_DONE && sqlite_result != SQLITE_ROW)
358 svn_error_t *err1, *err2;
360 err1 = svn_error_createf(SQLITE_ERROR_CODE(sqlite_result), NULL,
362 sqlite_result, sqlite3_errmsg(stmt->db->db3));
363 err2 = svn_sqlite__reset(stmt);
364 return svn_error_compose_create(err1, err2);
367 *got_row = (sqlite_result == SQLITE_ROW);
368 stmt->needs_reset = TRUE;
374 svn_sqlite__insert(apr_int64_t *row_id, svn_sqlite__stmt_t *stmt)
376 svn_boolean_t got_row;
378 SVN_ERR(svn_sqlite__step(&got_row, stmt));
380 *row_id = sqlite3_last_insert_rowid(stmt->db->db3);
382 return svn_error_trace(svn_sqlite__reset(stmt));
386 svn_sqlite__update(int *affected_rows, svn_sqlite__stmt_t *stmt)
388 SVN_ERR(step_with_expectation(stmt, FALSE));
391 *affected_rows = sqlite3_changes(stmt->db->db3);
393 return svn_error_trace(svn_sqlite__reset(stmt));
398 vbindf(svn_sqlite__stmt_t *stmt, const char *fmt, va_list ap)
402 for (count = 1; *fmt; fmt++, count++)
405 apr_size_t blob_size;
406 const svn_token_map_t *map;
411 SVN_ERR(svn_sqlite__bind_text(stmt, count,
412 va_arg(ap, const char *)));
416 SVN_ERR(svn_sqlite__bind_int(stmt, count,
422 SVN_ERR(svn_sqlite__bind_int64(stmt, count,
423 va_arg(ap, apr_int64_t)));
427 blob = va_arg(ap, const void *);
428 blob_size = va_arg(ap, apr_size_t);
429 SVN_ERR(svn_sqlite__bind_blob(stmt, count, blob, blob_size));
433 SVN_ERR(svn_sqlite__bind_revnum(stmt, count,
434 va_arg(ap, svn_revnum_t)));
438 map = va_arg(ap, const svn_token_map_t *);
439 SVN_ERR(svn_sqlite__bind_token(stmt, count, map, va_arg(ap, int)));
443 /* Skip this column: no binding */
447 SVN_ERR_MALFUNCTION();
455 svn_sqlite__bindf(svn_sqlite__stmt_t *stmt, const char *fmt, ...)
461 err = vbindf(stmt, fmt, ap);
463 return svn_error_trace(err);
467 svn_sqlite__bind_int(svn_sqlite__stmt_t *stmt,
471 SQLITE_ERR(sqlite3_bind_int(stmt->s3stmt, slot, val), stmt->db);
476 svn_sqlite__bind_int64(svn_sqlite__stmt_t *stmt,
480 SQLITE_ERR(sqlite3_bind_int64(stmt->s3stmt, slot, val), stmt->db);
485 svn_sqlite__bind_text(svn_sqlite__stmt_t *stmt,
489 SQLITE_ERR(sqlite3_bind_text(stmt->s3stmt, slot, val, -1, SQLITE_TRANSIENT),
495 svn_sqlite__bind_blob(svn_sqlite__stmt_t *stmt,
500 SQLITE_ERR(sqlite3_bind_blob(stmt->s3stmt, slot, val, (int) len,
507 svn_sqlite__bind_token(svn_sqlite__stmt_t *stmt,
509 const svn_token_map_t *map,
512 const char *word = svn_token__to_word(map, value);
514 SQLITE_ERR(sqlite3_bind_text(stmt->s3stmt, slot, word, -1, SQLITE_STATIC),
520 svn_sqlite__bind_revnum(svn_sqlite__stmt_t *stmt,
524 if (SVN_IS_VALID_REVNUM(value))
525 SQLITE_ERR(sqlite3_bind_int64(stmt->s3stmt, slot,
526 (sqlite_int64)value), stmt->db);
528 SQLITE_ERR(sqlite3_bind_null(stmt->s3stmt, slot), stmt->db);
534 svn_sqlite__bind_properties(svn_sqlite__stmt_t *stmt,
536 const apr_hash_t *props,
537 apr_pool_t *scratch_pool)
540 svn_stringbuf_t *properties;
543 return svn_error_trace(svn_sqlite__bind_blob(stmt, slot, NULL, 0));
545 SVN_ERR(svn_skel__unparse_proplist(&skel, props, scratch_pool));
546 properties = svn_skel__unparse(skel, scratch_pool);
547 return svn_error_trace(svn_sqlite__bind_blob(stmt,
554 svn_sqlite__bind_iprops(svn_sqlite__stmt_t *stmt,
556 const apr_array_header_t *inherited_props,
557 apr_pool_t *scratch_pool)
560 svn_stringbuf_t *properties;
562 if (inherited_props == NULL)
563 return svn_error_trace(svn_sqlite__bind_blob(stmt, slot, NULL, 0));
565 SVN_ERR(svn_skel__unparse_iproplist(&skel, inherited_props,
566 scratch_pool, scratch_pool));
567 properties = svn_skel__unparse(skel, scratch_pool);
568 return svn_error_trace(svn_sqlite__bind_blob(stmt,
575 svn_sqlite__bind_checksum(svn_sqlite__stmt_t *stmt,
577 const svn_checksum_t *checksum,
578 apr_pool_t *scratch_pool)
580 const char *csum_str;
582 if (checksum == NULL)
585 csum_str = svn_checksum_serialize(checksum, scratch_pool, scratch_pool);
587 return svn_error_trace(svn_sqlite__bind_text(stmt, slot, csum_str));
592 svn_sqlite__column_blob(svn_sqlite__stmt_t *stmt, int column,
593 apr_size_t *len, apr_pool_t *result_pool)
595 const void *val = sqlite3_column_blob(stmt->s3stmt, column);
596 *len = sqlite3_column_bytes(stmt->s3stmt, column);
598 if (result_pool && val != NULL)
599 val = apr_pmemdup(result_pool, val, *len);
605 svn_sqlite__column_text(svn_sqlite__stmt_t *stmt, int column,
606 apr_pool_t *result_pool)
608 /* cast from 'unsigned char' to regular 'char' */
609 const char *result = (const char *)sqlite3_column_text(stmt->s3stmt, column);
611 if (result_pool && result != NULL)
612 result = apr_pstrdup(result_pool, result);
618 svn_sqlite__column_revnum(svn_sqlite__stmt_t *stmt, int column)
620 if (svn_sqlite__column_is_null(stmt, column))
621 return SVN_INVALID_REVNUM;
622 return (svn_revnum_t) sqlite3_column_int64(stmt->s3stmt, column);
626 svn_sqlite__column_boolean(svn_sqlite__stmt_t *stmt, int column)
628 return sqlite3_column_int64(stmt->s3stmt, column) != 0;
632 svn_sqlite__column_int(svn_sqlite__stmt_t *stmt, int column)
634 return sqlite3_column_int(stmt->s3stmt, column);
638 svn_sqlite__column_int64(svn_sqlite__stmt_t *stmt, int column)
640 return sqlite3_column_int64(stmt->s3stmt, column);
644 svn_sqlite__column_token(svn_sqlite__stmt_t *stmt,
646 const svn_token_map_t *map)
648 /* cast from 'unsigned char' to regular 'char' */
649 const char *word = (const char *)sqlite3_column_text(stmt->s3stmt, column);
651 return svn_token__from_word_strict(map, word);
655 svn_sqlite__column_token_null(svn_sqlite__stmt_t *stmt,
657 const svn_token_map_t *map,
660 /* cast from 'unsigned char' to regular 'char' */
661 const char *word = (const char *)sqlite3_column_text(stmt->s3stmt, column);
666 return svn_token__from_word_strict(map, word);
670 svn_sqlite__column_properties(apr_hash_t **props,
671 svn_sqlite__stmt_t *stmt,
673 apr_pool_t *result_pool,
674 apr_pool_t *scratch_pool)
679 /* svn_skel__parse_proplist copies everything needed to result_pool */
680 val = svn_sqlite__column_blob(stmt, column, &len, NULL);
687 SVN_ERR(svn_skel__parse_proplist(props,
688 svn_skel__parse(val, len, scratch_pool),
695 svn_sqlite__column_iprops(apr_array_header_t **iprops,
696 svn_sqlite__stmt_t *stmt,
698 apr_pool_t *result_pool,
699 apr_pool_t *scratch_pool)
704 /* svn_skel__parse_iprops copies everything needed to result_pool */
705 val = svn_sqlite__column_blob(stmt, column, &len, NULL);
712 SVN_ERR(svn_skel__parse_iprops(iprops,
713 svn_skel__parse(val, len, scratch_pool),
720 svn_sqlite__column_checksum(const svn_checksum_t **checksum,
721 svn_sqlite__stmt_t *stmt, int column,
722 apr_pool_t *result_pool)
724 const char *digest = svn_sqlite__column_text(stmt, column, NULL);
729 SVN_ERR(svn_checksum_deserialize(checksum, digest,
730 result_pool, result_pool));
736 svn_sqlite__column_is_null(svn_sqlite__stmt_t *stmt, int column)
738 return sqlite3_column_type(stmt->s3stmt, column) == SQLITE_NULL;
742 svn_sqlite__column_bytes(svn_sqlite__stmt_t *stmt, int column)
744 return sqlite3_column_bytes(stmt->s3stmt, column);
748 svn_sqlite__finalize(svn_sqlite__stmt_t *stmt)
750 SQLITE_ERR(sqlite3_finalize(stmt->s3stmt), stmt->db);
755 svn_sqlite__reset(svn_sqlite__stmt_t *stmt)
757 /* No need to reset again after a first attempt */
758 stmt->needs_reset = FALSE;
760 /* Clear bindings first, as there are no documented reasons
761 why this would ever fail, but keeping variable bindings
762 when reset is not what we expect. */
763 SQLITE_ERR(sqlite3_clear_bindings(stmt->s3stmt), stmt->db);
765 /* Reset last, as this *will* fail if the statement failed since
766 the last time it was reset, while reporting just the same failure.
767 (In this case the statement is also properly reset).
769 See the sqlite3_reset() documentation for more details. */
770 SQLITE_ERR(sqlite3_reset(stmt->s3stmt), stmt->db);
776 svn_sqlite__read_schema_version(int *version,
777 svn_sqlite__db_t *db,
778 apr_pool_t *scratch_pool)
780 svn_sqlite__stmt_t *stmt;
782 SVN_ERR(prepare_statement(&stmt, db, "PRAGMA user_version;", scratch_pool));
783 SVN_ERR(svn_sqlite__step_row(stmt));
785 *version = svn_sqlite__column_int(stmt, 0);
787 return svn_error_trace(svn_sqlite__finalize(stmt));
791 static volatile svn_atomic_t sqlite_init_state = 0;
793 /* If possible, verify that SQLite was compiled in a thread-safe
795 /* Don't call this function directly! Use svn_atomic__init_once(). */
797 init_sqlite(void *baton, apr_pool_t *pool)
799 if (sqlite3_libversion_number() < SVN_SQLITE_MIN_VERSION_NUMBER)
801 return svn_error_createf(
802 SVN_ERR_SQLITE_ERROR, NULL,
803 _("SQLite compiled for %s, but running with %s"),
804 SVN_SQLITE_MIN_VERSION, sqlite3_libversion());
809 /* SQLite 3.5 allows verification of its thread-safety at runtime.
810 Older versions are simply expected to have been configured with
811 --enable-threadsafe, which compiles with -DSQLITE_THREADSAFE=1
812 (or -DTHREADSAFE, for older versions). */
813 if (! sqlite3_threadsafe())
814 return svn_error_create(SVN_ERR_SQLITE_ERROR, NULL,
815 _("SQLite is required to be compiled and run in "
816 "thread-safe mode"));
818 /* If SQLite has been already initialized, sqlite3_config() returns
821 int err = sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
822 if (err != SQLITE_OK && err != SQLITE_MISUSE)
823 return svn_error_createf(SQLITE_ERROR_CODE(err), NULL,
824 _("Could not configure SQLite [S%d]"), err);
826 SQLITE_ERR_MSG(sqlite3_initialize(), _("Could not initialize SQLite"));
828 #endif /* APR_HAS_THRADS */
834 internal_open(svn_sqlite__db_t *db, const char *path, svn_sqlite__mode_t mode,
835 apr_int32_t timeout, apr_pool_t *scratch_pool)
840 if (mode == svn_sqlite__mode_readonly)
841 flags = SQLITE_OPEN_READONLY;
842 else if (mode == svn_sqlite__mode_readwrite)
843 flags = SQLITE_OPEN_READWRITE;
844 else if (mode == svn_sqlite__mode_rwcreate)
845 flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
847 SVN_ERR_MALFUNCTION();
849 /* Turn off SQLite's mutexes. All svn objects are single-threaded,
850 so we can already guarantee that our use of the SQLite handle
851 will be serialized properly.
853 Note: in 3.6.x, we've already config'd SQLite into MULTITHREAD mode,
854 so this is probably redundant, but if we are running in a process where
855 somebody initialized SQLite before us it is needed anyway. */
856 flags |= SQLITE_OPEN_NOMUTEX;
858 #if !defined(WIN32) && !defined(SVN_SQLITE_INLINE)
859 if (mode == svn_sqlite__mode_rwcreate)
861 svn_node_kind_t kind;
863 /* Create the file before SQLite to avoid any permissions
864 problems with an SQLite build that uses the default
865 SQLITE_DEFAULT_FILE_PERMISSIONS of 644 modified by umask.
866 We simply want umask permissions. */
867 SVN_ERR(svn_io_check_path(path, &kind, scratch_pool));
868 if (kind == svn_node_none)
870 /* Another thread may have created the file, that's OK. */
871 svn_error_t *err = svn_io_file_create_empty(path, scratch_pool);
872 if (err && !APR_STATUS_IS_EEXIST(err->apr_err))
873 return svn_error_trace(err);
874 svn_error_clear(err);
879 /* Open the database. Note that a handle is returned, even when an error
880 occurs (except for out-of-memory); thus, we can safely use it to
881 extract an error message and construct an svn_error_t. SQLite always
882 requires sqlite3_close() after sqlite3_open_v2() while Subversion
883 typically does not require close() after an open() that returns an
884 error. So we must ensure we close the handle if this function, or
885 the caller svn_sqlite__open, returns an error to the application. */
887 const char *vFs = NULL;
889 #if defined(WIN32) && SQLITE_VERSION_AT_LEAST(3, 8, 1)
890 if (strlen(path) > 248)
893 vFs = "win32-longpath"; /* Enable long paths in sqlite */
895 /* Long paths must be absolute */
896 if (!svn_dirent_is_absolute(path))
897 SVN_ERR(svn_dirent_get_absolute(&path, path, scratch_pool));
899 /* Convert the path to a properly canonicalized \\?\C:\long\path */
900 SVN_ERR(svn_io__utf8_to_unicode_longpath(&win_path, path,
903 /* And convert it back to UTF-8 because there is no
904 sqlite3_open16_v2() yet */
905 SVN_ERR(svn_utf__win32_utf16_to_utf8(&path, win_path, NULL,
910 /* ### SQLITE_CANTOPEN */
911 SQLITE_ERR_CLOSE(sqlite3_open_v2(path, &db->db3, flags, vFs),
917 timeout = BUSY_TIMEOUT;
919 /* Retry until timeout when database is busy. */
920 SQLITE_ERR_CLOSE(sqlite3_busy_timeout(db->db3, timeout),
927 /* APR cleanup function used to close the database when its pool is destroyed.
928 DATA should be the svn_sqlite__db_t handle for the database. */
930 close_apr(void *data)
932 svn_sqlite__db_t *db = data;
933 svn_error_t *err = SVN_NO_ERROR;
937 /* Check to see if we've already closed this database. */
941 /* Finalize any prepared statements. */
942 if (db->prepared_stmts)
944 for (i = 0; i < db->nbr_statements + STMT_INTERNAL_LAST; i++)
946 if (db->prepared_stmts[i])
948 if (i < db->nbr_statements
949 && db->prepared_stmts[i]->needs_reset)
952 const char *stmt_text = db->statement_strings[i];
953 SVN_UNUSED(stmt_text);
955 SVN_ERR_MALFUNCTION_NO_RETURN();
957 err = svn_error_compose_create(err,
958 svn_sqlite__reset(db->prepared_stmts[i]));
961 err = svn_error_compose_create(
962 svn_sqlite__finalize(db->prepared_stmts[i]), err);
967 result = sqlite3_close(db->db3);
969 /* If there's a pre-existing error, return it. */
972 result = err->apr_err;
973 svn_error_clear(err);
977 if (result != SQLITE_OK)
978 return SQLITE_ERROR_CODE(result); /* ### lossy */
985 #ifdef SVN_UNICODE_NORMALIZATION_FIXES
986 /* Unicode normalizing collation for WC paths */
988 collate_ucs_nfd(void *baton,
989 int len1, const void *key1,
990 int len2, const void *key2)
992 svn_sqlite__db_t *db = baton;
995 if (svn_utf__normcmp(key1, len1, key2, len2,
996 &db->sqlext_buf1, &db->sqlext_buf2, &result))
998 /* There is really nothing we can do here if an error occurs
999 during Unicode normalizetion, and attempting to recover could
1000 result in the wc.db index being corrupted. Presumably this
1001 can only happen if the index already contains invalid UTF-8
1002 strings, which should never happen in any case ... */
1003 SVN_ERR_MALFUNCTION_NO_RETURN();
1010 glob_like_ucs_nfd_common(sqlite3_context *context,
1011 int argc, sqlite3_value **argv,
1012 svn_boolean_t sql_like)
1014 svn_sqlite__db_t *const db = sqlite3_user_data(context);
1016 const char *const pattern = (void*)sqlite3_value_text(argv[0]);
1017 const apr_size_t pattern_len = sqlite3_value_bytes(argv[0]);
1018 const char *const string = (void*)sqlite3_value_text(argv[1]);
1019 const apr_size_t string_len = sqlite3_value_bytes(argv[1]);
1021 const char *escape = NULL;
1022 apr_size_t escape_len = 0;
1024 svn_boolean_t match;
1027 if (pattern_len > SQLITE_MAX_LIKE_PATTERN_LENGTH)
1029 sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1);
1033 if (argc == 3 && sql_like)
1035 escape = (void*)sqlite3_value_text(argv[2]);
1036 escape_len = sqlite3_value_bytes(argv[2]);
1039 if (pattern && string)
1041 err = svn_utf__glob(pattern, pattern_len, string, string_len,
1042 escape, escape_len, sql_like,
1043 &db->sqlext_buf1, &db->sqlext_buf2, &db->sqlext_buf3,
1049 svn_membuf__ensure(&db->sqlext_buf1, 512);
1050 errmsg = svn_err_best_message(err,
1051 db->sqlext_buf1.data,
1052 db->sqlext_buf1.size - 1);
1053 svn_error_clear(err);
1054 sqlite3_result_error(context, errmsg, -1);
1058 sqlite3_result_int(context, match);
1062 /* Unicode normalizing implementation of GLOB */
1064 glob_ucs_nfd(sqlite3_context *context,
1065 int argc, sqlite3_value **argv)
1067 glob_like_ucs_nfd_common(context, argc, argv, FALSE);
1070 /* Unicode normalizing implementation of LIKE */
1072 like_ucs_nfd(sqlite3_context *context,
1073 int argc, sqlite3_value **argv)
1075 glob_like_ucs_nfd_common(context, argc, argv, TRUE);
1077 #endif /* SVN_UNICODE_NORMALIZATION_FIXES */
1080 svn_sqlite__open(svn_sqlite__db_t **db, const char *path,
1081 svn_sqlite__mode_t mode, const char * const statements[],
1082 int unused1, const char * const *unused2,
1083 apr_int32_t timeout,
1084 apr_pool_t *result_pool, apr_pool_t *scratch_pool)
1086 SVN_ERR(svn_atomic__init_once(&sqlite_init_state,
1087 init_sqlite, NULL, scratch_pool));
1089 *db = apr_pcalloc(result_pool, sizeof(**db));
1091 SVN_ERR(internal_open(*db, path, mode, timeout, scratch_pool));
1093 #if SQLITE_VERSION_NUMBER >= 3008000 && SQLITE_VERSION_NUMBER < 3009000
1094 /* disable SQLITE_ENABLE_STAT3/4 from 3.8.1 - 3.8.3 (but not 3.8.3.1+)
1095 * to prevent using it when it's buggy.
1096 * See: https://www.sqlite.org/src/info/4c86b126f2 */
1097 if (sqlite3_libversion_number() > 3008000 &&
1098 sqlite3_libversion_number() < 3008004 &&
1099 strcmp(sqlite3_sourceid(),"2014-02-11")<0)
1101 sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, (*db)->db3, 0x800);
1105 #ifdef SVN_UNICODE_NORMALIZATION_FIXES
1106 /* Create extension buffers with space for 200 UCS-4 characters. */
1107 svn_membuf__create(&(*db)->sqlext_buf1, 800, result_pool);
1108 svn_membuf__create(&(*db)->sqlext_buf2, 800, result_pool);
1109 svn_membuf__create(&(*db)->sqlext_buf3, 800, result_pool);
1111 /* Register collation and LIKE and GLOB operator replacements. */
1112 SQLITE_ERR_CLOSE(sqlite3_create_collation((*db)->db3,
1113 "svn-ucs-nfd", SQLITE_UTF8,
1114 *db, collate_ucs_nfd),
1116 /* ### Is it really necessary to override these functions?
1117 I would assume the default implementation to be collation agnostic?
1118 And otherwise our implementation should be...
1120 The default implementation is in some cases index backed, while our
1121 implementation can't be. With an index based on the collation it could
1123 SQLITE_ERR_CLOSE(sqlite3_create_function((*db)->db3, "glob", 2,
1124 SQLITE_UTF8 | SQLITE_DETERMINISTIC,
1125 *db, glob_ucs_nfd, NULL, NULL),
1127 SQLITE_ERR_CLOSE(sqlite3_create_function((*db)->db3, "like", 2,
1128 SQLITE_UTF8 | SQLITE_DETERMINISTIC,
1129 *db, like_ucs_nfd, NULL, NULL),
1131 SQLITE_ERR_CLOSE(sqlite3_create_function((*db)->db3, "like", 3,
1132 SQLITE_UTF8 | SQLITE_DETERMINISTIC,
1133 *db, like_ucs_nfd, NULL, NULL),
1135 #endif /* SVN_UNICODE_NORMALIZATION_FIXES */
1137 #ifdef SQLITE3_DEBUG
1138 sqlite3_trace((*db)->db3, sqlite_tracer, (*db)->db3);
1140 #ifdef SQLITE3_PROFILE
1141 sqlite3_profile((*db)->db3, sqlite_profiler, (*db)->db3);
1144 SVN_ERR_CLOSE(exec_sql(*db,
1145 /* The default behavior of the LIKE operator is to ignore case
1146 for ASCII characters. Hence, by default 'a' LIKE 'A' is true.
1147 The case_sensitive_like pragma installs a new application-
1148 defined LIKE function that is either case sensitive or
1149 insensitive depending on the value of the case_sensitive_like
1151 "PRAGMA case_sensitive_like=1;"
1152 /* Disable synchronization to disable the explicit disk flushes
1153 that make Sqlite up to 50 times slower; especially on small
1156 This removes some stability guarantees on specific hardware
1157 and power failures, but still guarantees atomic commits on
1158 application crashes. With our dependency on external data
1159 like pristine files (Wc) and revision files (repository),
1160 we can't keep up these additional guarantees anyway.
1162 ### Maybe switch to NORMAL(1) when we use larger transaction
1164 "PRAGMA synchronous=OFF;"
1165 /* Enable recursive triggers so that a user trigger will fire
1166 in the deletion phase of an INSERT OR REPLACE statement.
1167 Requires SQLite >= 3.6.18 */
1168 "PRAGMA recursive_triggers=ON;"
1169 /* Enforce current Sqlite default behavior. Some distributions
1170 might change the Sqlite defaults without realizing how this
1171 affects application(read: Subversion) performance/behavior. */
1172 "PRAGMA foreign_keys=OFF;" /* SQLITE_DEFAULT_FOREIGN_KEYS*/
1173 "PRAGMA locking_mode = NORMAL;" /* SQLITE_DEFAULT_LOCKING_MODE */
1174 /* Testing shows TRUNCATE is faster than DELETE on Windows. */
1175 "PRAGMA journal_mode = TRUNCATE;"
1179 #if defined(SVN_DEBUG)
1180 /* When running in debug mode, enable the checking of foreign key
1181 constraints. This has possible performance implications, so we don't
1182 bother to do it for production...for now. */
1183 SVN_ERR_CLOSE(exec_sql(*db, "PRAGMA foreign_keys=ON;"),
1187 #ifdef SVN_SQLITE_REVERSE_UNORDERED_SELECTS
1188 /* When enabled, this PRAGMA causes SELECT statements without an ORDER BY
1189 clause to emit their results in the reverse order of what they normally
1190 would. This can help detecting invalid assumptions about the result
1192 SVN_ERR_CLOSE(exec_sql(*db, "PRAGMA reverse_unordered_selects=ON;"),
1196 /* Store temporary tables in RAM instead of in temporary files, but don't
1197 fail on this if this option is disabled in the sqlite compilation by
1198 setting SQLITE_TEMP_STORE to 0 (always to disk) */
1199 svn_error_clear(exec_sql(*db, "PRAGMA temp_store = MEMORY;"));
1201 /* Store the provided statements. */
1204 (*db)->statement_strings = statements;
1205 (*db)->nbr_statements = 0;
1206 while (*statements != NULL)
1209 (*db)->nbr_statements++;
1212 (*db)->prepared_stmts = apr_pcalloc(
1214 ((*db)->nbr_statements + STMT_INTERNAL_LAST)
1215 * sizeof(svn_sqlite__stmt_t *));
1219 (*db)->nbr_statements = 0;
1220 (*db)->prepared_stmts = apr_pcalloc(result_pool,
1221 (0 + STMT_INTERNAL_LAST)
1222 * sizeof(svn_sqlite__stmt_t *));
1225 (*db)->state_pool = result_pool;
1226 apr_pool_cleanup_register(result_pool, *db, close_apr, apr_pool_cleanup_null);
1228 return SVN_NO_ERROR;
1232 svn_sqlite__close(svn_sqlite__db_t *db)
1234 apr_status_t result = apr_pool_cleanup_run(db->state_pool, db, close_apr);
1236 if (result == APR_SUCCESS)
1237 return SVN_NO_ERROR;
1239 return svn_error_wrap_apr(result, NULL);
1242 static svn_error_t *
1243 reset_all_statements(svn_sqlite__db_t *db,
1244 svn_error_t *error_to_wrap)
1249 /* ### Should we reorder the errors in this specific case
1250 ### to avoid returning the normal error as top level error? */
1252 err = svn_error_compose_create(error_to_wrap,
1253 svn_error_create(SVN_ERR_SQLITE_RESETTING_FOR_ROLLBACK,
1256 for (i = 0; i < db->nbr_statements; i++)
1257 if (db->prepared_stmts[i] && db->prepared_stmts[i]->needs_reset)
1258 err = svn_error_compose_create(err,
1259 svn_sqlite__reset(db->prepared_stmts[i]));
1265 svn_sqlite__begin_transaction(svn_sqlite__db_t *db)
1267 svn_sqlite__stmt_t *stmt;
1269 SVN_ERR(get_internal_statement(&stmt, db,
1270 STMT_INTERNAL_BEGIN_TRANSACTION));
1271 SVN_ERR(svn_sqlite__step_done(stmt));
1272 return SVN_NO_ERROR;
1276 svn_sqlite__begin_immediate_transaction(svn_sqlite__db_t *db)
1278 svn_sqlite__stmt_t *stmt;
1280 SVN_ERR(get_internal_statement(&stmt, db,
1281 STMT_INTERNAL_BEGIN_IMMEDIATE_TRANSACTION));
1282 SVN_ERR(svn_sqlite__step_done(stmt));
1283 return SVN_NO_ERROR;
1287 svn_sqlite__begin_savepoint(svn_sqlite__db_t *db)
1289 svn_sqlite__stmt_t *stmt;
1291 SVN_ERR(get_internal_statement(&stmt, db,
1292 STMT_INTERNAL_SAVEPOINT_SVN));
1293 SVN_ERR(svn_sqlite__step_done(stmt));
1294 return SVN_NO_ERROR;
1298 svn_sqlite__finish_transaction(svn_sqlite__db_t *db,
1301 svn_sqlite__stmt_t *stmt;
1303 /* Commit or rollback the sqlite transaction. */
1308 err2 = get_internal_statement(&stmt, db,
1309 STMT_INTERNAL_ROLLBACK_TRANSACTION);
1311 err2 = svn_sqlite__step_done(stmt);
1313 if (err2 && err2->apr_err == SVN_ERR_SQLITE_BUSY)
1315 /* ### Houston, we have a problem!
1317 We are trying to rollback but we can't because some
1318 statements are still busy. This leaves the database
1319 unusable for future transactions as the current transaction
1322 As we are returning the actual error as the most relevant
1323 error in the chain, our caller might assume that it can
1324 retry/compensate on this error (e.g. SVN_WC_LOCKED), while
1325 in fact the SQLite database is unusable until the statements
1326 started within this transaction are reset and the transaction
1329 We try to compensate by resetting all prepared but unreset
1330 statements; but we leave the busy error in the chain anyway to
1331 help diagnosing the original error and help in finding where
1332 a reset statement is missing. */
1334 err2 = reset_all_statements(db, err2);
1335 err2 = svn_error_compose_create(
1336 svn_sqlite__step_done(stmt),
1340 return svn_error_compose_create(err,
1344 SVN_ERR(get_internal_statement(&stmt, db, STMT_INTERNAL_COMMIT_TRANSACTION));
1345 return svn_error_trace(svn_sqlite__step_done(stmt));
1349 svn_sqlite__finish_savepoint(svn_sqlite__db_t *db,
1352 svn_sqlite__stmt_t *stmt;
1358 err2 = get_internal_statement(&stmt, db,
1359 STMT_INTERNAL_ROLLBACK_TO_SAVEPOINT_SVN);
1362 err2 = svn_sqlite__step_done(stmt);
1364 if (err2 && err2->apr_err == SVN_ERR_SQLITE_BUSY)
1366 /* Ok, we have a major problem. Some statement is still open, which
1367 makes it impossible to release this savepoint.
1369 ### See huge comment in svn_sqlite__finish_transaction for
1372 err2 = reset_all_statements(db, err2);
1373 err2 = svn_error_compose_create(svn_sqlite__step_done(stmt), err2);
1376 err = svn_error_compose_create(err, err2);
1377 err2 = get_internal_statement(&stmt, db,
1378 STMT_INTERNAL_RELEASE_SAVEPOINT_SVN);
1381 err2 = svn_sqlite__step_done(stmt);
1383 return svn_error_trace(svn_error_compose_create(err, err2));
1386 SVN_ERR(get_internal_statement(&stmt, db,
1387 STMT_INTERNAL_RELEASE_SAVEPOINT_SVN));
1389 return svn_error_trace(svn_sqlite__step_done(stmt));
1393 svn_sqlite__with_transaction(svn_sqlite__db_t *db,
1394 svn_sqlite__transaction_callback_t cb_func,
1396 apr_pool_t *scratch_pool /* NULL allowed */)
1398 SVN_SQLITE__WITH_TXN(cb_func(cb_baton, db, scratch_pool), db);
1399 return SVN_NO_ERROR;
1403 svn_sqlite__with_immediate_transaction(
1404 svn_sqlite__db_t *db,
1405 svn_sqlite__transaction_callback_t cb_func,
1407 apr_pool_t *scratch_pool /* NULL allowed */)
1409 SVN_SQLITE__WITH_IMMEDIATE_TXN(cb_func(cb_baton, db, scratch_pool), db);
1410 return SVN_NO_ERROR;
1414 svn_sqlite__with_lock(svn_sqlite__db_t *db,
1415 svn_sqlite__transaction_callback_t cb_func,
1417 apr_pool_t *scratch_pool /* NULL allowed */)
1419 SVN_SQLITE__WITH_LOCK(cb_func(cb_baton, db, scratch_pool), db);
1420 return SVN_NO_ERROR;
1424 svn_sqlite__hotcopy(const char *src_path,
1425 const char *dst_path,
1426 apr_pool_t *scratch_pool)
1428 svn_sqlite__db_t *src_db;
1430 SVN_ERR(svn_sqlite__open(&src_db, src_path, svn_sqlite__mode_readonly,
1432 scratch_pool, scratch_pool));
1435 svn_sqlite__db_t *dst_db;
1436 sqlite3_backup *backup;
1439 SVN_ERR(svn_sqlite__open(&dst_db, dst_path, svn_sqlite__mode_rwcreate,
1440 NULL, 0, NULL, 0, scratch_pool, scratch_pool));
1441 backup = sqlite3_backup_init(dst_db->db3, "main", src_db->db3, "main");
1443 return svn_error_createf(SVN_ERR_SQLITE_ERROR, NULL,
1444 _("SQLite hotcopy failed for %s"), src_path);
1447 /* Pages are usually 1024 byte (SQLite docs). On my laptop
1448 copying gets faster as the number of pages is increased up
1449 to about 64, beyond that speed levels off. Lets put the
1450 number of pages an order of magnitude higher, this is still
1451 likely to be a fraction of large databases. */
1452 rc1 = sqlite3_backup_step(backup, 1024);
1454 /* Should we sleep on SQLITE_OK? That would make copying a
1455 large database take much longer. When we do sleep how,
1456 long should we sleep? Should the sleep get longer if we
1457 keep getting BUSY/LOCKED? I have no real reason for
1459 if (rc1 == SQLITE_BUSY || rc1 == SQLITE_LOCKED)
1462 while (rc1 == SQLITE_OK || rc1 == SQLITE_BUSY || rc1 == SQLITE_LOCKED);
1463 rc2 = sqlite3_backup_finish(backup);
1464 if (rc1 != SQLITE_DONE)
1465 SQLITE_ERR(rc1, dst_db);
1466 SQLITE_ERR(rc2, dst_db);
1467 SVN_ERR(svn_sqlite__close(dst_db));
1470 SVN_ERR(svn_sqlite__close(src_db));
1472 SVN_ERR(svn_io_copy_perms(src_path, dst_path, scratch_pool));
1474 return SVN_NO_ERROR;
1477 struct function_wrapper_baton_t
1479 svn_sqlite__func_t func;
1484 wrapped_func(sqlite3_context *context,
1486 sqlite3_value *values[])
1488 struct function_wrapper_baton_t *fwb = sqlite3_user_data(context);
1489 svn_sqlite__context_t sctx;
1491 void *void_values = values;
1493 sctx.context = context;
1495 err = fwb->func(&sctx, argc, void_values, fwb->baton);
1500 sqlite3_result_error(context,
1501 svn_err_best_message(err, buf, sizeof(buf)),
1503 svn_error_clear(err);
1509 svn_sqlite__create_scalar_function(svn_sqlite__db_t *db,
1510 const char *func_name,
1512 svn_boolean_t deterministic,
1513 svn_sqlite__func_t func,
1517 struct function_wrapper_baton_t *fwb = apr_pcalloc(db->state_pool,
1523 eTextRep = SQLITE_ANY;
1525 eTextRep |= SQLITE_DETERMINISTIC;
1527 SQLITE_ERR(sqlite3_create_function(db->db3, func_name, argc, eTextRep,
1528 fwb, wrapped_func, NULL, NULL),
1531 return SVN_NO_ERROR;
1535 svn_sqlite__value_type(svn_sqlite__value_t *val)
1538 return sqlite3_value_type(v);
1542 svn_sqlite__value_text(svn_sqlite__value_t *val)
1545 return (const char *) sqlite3_value_text(v);
1549 svn_sqlite__result_null(svn_sqlite__context_t *sctx)
1551 sqlite3_result_null(sctx->context);
1555 svn_sqlite__result_int64(svn_sqlite__context_t *sctx, apr_int64_t val)
1557 sqlite3_result_int64(sctx->context, val);
1561 svn_sqlite__result_error(svn_sqlite__context_t *sctx, const char *msg, int num)
1563 sqlite3_result_error(sctx->context, msg, num);