From f052052fadf2d0821a4c09a40042a7edd9dd4b00 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 4 May 2018 13:54:19 -0500 Subject: [PATCH 01/43] Remove dead code from pool. --- playhouse/pool.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/playhouse/pool.py b/playhouse/pool.py index 3e4227c07..0c81fad81 100644 --- a/playhouse/pool.py +++ b/playhouse/pool.py @@ -60,12 +60,7 @@ def do_something(foo, bar): """ import heapq import logging -import threading import time -try: - from Queue import Queue -except ImportError: - from queue import Queue try: from psycopg2 import extensions as pg_extensions @@ -100,11 +95,6 @@ def __init__(self, database, max_connections=20, stale_timeout=None, self._connections = [] self._in_use = {} self.conn_key = id - - if self._wait_timeout: - self._event = threading.Event() - self._ready_queue = Queue() - super(PooledDatabase, self).__init__(database, **kwargs) def init(self, database, max_connections=None, stale_timeout=None, From eae4e6bf44054da011e0b455d707497a0449cc60 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 7 May 2018 09:51:46 -0500 Subject: [PATCH 02/43] Modify sqlite backup interface to match pysqlite more closely. --- playhouse/_sqlite_ext.pyx | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/playhouse/_sqlite_ext.pyx b/playhouse/_sqlite_ext.pyx index 1a23cb324..4beab461a 100644 --- a/playhouse/_sqlite_ext.pyx +++ b/playhouse/_sqlite_ext.pyx @@ -121,7 +121,15 @@ cdef extern from "sqlite3.h": # Return values. cdef int SQLITE_OK = 0 cdef int SQLITE_ERROR = 1 + cdef int SQLITE_INTERNAL = 2 + cdef int SQLITE_PERM = 3 + cdef int SQLITE_ABORT = 4 + cdef int SQLITE_BUSY = 5 + cdef int SQLITE_LOCKED = 6 cdef int SQLITE_NOMEM = 7 + cdef int SQLITE_READONLY = 8 + cdef int SQLITE_INTERRUPT = 9 + cdef int SQLITE_DONE = 101 # Function type. cdef int SQLITE_DETERMINISTIC = 0x800 @@ -1425,29 +1433,42 @@ cdef void _update_callback(void *userData, int queryType, char *database, fn(query, decode(database), decode(table), rowid) -def backup(src_conn, dest_conn): +def backup(src_conn, dest_conn, pages=-1, name='main', progress=None): cdef: + int rc pysqlite_Connection *src = src_conn pysqlite_Connection *dest = dest_conn sqlite3 *src_db = src.db sqlite3 *dest_db = dest.db sqlite3_backup *backup - backup = sqlite3_backup_init(dest_db, 'main', src_db, 'main') - if (backup == NULL): + # We always backup to the "main" database in the dest db. + backup = sqlite3_backup_init(dest_db, 'main', src_db, name) + if backup == NULL: raise OperationalError('Unable to initialize backup.') - sqlite3_backup_step(backup, -1) + while True: + rc = sqlite3_backup_step(backup, pages) + if progress is not None: + # Progress-handler is called with (remaining, page count, is done?) + remaining = sqlite3_backup_remaining(backup) + page_count = sqlite3_backup_pagecount(backup) + progress(remaining, page_count, rc == SQLITE_DONE) + if rc == SQLITE_BUSY or rc == SQLITE_LOCKED: + sqlite3_sleep(250) + elif rc == SQLITE_DONE: + break + sqlite3_backup_finish(backup) if sqlite3_errcode(dest_db): - raise OperationalError('Error finishing backup: %s' % + raise OperationalError('Error backuping up database: %s' % sqlite3_errmsg(dest_db)) return True -def backup_to_file(src_conn, filename): +def backup_to_file(src_conn, filename, pages=-1, name='main', progress=None): dest_conn = pysqlite.connect(filename) - backup(src_conn, dest_conn) + backup(src_conn, dest_conn, pages=pages, name=name, progress=progress) dest_conn.close() return True From 166f44a1d270621aa42b6b7a4796b28edd5d4da9 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 7 May 2018 09:55:58 -0500 Subject: [PATCH 03/43] Docs for enhancements to backup API. --- docs/peewee/sqlite_ext.rst | 16 +++++++++++++++- playhouse/sqlite_ext.py | 13 ++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index 2097ff3e5..4b8b2a5ed 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -146,10 +146,17 @@ APIs >>> db.autocommit True - .. py:method:: backup(destination) + .. py:method:: backup(destination[, pages=-1, name='main', progress=None]) :param SqliteDatabase destination: Database object to serve as destination for the backup. + :param int pages: Number of pages per iteration. Default value of -1 + indicates all pages should be backed-up in a single step. + :param str name: Name of source database (may differ if you used ATTACH + DATABASE to load multiple databases). + :param progress: Progress callback, called with three parameters: the + number of pages remaining, the total page count, and whether the + backup is complete. Example: @@ -164,6 +171,13 @@ APIs .. py:method:: backup_to_file(filename) :param filename: Filename to store the database backup. + :param int pages: Number of pages per iteration. Default value of -1 + indicates all pages should be backed-up in a single step. + :param str name: Name of source database (may differ if you used ATTACH + DATABASE to load multiple databases). + :param progress: Progress callback, called with three parameters: the + number of pages remaining, the total page count, and whether the + backup is complete. Backup the current database to a file. The backed-up data is not a database dump, but an actual SQLite database file. diff --git a/playhouse/sqlite_ext.py b/playhouse/sqlite_ext.py index 89aa8f319..7847164a2 100644 --- a/playhouse/sqlite_ext.py +++ b/playhouse/sqlite_ext.py @@ -1080,11 +1080,14 @@ def last_insert_rowid(self): def autocommit(self): return self._conn_helper.autocommit() - def backup(self, destination): - return backup(self.connection(), destination.connection()) - - def backup_to_file(self, filename): - return backup_to_file(self.connection(), filename) + def backup(self, destination, pages=-1, name='main', progress=None): + return backup(self.connection(), destination.connection(), + pages=pages, name=name, progress=progress) + + def backup_to_file(self, filename, pages=-1, name='main', + progress=None): + return backup_to_file(self.connection(), filename, pages=pages, + name=name, progress=progress) def blob_open(self, table, column, rowid, read_only=False): return Blob(self, table, column, rowid, read_only) From afdebce48db2e2e9c430532acad5d5b79c00253e Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 7 May 2018 10:00:54 -0500 Subject: [PATCH 04/43] Use bytes for database names. --- playhouse/_sqlite_ext.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playhouse/_sqlite_ext.pyx b/playhouse/_sqlite_ext.pyx index 4beab461a..a5d51a4ce 100644 --- a/playhouse/_sqlite_ext.pyx +++ b/playhouse/_sqlite_ext.pyx @@ -1443,7 +1443,7 @@ def backup(src_conn, dest_conn, pages=-1, name='main', progress=None): sqlite3_backup *backup # We always backup to the "main" database in the dest db. - backup = sqlite3_backup_init(dest_db, 'main', src_db, name) + backup = sqlite3_backup_init(dest_db, b'main', src_db, encode(name)) if backup == NULL: raise OperationalError('Unable to initialize backup.') From 074dbaa87004dced59c5236bc937b6b3a69d2c29 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 7 May 2018 15:04:54 -0500 Subject: [PATCH 05/43] Simplify backup parameter spec and release GIL when doing backup ops. --- docs/peewee/sqlite_ext.rst | 8 ++++---- playhouse/_sqlite_ext.pyx | 23 ++++++++++++++--------- playhouse/sqlite_ext.py | 4 ++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index 4b8b2a5ed..2bb96a87d 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -146,14 +146,14 @@ APIs >>> db.autocommit True - .. py:method:: backup(destination[, pages=-1, name='main', progress=None]) + .. py:method:: backup(destination[, pages=None, name=None, progress=None]) :param SqliteDatabase destination: Database object to serve as destination for the backup. :param int pages: Number of pages per iteration. Default value of -1 indicates all pages should be backed-up in a single step. :param str name: Name of source database (may differ if you used ATTACH - DATABASE to load multiple databases). + DATABASE to load multiple databases). Defaults to "main". :param progress: Progress callback, called with three parameters: the number of pages remaining, the total page count, and whether the backup is complete. @@ -168,13 +168,13 @@ APIs # Backup the contents of master to replica. master.backup(replica) - .. py:method:: backup_to_file(filename) + .. py:method:: backup_to_file(filename[, pages, name, progress]) :param filename: Filename to store the database backup. :param int pages: Number of pages per iteration. Default value of -1 indicates all pages should be backed-up in a single step. :param str name: Name of source database (may differ if you used ATTACH - DATABASE to load multiple databases). + DATABASE to load multiple databases). Defaults to "main". :param progress: Progress callback, called with three parameters: the number of pages remaining, the total page count, and whether the backup is complete. diff --git a/playhouse/_sqlite_ext.pyx b/playhouse/_sqlite_ext.pyx index a5d51a4ce..139993cfe 100644 --- a/playhouse/_sqlite_ext.pyx +++ b/playhouse/_sqlite_ext.pyx @@ -245,7 +245,7 @@ cdef extern from "sqlite3.h": # Misc. cdef int sqlite3_busy_handler(sqlite3 *db, int(*)(void *, int), void *) - cdef int sqlite3_sleep(int ms) + cdef int sqlite3_sleep(int ms) nogil cdef sqlite3_backup *sqlite3_backup_init( sqlite3 *pDest, const char *zDestName, @@ -253,8 +253,8 @@ cdef extern from "sqlite3.h": const char *zSourceName) # Backup. - cdef int sqlite3_backup_step(sqlite3_backup *p, int nPage) - cdef int sqlite3_backup_finish(sqlite3_backup *p) + cdef int sqlite3_backup_step(sqlite3_backup *p, int nPage) nogil + cdef int sqlite3_backup_finish(sqlite3_backup *p) nogil cdef int sqlite3_backup_remaining(sqlite3_backup *p) cdef int sqlite3_backup_pagecount(sqlite3_backup *p) @@ -1433,8 +1433,10 @@ cdef void _update_callback(void *userData, int queryType, char *database, fn(query, decode(database), decode(table), rowid) -def backup(src_conn, dest_conn, pages=-1, name='main', progress=None): +def backup(src_conn, dest_conn, pages=None, name=None, progress=None): cdef: + bytes bname = encode(name or 'main') + int page_step = pages or -1 int rc pysqlite_Connection *src = src_conn pysqlite_Connection *dest = dest_conn @@ -1443,30 +1445,33 @@ def backup(src_conn, dest_conn, pages=-1, name='main', progress=None): sqlite3_backup *backup # We always backup to the "main" database in the dest db. - backup = sqlite3_backup_init(dest_db, b'main', src_db, encode(name)) + backup = sqlite3_backup_init(dest_db, b'main', src_db, bname) if backup == NULL: raise OperationalError('Unable to initialize backup.') while True: - rc = sqlite3_backup_step(backup, pages) + with nogil: + rc = sqlite3_backup_step(backup, page_step) if progress is not None: # Progress-handler is called with (remaining, page count, is done?) remaining = sqlite3_backup_remaining(backup) page_count = sqlite3_backup_pagecount(backup) progress(remaining, page_count, rc == SQLITE_DONE) if rc == SQLITE_BUSY or rc == SQLITE_LOCKED: - sqlite3_sleep(250) + with nogil: + sqlite3_sleep(250) elif rc == SQLITE_DONE: break - sqlite3_backup_finish(backup) + with nogil: + sqlite3_backup_finish(backup) if sqlite3_errcode(dest_db): raise OperationalError('Error backuping up database: %s' % sqlite3_errmsg(dest_db)) return True -def backup_to_file(src_conn, filename, pages=-1, name='main', progress=None): +def backup_to_file(src_conn, filename, pages=None, name=None, progress=None): dest_conn = pysqlite.connect(filename) backup(src_conn, dest_conn, pages=pages, name=name, progress=progress) dest_conn.close() diff --git a/playhouse/sqlite_ext.py b/playhouse/sqlite_ext.py index 7847164a2..efd3fff89 100644 --- a/playhouse/sqlite_ext.py +++ b/playhouse/sqlite_ext.py @@ -1080,11 +1080,11 @@ def last_insert_rowid(self): def autocommit(self): return self._conn_helper.autocommit() - def backup(self, destination, pages=-1, name='main', progress=None): + def backup(self, destination, pages=None, name=None, progress=None): return backup(self.connection(), destination.connection(), pages=pages, name=name, progress=progress) - def backup_to_file(self, filename, pages=-1, name='main', + def backup_to_file(self, filename, pages=None, name=None, progress=None): return backup_to_file(self.connection(), filename, pages=pages, name=name, progress=progress) From cbe2267dfc75430095676845f610a7151e05b472 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 8 May 2018 09:44:25 -0500 Subject: [PATCH 06/43] Add two new join types, LATERAL and NATURAL. --- peewee.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/peewee.py b/peewee.py index 0f1bc13a4..0bf6c3c42 100644 --- a/peewee.py +++ b/peewee.py @@ -316,7 +316,9 @@ def __add__(self, rhs): d = attrdict(self); d.update(rhs); return d RIGHT_OUTER='RIGHT OUTER', FULL='FULL', FULL_OUTER='FULL OUTER', - CROSS='CROSS') + CROSS='CROSS', + LATERAL='LATERAL', + NATURAL='NATURAL') # Row representations. ROW = attrdict( From d4c148d6ade02d97c6fbf11d4c60ee6fe1db6df3 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 9 May 2018 09:38:45 -0500 Subject: [PATCH 07/43] Declare sqlite library functions as nogil. --- playhouse/_sqlite_ext.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/playhouse/_sqlite_ext.pyx b/playhouse/_sqlite_ext.pyx index 139993cfe..07b6358f6 100644 --- a/playhouse/_sqlite_ext.pyx +++ b/playhouse/_sqlite_ext.pyx @@ -46,7 +46,7 @@ cdef struct sqlite3_index_constraint_usage: unsigned char omit -cdef extern from "sqlite3.h": +cdef extern from "sqlite3.h" nogil: ctypedef struct sqlite3: int busyTimeout ctypedef struct sqlite3_backup @@ -245,7 +245,7 @@ cdef extern from "sqlite3.h": # Misc. cdef int sqlite3_busy_handler(sqlite3 *db, int(*)(void *, int), void *) - cdef int sqlite3_sleep(int ms) nogil + cdef int sqlite3_sleep(int ms) cdef sqlite3_backup *sqlite3_backup_init( sqlite3 *pDest, const char *zDestName, @@ -253,8 +253,8 @@ cdef extern from "sqlite3.h": const char *zSourceName) # Backup. - cdef int sqlite3_backup_step(sqlite3_backup *p, int nPage) nogil - cdef int sqlite3_backup_finish(sqlite3_backup *p) nogil + cdef int sqlite3_backup_step(sqlite3_backup *p, int nPage) + cdef int sqlite3_backup_finish(sqlite3_backup *p) cdef int sqlite3_backup_remaining(sqlite3_backup *p) cdef int sqlite3_backup_pagecount(sqlite3_backup *p) @@ -1478,7 +1478,7 @@ def backup_to_file(src_conn, filename, pages=None, name=None, progress=None): return True -cdef int _aggressive_busy_handler(void *ptr, int n): +cdef int _aggressive_busy_handler(void *ptr, int n) nogil: # In concurrent environments, it often seems that if multiple queries are # kicked off at around the same time, they proceed in lock-step to check # for the availability of the lock. By introducing some "jitter" we can From a884872f540188f677ccf4e985beb2ebc2e525ac Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 9 May 2018 10:54:20 -0500 Subject: [PATCH 08/43] Add test for integer where-clause chaining and joins. --- tests/sqlite.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/sqlite.py b/tests/sqlite.py index 00648c520..0ac6c0b76 100644 --- a/tests/sqlite.py +++ b/tests/sqlite.py @@ -1661,3 +1661,44 @@ def test_json_contains(self): self.assertContains({'x1': {'y2': 'z2', 'y3': [0, 1, 2, 4]}}, []) self.assertContains({'x1': {'y2': 'z2', 'y3': [0, 2]}}, []) + + +class CalendarMonth(TestModel): + name = TextField() + value = IntegerField() + +class CalendarDay(TestModel): + month = ForeignKeyField(CalendarMonth, backref='days') + value = IntegerField() + + +class TestIntWhereChain(ModelTestCase): + database = database + requires = [CalendarMonth, CalendarDay] + + def test_int_where_chain(self): + with self.database.atomic(): + jan = CalendarMonth.create(name='january', value=1) + feb = CalendarMonth.create(name='february', value=2) + CalendarDay.insert_many([{'month': jan, 'value': i + 1} + for i in range(31)]).execute() + CalendarDay.insert_many([{'month': feb, 'value': i + 1} + for i in range(28)]).execute() + + def assertValues(query, expected): + self.assertEqual(sorted([d.value for d in query]), list(expected)) + + q = CalendarDay.select().join(CalendarMonth) + jq = q.where(CalendarMonth.name == 'january') + jq1 = jq.where(CalendarDay.value >= 25) + assertValues(jq1, range(25, 32)) + + jq2 = jq1.where(CalendarDay.value < 30) + assertValues(jq2, range(25, 30)) + + fq = q.where(CalendarMonth.name == 'february') + fq1 = fq.where(CalendarDay.value >= 25) + assertValues(fq1, range(25, 29)) + + fq2 = fq1.where(CalendarDay.value < 30) + assertValues(fq2, range(25, 29)) From 909b371123a73ea00b7db18bbdb870083286d603 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 9 May 2018 18:43:41 -0500 Subject: [PATCH 09/43] Add test for union with join, and alias on join predicate. --- tests/models.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/models.py b/tests/models.py index d1cffc12f..df04ebfad 100644 --- a/tests/models.py +++ b/tests/models.py @@ -747,6 +747,37 @@ def test_union_column_resolution(self): (2, 'u1-t2', 'u1'), (3, 'u2-t1', 'u2')]) + @requires_models(User, Tweet) + def test_union_with_join(self): + u1, u2 = [User.create(username='u%s' % i) for i in (1, 2)] + for u, ts in ((u1, ('t1', 't2')), (u2, ('t1',))): + for t in ts: + Tweet.create(user=u, content='%s-%s' % (u.username, t)) + + q1 = (User + .select(User, Tweet) + .join(Tweet, on=(Tweet.user == User.id).alias('tweet'))) + q2 = (User + .select(User, Tweet) + .join(Tweet, on=(Tweet.user == User.id).alias('tweet'))) + + with self.assertQueryCount(1): + self.assertEqual( + sorted([(user.username, user.tweet.content) for user in q1]), + [('u1', 'u1-t1'), ('u1', 'u1-t2'), ('u2', 'u2-t1')]) + + uq = q1.union_all(q2) + with self.assertQueryCount(1): + result = [(user.username, user.tweet.content) for user in uq] + self.assertEqual(sorted(result), [ + ('u1', 'u1-t1'), + ('u1', 'u1-t1'), + ('u1', 'u1-t2'), + ('u1', 'u1-t2'), + ('u2', 'u2-t1'), + ('u2', 'u2-t1'), + ]) + @requires_models(Category) def test_self_referential_fk(self): self.assertTrue(Category.parent.rel_model is Category) From 809c977217c4bdc5b0dd4769c0675caabaea91e4 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 9 May 2018 18:47:13 -0500 Subject: [PATCH 10/43] Use an unambiguous alias to ensure attribute is preserved. --- tests/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/models.py b/tests/models.py index df04ebfad..d33938da1 100644 --- a/tests/models.py +++ b/tests/models.py @@ -756,19 +756,19 @@ def test_union_with_join(self): q1 = (User .select(User, Tweet) - .join(Tweet, on=(Tweet.user == User.id).alias('tweet'))) + .join(Tweet, on=(Tweet.user == User.id).alias('foo'))) q2 = (User .select(User, Tweet) - .join(Tweet, on=(Tweet.user == User.id).alias('tweet'))) + .join(Tweet, on=(Tweet.user == User.id).alias('foo'))) with self.assertQueryCount(1): self.assertEqual( - sorted([(user.username, user.tweet.content) for user in q1]), + sorted([(user.username, user.foo.content) for user in q1]), [('u1', 'u1-t1'), ('u1', 'u1-t2'), ('u2', 'u2-t1')]) uq = q1.union_all(q2) with self.assertQueryCount(1): - result = [(user.username, user.tweet.content) for user in uq] + result = [(user.username, user.foo.content) for user in uq] self.assertEqual(sorted(result), [ ('u1', 'u1-t1'), ('u1', 'u1-t1'), From d88c238a7078e1914ce7e054439b0d7695992c65 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 10 May 2018 13:12:57 -0500 Subject: [PATCH 11/43] Test behavior of mixing union + cte queries. --- tests/models.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/models.py b/tests/models.py index d33938da1..dcbd00daf 100644 --- a/tests/models.py +++ b/tests/models.py @@ -778,6 +778,25 @@ def test_union_with_join(self): ('u2', 'u2-t1'), ]) + @skip_if(IS_SQLITE_OLD or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) + @requires_models(User) + def test_union_cte(self): + with self.database.atomic(): + (User + .insert_many({'username': 'u%s' % i} for i in range(10)) + .execute()) + + lhs = User.select().where(User.username.in_(['u1', 'u3'])) + rhs = User.select().where(User.username.in_(['u5', 'u7'])) + u_cte = (lhs | rhs).cte('users_union') + + query = (User + .select(User.username) + .join(u_cte, on=(User.id == u_cte.c.id)) + .where(User.username.in_(['u1', 'u7'])) + .with_cte(u_cte)) + self.assertEqual(sorted([u.username for u in query]), ['u1', 'u7']) + @requires_models(Category) def test_self_referential_fk(self): self.assertTrue(Category.parent.rel_model is Category) From 2f0b3e38ca133bd39dff9efc9cee7fbe6749cb93 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 10 May 2018 13:29:47 -0500 Subject: [PATCH 12/43] Fix outdated SQL example in "hacks". --- docs/peewee/hacks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/peewee/hacks.rst b/docs/peewee/hacks.rst index e23e6ca24..58d806e0e 100644 --- a/docs/peewee/hacks.rst +++ b/docs/peewee/hacks.rst @@ -212,11 +212,11 @@ To accomplish this with peewee we'll need to express the lateral join as a :py:c # Now we join the outer and inner queries using the LEFT LATERAL # JOIN. The join predicate is *ON TRUE*, since we're effectively # joining in the tweet subquery's WHERE clause. - join_clause = Clause( + join_clause = NodeList(( user_query, SQL('LEFT JOIN LATERAL'), tweet_query, - SQL('ON %s', True)) + SQL('ON %s', True))) # Finally, we'll wrap these up and SELECT from the result. query = (Tweet From 3ef5de11e273017761878d8ce1a955eb38de3916 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 10 May 2018 13:54:51 -0500 Subject: [PATCH 13/43] Add test for postgres lateral join. --- docs/peewee/hacks.rst | 5 +++-- peewee.py | 1 - tests/models.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/docs/peewee/hacks.rst b/docs/peewee/hacks.rst index 58d806e0e..18e795f13 100644 --- a/docs/peewee/hacks.rst +++ b/docs/peewee/hacks.rst @@ -216,11 +216,12 @@ To accomplish this with peewee we'll need to express the lateral join as a :py:c user_query, SQL('LEFT JOIN LATERAL'), tweet_query, - SQL('ON %s', True))) + SQL('ON %s', [True]))) # Finally, we'll wrap these up and SELECT from the result. query = (Tweet - .select(SQL('*')) + .select(user_query.c.username, tweet_query.c.message, + tweet_query.c.create_date) .from_(join_clause)) Window functions diff --git a/peewee.py b/peewee.py index 0bf6c3c42..cdac3be8a 100644 --- a/peewee.py +++ b/peewee.py @@ -317,7 +317,6 @@ def __add__(self, rhs): d = attrdict(self); d.update(rhs); return d FULL='FULL', FULL_OUTER='FULL OUTER', CROSS='CROSS', - LATERAL='LATERAL', NATURAL='NATURAL') # Row representations. diff --git a/tests/models.py b/tests/models.py index dcbd00daf..dcad31144 100644 --- a/tests/models.py +++ b/tests/models.py @@ -4,6 +4,7 @@ import unittest from peewee import * +from peewee import NodeList from peewee import QualifiedNames from peewee import sort_models @@ -3210,3 +3211,48 @@ def test_update_from_simple(self): query.execute() self.assertEqual(User.get(User.id == u.id).username, 't2') + + +@requires_postgresql +class TestLateralJoin(ModelTestCase): + requires = [User, Tweet] + + def test_lateral_join(self): + with self.database.atomic(): + for i in range(3): + u = User.create(username='u%s' % i) + for j in range(4): + Tweet.create(user=u, content='u%s-t%s' % (i, j)) + + # GOAL: query users and their 2 most-recent tweets (by ID). + TA = Tweet.alias() + + # The "outer loop" will be iterating over the users whose tweets we are + # trying to find. + user_query = (User + .select(User.id, User.username) + .order_by(User.id) + .alias('uq')) + + # The inner loop will select tweets and is correlated to the outer loop + # via the WHERE clause. Note that we are using a LIMIT clause. + tweet_query = (TA + .select(TA.id, TA.content) + .where(TA.user == user_query.c.id) + .order_by(TA.id.desc()) + .limit(2) + .alias('pq')) + + join = NodeList((user_query, SQL('LEFT JOIN LATERAL'), tweet_query, + SQL('ON %s', [True]))) + query = (Tweet + .select(user_query.c.username, tweet_query.c.content) + .from_(join) + .dicts()) + self.assertEqual([row for row in query], [ + {'username': 'u0', 'content': 'u0-t3'}, + {'username': 'u0', 'content': 'u0-t2'}, + {'username': 'u1', 'content': 'u1-t3'}, + {'username': 'u1', 'content': 'u1-t2'}, + {'username': 'u2', 'content': 'u2-t3'}, + {'username': 'u2', 'content': 'u2-t2'}]) From 6ade1660cd2613592da08ef5d7f659da4fea6380 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 10 May 2018 14:19:51 -0500 Subject: [PATCH 14/43] Properly handle errors in backup progress handler. Add tests. --- playhouse/_sqlite_ext.pyx | 6 ++++- tests/cysqlite.py | 46 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/playhouse/_sqlite_ext.pyx b/playhouse/_sqlite_ext.pyx index 07b6358f6..3a3c4edac 100644 --- a/playhouse/_sqlite_ext.pyx +++ b/playhouse/_sqlite_ext.pyx @@ -1456,7 +1456,11 @@ def backup(src_conn, dest_conn, pages=None, name=None, progress=None): # Progress-handler is called with (remaining, page count, is done?) remaining = sqlite3_backup_remaining(backup) page_count = sqlite3_backup_pagecount(backup) - progress(remaining, page_count, rc == SQLITE_DONE) + try: + progress(remaining, page_count, rc == SQLITE_DONE) + except: + sqlite3_backup_finish(backup) + raise if rc == SQLITE_BUSY or rc == SQLITE_LOCKED: with nogil: sqlite3_sleep(250) diff --git a/tests/cysqlite.py b/tests/cysqlite.py index 6ae809b24..a39830efb 100644 --- a/tests/cysqlite.py +++ b/tests/cysqlite.py @@ -154,14 +154,27 @@ def tearDown(self): if os.path.exists(self.backup_filename): os.unlink(self.backup_filename) - def test_backup_to_file(self): - # Populate the database with some test data. + def _populate_test_data(self, nrows=100): self.execute('CREATE TABLE register (id INTEGER NOT NULL PRIMARY KEY, ' 'value INTEGER NOT NULL)') with self.database.atomic(): - for i in range(100): + for i in range(nrows): self.execute('INSERT INTO register (value) VALUES (?)', i) + def test_backup(self): + self._populate_test_data() + + # Back-up to an in-memory database and verify contents. + other_db = CSqliteExtDatabase(':memory:') + self.database.backup(other_db) + cursor = other_db.execute_sql('SELECT value FROM register ORDER BY ' + 'value;') + self.assertEqual([val for val, in cursor.fetchall()], list(range(100))) + other_db.close() + + def test_backup_to_file(self): + self._populate_test_data() + self.database.backup_to_file(self.backup_filename) backup_db = CSqliteExtDatabase(self.backup_filename) cursor = backup_db.execute_sql('SELECT value FROM register ORDER BY ' @@ -169,6 +182,33 @@ def test_backup_to_file(self): self.assertEqual([val for val, in cursor.fetchall()], list(range(100))) backup_db.close() + def test_backup_progress(self): + self._populate_test_data() + + accum = [] + def progress(remaining, total, is_done): + accum.append((remaining, total, is_done)) + + other_db = CSqliteExtDatabase(':memory:') + self.database.backup(other_db, pages=1, progress=progress) + self.assertTrue(len(accum) > 0) + + sql = 'select value from register order by value;' + self.assertEqual([r for r, in other_db.execute_sql(sql)], + list(range(100))) + other_db.close() + + def test_backup_progress_error(self): + self._populate_test_data() + + def broken_progress(remaining, total, is_done): + raise ValueError('broken') + + other_db = CSqliteExtDatabase(':memory:') + self.assertRaises(ValueError, self.database.backup, other_db, + progress=broken_progress) + other_db.close() + class TestBlob(CyDatabaseTestCase): def setUp(self): From 62e614924f3b2887380faf989f8df1aee10396f3 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 11 May 2018 15:01:23 -0500 Subject: [PATCH 15/43] Treat limit and offset as parameters rather than literals --- peewee.py | 4 ++-- tests/model_sql.py | 4 ++-- tests/models.py | 2 +- tests/sql.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/peewee.py b/peewee.py index cdac3be8a..c4e9014e6 100644 --- a/peewee.py +++ b/peewee.py @@ -1691,9 +1691,9 @@ def _apply_ordering(self, ctx): .sql(CommaNodeList(self._order_by))) if self._limit is not None or (self._offset is not None and ctx.state.limit_max): - ctx.literal(' LIMIT %d' % (self._limit or ctx.state.limit_max)) + ctx.literal(' LIMIT ').sql(self._limit or ctx.state.limit_max) if self._offset is not None: - ctx.literal(' OFFSET %d' % self._offset) + ctx.literal(' OFFSET ').sql(self._offset) return ctx def __sql__(self, ctx): diff --git a/tests/model_sql.py b/tests/model_sql.py index a15ecdbab..09f3fc138 100644 --- a/tests/model_sql.py +++ b/tests/model_sql.py @@ -581,9 +581,9 @@ def test_limit(self): # This may be invalid SQL, but this at least documents the behavior. self.assertSQL(compound, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" ' - 'ORDER BY "t1"."alpha" LIMIT 3 UNION ' + 'ORDER BY "t1"."alpha" LIMIT ? UNION ' 'SELECT "t2"."beta" FROM "beta" AS "t2" ' - 'ORDER BY "t2"."beta" LIMIT 4 LIMIT 5'), []) + 'ORDER BY "t2"."beta" LIMIT ? LIMIT ?'), [3, 4, 5]) def test_union_from(self): lhs = Alpha.select(Alpha.alpha).where(Alpha.alpha < 2) diff --git a/tests/models.py b/tests/models.py index dcad31144..2ae91b58c 100644 --- a/tests/models.py +++ b/tests/models.py @@ -2978,7 +2978,7 @@ def test_count_union(self): 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'INTERSECT ' 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2" ' - 'LIMIT 3'), []) + 'LIMIT ?'), [3]) self.assertEqual(query.count(), 3) diff --git a/tests/sql.py b/tests/sql.py index 2b26f0d09..f062f5e22 100644 --- a/tests/sql.py +++ b/tests/sql.py @@ -577,8 +577,8 @@ def test_delete_query(self): .limit(3)) self.assertSQL( query, - 'DELETE FROM "users" WHERE ("username" != ?) LIMIT 3', - ['charlie']) + 'DELETE FROM "users" WHERE ("username" != ?) LIMIT ?', + ['charlie', 3]) def test_delete_subquery(self): count = fn.COUNT(Tweet.c.id).alias('ct') From 056d6adb04d4321181bab21d52a115b03ed0ee54 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 11 May 2018 15:45:23 -0500 Subject: [PATCH 16/43] Test FOR UPDATE integration with newer mysql. --- tests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models.py b/tests/models.py index 2ae91b58c..031bcef98 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1621,7 +1621,7 @@ def test_bounds(self): (3, 100., 104.)]) -@requires_postgresql +@skip_if(IS_SQLITE or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) class TestForUpdateIntegration(ModelTestCase): requires = [User] From 5ff2d8a31597fdbd6fb9e710337bf61d8aa4ccce Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 11 May 2018 15:56:44 -0500 Subject: [PATCH 17/43] Improved identifier quoting support to allow open and close quote. This may make it easier to quote identifiers in databases like MSSQL which use square brackets, e.g. [thetable].[thecolumn]. --- peewee.py | 15 +++++++-------- playhouse/_speedups.pyx | 5 ++--- tests/database.py | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/peewee.py b/peewee.py index c4e9014e6..b1c12f57d 100644 --- a/peewee.py +++ b/peewee.py @@ -143,11 +143,10 @@ def emit(self, record): try: from playhouse._speedups import quote except ImportError: - def quote(path, quote_char): - quotes = (quote_char, quote_char) + def quote(path, quote_chars): if len(path) == 1: - return path[0].join(quotes) - return '.'.join([part.join(quotes) for part in path]) + return path[0].join(quote_chars) + return '.'.join([part.join(quote_chars) for part in path]) if sys.version_info[0] == 2: @@ -1270,7 +1269,7 @@ def __hash__(self): return hash((self.__class__.__name__, tuple(self._path))) def __sql__(self, ctx): - return ctx.literal(quote(self._path, ctx.state.quote or '"')) + return ctx.literal(quote(self._path, ctx.state.quote or '""')) class SQL(ColumnBase): @@ -2433,7 +2432,7 @@ class Database(_callable_context_manager): field_types = {} operations = {} param = '?' - quote = '"' + quote = '""' # Feature toggles. commit_select = False @@ -3219,7 +3218,7 @@ class MySQLDatabase(Database): 'ILIKE': 'LIKE', 'XOR': 'XOR'} param = '%s' - quote = '`' + quote = '``' commit_select = True for_update = True @@ -3422,7 +3421,7 @@ class _savepoint(_callable_context_manager): def __init__(self, db, sid=None): self.db = db self.sid = sid or 's' + uuid.uuid4().hex - self.quoted_sid = self.sid.join((self.db.quote, self.db.quote)) + self.quoted_sid = self.sid.join(self.db.quote) def _begin(self): self.db.execute_sql('SAVEPOINT %s;' % self.quoted_sid) diff --git a/playhouse/_speedups.pyx b/playhouse/_speedups.pyx index de566557e..a0500aec7 100644 --- a/playhouse/_speedups.pyx +++ b/playhouse/_speedups.pyx @@ -2,10 +2,9 @@ def quote(list path, str quote_char): cdef: int n = len(path) str part - tuple quotes = (quote_char, quote_char) if n == 1: - return path[0].join(quotes) + return path[0].join(quote_char) elif n > 1: - return '.'.join([part.join(quotes) for part in path]) + return '.'.join([part.join(quote_char) for part in path]) return '' diff --git a/tests/database.py b/tests/database.py index 6b6eb2907..09d9a3692 100644 --- a/tests/database.py +++ b/tests/database.py @@ -93,7 +93,7 @@ class TestDatabase(Database): self.assertEqual(state.operations['ILIKE'], 'ILIKE') self.assertEqual(state.param, '$') - self.assertEqual(state.quote, '"') + self.assertEqual(state.quote, '""') test_db2 = TestDatabase(None, field_types={'BIGINT': 'XXX_BIGINT', 'INT': 'XXX_INT'}) From c8a7eed81e27e35881921abdde393300d13f40cd Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 11 May 2018 16:07:00 -0500 Subject: [PATCH 18/43] Updates to changelog, sqlite docs, and remove mysql flag in test. --- CHANGELOG.md | 15 +++++++++++++++ docs/peewee/sqlite_ext.rst | 14 +++++++++++++- tests/models.py | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fa213698..7a1cfce42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ releases, visit GitHub: https://github.com/coleifer/peewee/releases +## master + +* LIMIT and OFFSET parameters are now treated as parameterized values instead + of literals. +* SQLite backup interface supports specifying page-counts and a user-defined + progress handler. +* GIL is released when doing backups or during SQLite busy timeouts (when using + the peewee SQLite busy-handler). +* Add NATURAL join-type to the `JOIN` helper. +* Improved identifier quoting to allow specifying distinct open/close-quote + characters. Enables adding support for MSSQL, for instance, which uses square + brackets, e.g. `[table].[column]`. + +[View commits](https://github.com/coleifer/peewee/compare/3.3.4...HEAD) + ## 3.3.4 * Added a `BinaryUUIDField` class for efficiently storing UUIDs in 16-bytes. diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index 2bb96a87d..bf3fe28da 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -60,7 +60,19 @@ APIs Extends :py:class:`SqliteDatabase` and inherits methods for declaring user-defined functions, pragmas, etc. -.. py:class:: CSqliteExtDatabase(database[, pragmas=None[, timeout=5[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False]]]]]]]) +.. py:class:: CSqliteExtDatabase(database[, pragmas=None[, timeout=5[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False[, replace_busy_handler=False]]]]]]]]) + + :param list pragmas: A list of 2-tuples containing pragma key and value to + set every time a connection is opened. + :param timeout: Set the busy-timeout on the SQLite driver (in seconds). + :param bool c_extensions: Declare that C extension speedups must/must-not + be used. If set to ``True`` and the extension module is not available, + will raise an :py:class:`ImproperlyConfigured` exception. + :param bool rank_functions: Make search result ranking functions available. + :param bool hash_functions: Make hashing functions available (md5, sha1, etc). + :param bool regexp_function: Make the REGEXP function available. + :param bool bloomfilter: Make the :ref:`sqlite-bloomfilter` available. + :param bool replace_busy_handler: Use a smarter busy-handler implementation. Extends :py:class:`SqliteExtDatabase` and requires that the ``playhouse._sqlite_ext`` extension module be available. diff --git a/tests/models.py b/tests/models.py index 031bcef98..2ae91b58c 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1621,7 +1621,7 @@ def test_bounds(self): (3, 100., 104.)]) -@skip_if(IS_SQLITE or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) +@requires_postgresql class TestForUpdateIntegration(ModelTestCase): requires = [User] From 65a3931bef498713d0ee6ff184e325de9b0ebb4c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 11 May 2018 17:08:01 -0500 Subject: [PATCH 19/43] Slight re-org and cleanup in join code. --- peewee.py | 53 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/peewee.py b/peewee.py index b1c12f57d..665221965 100644 --- a/peewee.py +++ b/peewee.py @@ -5745,12 +5745,16 @@ def _normalize_join(self, src, dest, on, attr): attr = on._alias on = on.alias() + # Obtain references to the source and destination models being joined. src_model, src_is_model = self._get_model(src) dest_model, dest_is_model = self._get_model(dest) + if src_model and dest_model: self._join_ctx = dest constructor = dest_model + # In the case where the "on" clause is a Column or Field, we will + # convert that field into the appropriate predicate expression. if not (src_is_model and dest_is_model) and isinstance(on, Column): if on.source is src: to_field = src_model._meta.columns[on.name] @@ -5795,31 +5799,17 @@ def _normalize_join(self, src, dest, on, attr): return (on, attr, constructor) - @Node.copy - def join(self, dest, join_type='INNER', on=None, src=None, attr=None): - src = self._join_ctx if src is None else src - - on, attr, constructor = self._normalize_join(src, dest, on, attr) - if attr: - self._joins.setdefault(src, []) - self._joins[src].append((dest, attr, constructor)) - - if not self._from_list: - raise ValueError('No sources to join on.') - item = self._from_list.pop() - self._from_list.append(Join(item, dest, join_type, on)) - - def join_from(self, src, dest, join_type='INNER', on=None, attr=None): - return self.join(dest, join_type, on, src, attr) - def _generate_on_clause(self, src, dest, to_field=None, on=None): meta = src._meta - backref = fk_fields = False + is_backref = fk_fields = False + + # Get all the foreign keys between source and dest, and determine if + # the join is via a back-reference. if dest in meta.model_refs: fk_fields = meta.model_refs[dest] elif dest in meta.model_backrefs: fk_fields = meta.model_backrefs[dest] - backref = True + is_backref = True if not fk_fields: if on is not None: @@ -5827,12 +5817,14 @@ def _generate_on_clause(self, src, dest, to_field=None, on=None): raise ValueError('Unable to find foreign key between %s and %s. ' 'Please specify an explicit join condition.' % (src, dest)) - if to_field is not None: + elif to_field is not None: + # If the foreign-key field was specified explicitly, remove all + # other foreign-key fields from the list. target = (to_field.field if isinstance(to_field, FieldAlias) else to_field) fk_fields = [f for f in fk_fields if ( (f is target) or - (backref and f.rel_field is to_field))] + (is_backref and f.rel_field is to_field))] if len(fk_fields) > 1: if on is None: @@ -5841,9 +5833,24 @@ def _generate_on_clause(self, src, dest, to_field=None, on=None): (src, dest)) return None, False else: - fk_field = fk_fields[0] + return fk_fields[0], is_backref + + @Node.copy + def join(self, dest, join_type='INNER', on=None, src=None, attr=None): + src = self._join_ctx if src is None else src + + on, attr, constructor = self._normalize_join(src, dest, on, attr) + if attr: + self._joins.setdefault(src, []) + self._joins[src].append((dest, attr, constructor)) - return fk_field, backref + if not self._from_list: + raise ValueError('No sources to join on.') + item = self._from_list.pop() + self._from_list.append(Join(item, dest, join_type, on)) + + def join_from(self, src, dest, join_type='INNER', on=None, attr=None): + return self.join(dest, join_type, on, src, attr) def _get_model_cursor_wrapper(self, cursor): if len(self._from_list) == 1 and not self._joins: From 171af17f5cced96c15d5e32e832474aa1618ab8c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 11 May 2018 17:22:15 -0500 Subject: [PATCH 20/43] Unify docs and semantics around timeout on SQLite. --- docs/peewee/api.rst | 6 +++--- docs/peewee/sqlite_ext.rst | 8 ++++---- peewee.py | 10 +++++----- tests/cysqlite.py | 2 +- tests/sqlite.py | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index e8f29acd0..47251777d 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -456,11 +456,11 @@ Database pass -.. py:class:: SqliteDatabase(database[, pragmas=None[, timeout=5[, **kwargs]]]) +.. py:class:: SqliteDatabase(database[, pragmas=None[, timeout=5000[, **kwargs]]]) :param list pragmas: A list of 2-tuples containing pragma key and value to set every time a connection is opened. - :param timeout: Set the busy-timeout on the SQLite driver (in seconds). + :param timeout: Set the busy-timeout on the SQLite driver (in milliseconds). Sqlite database implementation. :py:class:`SqliteDatabase` that provides some advanced features only offered by Sqlite. @@ -534,7 +534,7 @@ Database .. py:attribute:: timeout - Get or set the busy timeout (seconds). + Get or set the busy timeout (milliseconds). .. py:method:: register_aggregate(klass[, name=None[, num_params=-1]]) diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index bf3fe28da..b9785d32d 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -44,11 +44,11 @@ Instantiating a :py:class:`SqliteExtDatabase`: APIs ---- -.. py:class:: SqliteExtDatabase(database[, pragmas=None[, timeout=5[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False]]]]]]]) +.. py:class:: SqliteExtDatabase(database[, pragmas=None[, timeout=5000[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False]]]]]]]) :param list pragmas: A list of 2-tuples containing pragma key and value to set every time a connection is opened. - :param timeout: Set the busy-timeout on the SQLite driver (in seconds). + :param int timeout: Set the busy-timeout on the SQLite driver (in milliseconds). :param bool c_extensions: Declare that C extension speedups must/must-not be used. If set to ``True`` and the extension module is not available, will raise an :py:class:`ImproperlyConfigured` exception. @@ -60,11 +60,11 @@ APIs Extends :py:class:`SqliteDatabase` and inherits methods for declaring user-defined functions, pragmas, etc. -.. py:class:: CSqliteExtDatabase(database[, pragmas=None[, timeout=5[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False[, replace_busy_handler=False]]]]]]]]) +.. py:class:: CSqliteExtDatabase(database[, pragmas=None[, timeout=5000[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False[, replace_busy_handler=False]]]]]]]]) :param list pragmas: A list of 2-tuples containing pragma key and value to set every time a connection is opened. - :param timeout: Set the busy-timeout on the SQLite driver (in seconds). + :param int timeout: Set the busy-timeout on the SQLite driver (in milliseconds). :param bool c_extensions: Declare that C extension speedups must/must-not be used. If set to ``True`` and the extension module is not available, will raise an :py:class:`ImproperlyConfigured` exception. diff --git a/peewee.py b/peewee.py index 665221965..929e1d987 100644 --- a/peewee.py +++ b/peewee.py @@ -2769,7 +2769,7 @@ def __init__(self, database, *args, **kwargs): self.register_function(_sqlite_date_part, 'date_part', 2) self.register_function(_sqlite_date_trunc, 'date_trunc', 2) - def init(self, database, pragmas=None, timeout=5, **kwargs): + def init(self, database, pragmas=None, timeout=5000, **kwargs): if pragmas is not None: self._pragmas = pragmas self._timeout = timeout @@ -2835,13 +2835,13 @@ def timeout(self): return self._timeout @timeout.setter - def timeout(self, seconds): - if self._timeout == seconds: + def timeout(self, milliseconds): + if self._timeout == milliseconds: return - self._timeout = seconds + self._timeout = milliseconds if not self.is_closed(): - self.execute_sql('PRAGMA busy_timeout=%d;' % (seconds * 1000)) + self.execute_sql('PRAGMA busy_timeout=%d;' % milliseconds) def _load_aggregates(self, conn): for name, (klass, num_params) in self._aggregates.items(): diff --git a/tests/cysqlite.py b/tests/cysqlite.py index a39830efb..360f61fe2 100644 --- a/tests/cysqlite.py +++ b/tests/cysqlite.py @@ -9,7 +9,7 @@ from .base import DatabaseTestCase -database = CSqliteExtDatabase('peewee_test.db', timeout=0.1, +database = CSqliteExtDatabase('peewee_test.db', timeout=100, hash_functions=1) diff --git a/tests/sqlite.py b/tests/sqlite.py index 0ac6c0b76..30a672136 100644 --- a/tests/sqlite.py +++ b/tests/sqlite.py @@ -16,7 +16,7 @@ from .sqlite_helpers import json_installed -database = SqliteExtDatabase(':memory:', c_extensions=False, timeout=0.1) +database = SqliteExtDatabase(':memory:', c_extensions=False, timeout=100) CLOSURE_EXTENSION = os.environ.get('PEEWEE_CLOSURE_EXTENSION') From e21e8e9fdb3fbab091a3210566f85762dae9a962 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 11 May 2018 23:48:33 -0500 Subject: [PATCH 21/43] Test for update integration with mysql (fixes) --- tests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models.py b/tests/models.py index 2ae91b58c..2a55b6c28 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1621,7 +1621,7 @@ def test_bounds(self): (3, 100., 104.)]) -@requires_postgresql +@skip_if(IS_SQLITE or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) class TestForUpdateIntegration(ModelTestCase): requires = [User] @@ -1680,7 +1680,7 @@ def will_fail(): .for_update('FOR UPDATE NOWAIT') .get()) - self.assertRaises(OperationalError, will_fail) + self.assertRaises((OperationalError, InternalError), will_fail) class ServerDefault(TestModel): From 01db0013f9434e3474156a8b4da8c81d76d329d7 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sat, 12 May 2018 01:35:35 -0500 Subject: [PATCH 22/43] Add querying doc for upsert and replace functionality. --- docs/peewee/querying.rst | 68 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 29f9fd56a..8f72527bc 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -265,6 +265,74 @@ write such a query: >>> update = User.update(num_tweets=subquery) >>> update.execute() +Upsert +^^^^^^ + +Peewee provides support for varying types of upsert functionality. With SQLite +prior to 3.24.0 and MySQL, Peewee offers the :py:meth:`~Model.replace`, which +allows you to insert a record or, in the event of a constraint violation, +replace the existing record. + +Example of using `~Model.replace` and :py:meth:`~Insert.on_conflict_replace`: + +.. code-block:: python + + class User(Model): + username = TextField(unique=True) + last_login = DateTimeField(null=True) + + # Insert or update the user. The "last_login" value will be updated + # regardless of whether the user existed previously. + user_id = (User + .replace(username='the-user', last_login=datetime.now()) + .execute()) + + # This query is equivalent: + user_id = (User + .insert(username='the-user', last_login=datetime.now()) + .on_conflict_replace() + .execute()) + +.. note:: + In addition to *replace*, SQLite, MySQL and Postgresql provide an *ignore* + action (see: :py:meth:`~Insert.on_conflict_ignore`) if you simply wish to + insert and ignore any potential constraint violation. + +Postgresql and SQLite (3.24.0 and newer) provide a different syntax that allows +for more granular control over which constraint violation should trigger the +conflict resolution, and what values should be updated or preserved. + +Example of using :py:meth:`~Insert.on_conflict` with Postgresql to update +certain values: + +.. code-block:: python + + class User(Model): + username = TextField(unique=True) + last_login = DateTimeField(null=True) + login_count = IntegerField() + + # Insert a new user. + User.create(username='huey', login_count=0) + + # Simulate the user logging in. The login count and timestamp will be + # either created or updated correctly. + now = datetime.now() + rowid = (User + .insert(username='huey', last_login=now, login_count=1) + .on_conflict( + conflict_target=(User.username,), # Which constraint? + preserve=(User.last_login,), # Use the value we would have inserted. + update={User.login_count: User.login_count + 1}) + .execute()) + +In the above example, we could safely invoke the upsert query as many times as +we wanted. The login count will be incremented atomically, the last login +column will be updated, and no duplicate rows will be created. + +For more information, see :py:meth:`Insert.on_conflict` and +:py:class:`OnConflict`. + Deleting records ---------------- From b03677476519339280bbd22e631b55ceec68dd40 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sat, 12 May 2018 01:38:30 -0500 Subject: [PATCH 23/43] Make wording a little less ambiguous --- docs/peewee/querying.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 8f72527bc..7f614b9df 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -302,8 +302,8 @@ Postgresql and SQLite (3.24.0 and newer) provide a different syntax that allows for more granular control over which constraint violation should trigger the conflict resolution, and what values should be updated or preserved. -Example of using :py:meth:`~Insert.on_conflict` with Postgresql to update -certain values: +Example of using :py:meth:`~Insert.on_conflict` to perform a Postgresql-style +upsert (or SQLite 3.24+): .. code-block:: python From 5eaa3d7747be65ba546a3e3ee9c4a1fe3603b5a1 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sat, 12 May 2018 01:50:54 -0500 Subject: [PATCH 24/43] Improve example and params for Insert.on_conflict doc. --- docs/peewee/api.rst | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 47251777d..4d32797ff 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -1459,7 +1459,7 @@ Query-builder :param str action: Action to take when resolving conflict. :param update: A dictionary mapping column to new value. - :param preserve: A list of columns whose values should be preserved. + :param preserve: A list of columns whose values should be preserved from the original INSERT. :param where: Expression to restrict the conflict resolution. :param conflict_target: Name of column or constraint to check. @@ -2051,10 +2051,42 @@ Query-builder Specify REPLACE conflict resolution strategy. - .. py:method:: on_conflict(*args, **kwargs) + .. py:method:: on_conflict([action=None[, update=None[, preserve=None[, where=None[, conflict_target=None]]]]]) - Specify an ON CONFLICT clause by populating a :py:class:`OnConflict` - object. + :param str action: Action to take when resolving conflict. If blank, + action is assumed to be "update". + :param update: A dictionary mapping column to new value. + :param preserve: A list of columns whose values should be preserved from the original INSERT. + :param where: Expression to restrict the conflict resolution. + :param conflict_target: Name of column or constraint to check. + + Specify the parameters for an :py:class:`OnConflict` clause to use for + conflict resolution. + + Example: + + .. code-block:: python + + class User(Model): + username = TextField(unique=True) + last_login = DateTimeField(null=True) + login_count = IntegerField() + + def log_user_in(username): + now = datetime.datetime.now() + + # INSERT a new row for the user with the current timestamp and + # login count set to 1. If the user already exists, then we + # will preserve the last_login value from the "insert()" clause + # and atomically increment the login-count. + userid = (User + .insert(username=username, last_login=now, login_count=1) + .on_conflict( + conflict_target=[User.username], + preserve=[User.last_login], + update={User.login_count: User.login_count + 1}) + .execute()) + return userid .. py:class:: Delete() From 3110e8ee08181b464caf613129a437cf86d950b2 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 14 May 2018 10:56:47 -0500 Subject: [PATCH 25/43] Add test for multiple self-join. --- tests/models.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/models.py b/tests/models.py index 2a55b6c28..e437dfa44 100644 --- a/tests/models.py +++ b/tests/models.py @@ -3256,3 +3256,70 @@ def test_lateral_join(self): {'username': 'u1', 'content': 'u1-t2'}, {'username': 'u2', 'content': 'u2-t3'}, {'username': 'u2', 'content': 'u2-t2'}]) + + +class Task(TestModel): + heading = ForeignKeyField('self', backref='tasks', null=True) + project = ForeignKeyField('self', backref='projects', null=True) + title = TextField() + type = IntegerField() + + PROJECT = 1 + HEADING = 2 + + +class TestMultiSelfJoin(ModelTestCase): + requires = [Task] + + def setUp(self): + super(TestMultiSelfJoin, self).setUp() + + with self.database.atomic(): + p_dev = Task.create(title='dev', type=Task.PROJECT) + p_p = Task.create(title='peewee', project=p_dev, type=Task.PROJECT) + p_h = Task.create(title='huey', project=p_dev, type=Task.PROJECT) + + heading_data = ( + ('peewee-1', p_p, 2), + ('peewee-2', p_p, 0), + ('huey-1', p_h, 1), + ('huey-2', p_h, 1)) + for title, proj, n_subtasks in heading_data: + t = Task.create(title=title, project=proj, type=Task.HEADING) + for i in range(n_subtasks): + Task.create(title='%s-%s' % (title, i + 1), project=proj, + heading=t, type=Task.HEADING) + + def test_multi_self_join(self): + Project = Task.alias() + Heading = Task.alias() + query = (Task + .select(Task, Project, Heading) + .join(Heading, JOIN.LEFT_OUTER, + on=(Task.heading == Heading.id).alias('heading')) + .switch(Task) + .join(Project, JOIN.LEFT_OUTER, + on=(Task.project == Project.id).alias('project')) + .order_by(Task.id)) + + with self.assertQueryCount(1): + accum = [] + for task in query: + h_title = task.heading.title if task.heading else None + p_title = task.project.title if task.project else None + accum.append((task.title, h_title, p_title)) + + self.assertEqual(accum, [ + # title - heading - project + ('dev', None, None), + ('peewee', None, 'dev'), + ('huey', None, 'dev'), + ('peewee-1', None, 'peewee'), + ('peewee-1-1', 'peewee-1', 'peewee'), + ('peewee-1-2', 'peewee-1', 'peewee'), + ('peewee-2', None, 'peewee'), + ('huey-1', None, 'huey'), + ('huey-1-1', 'huey-1', 'huey'), + ('huey-2', None, 'huey'), + ('huey-2-1', 'huey-2', 'huey'), + ]) From e0061e62308d72f7d6f56213a3fe7618af391209 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 14 May 2018 12:03:21 -0500 Subject: [PATCH 26/43] Make tests more resilient to variance in sqlite compile-time options. --- tests/database.py | 1 + tests/prefetch.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/database.py b/tests/database.py index 09d9a3692..59a51c70b 100644 --- a/tests/database.py +++ b/tests/database.py @@ -65,6 +65,7 @@ def test_pragmas_deferred(self): def test_pragmas_permanent(self): db = SqliteDatabase(':memory:') + db.execute_sql('pragma foreign_keys=0') self.assertEqual(db.foreign_keys, 0) db.pragma('foreign_keys', 1, True) diff --git a/tests/prefetch.py b/tests/prefetch.py index 83523a2c9..17dffe950 100644 --- a/tests/prefetch.py +++ b/tests/prefetch.py @@ -290,6 +290,7 @@ def test_prefetch_specific_model(self): @requires_models(Relationship) def test_multiple_foreign_keys(self): + self.database.pragma('foreign_keys', 0) Person.delete().execute() c, h, z = [Person.create(name=name) for name in ('charlie', 'huey', 'zaizee')] From f8d1cecc21024f402dfdc4ad0e3d2ef6e6116e26 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 14 May 2018 14:28:08 -0500 Subject: [PATCH 27/43] Unify sqlite timeout behavior, to use seconds as unit. --- CHANGELOG.md | 2 ++ docs/peewee/api.rst | 6 +++--- docs/peewee/sqlite_ext.rst | 8 ++++---- peewee.py | 12 +++++++----- playhouse/_sqlite_ext.pyx | 4 ++-- playhouse/apsw_ext.py | 4 ++-- playhouse/sqlite_ext.py | 3 ++- 7 files changed, 22 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a1cfce42..950b3a785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ https://github.com/coleifer/peewee/releases * Improved identifier quoting to allow specifying distinct open/close-quote characters. Enables adding support for MSSQL, for instance, which uses square brackets, e.g. `[table].[column]`. +* Unify timeout interfaces for SQLite databases (use seconds everywhere rather + than mixing seconds and milliseconds, which was confusing). [View commits](https://github.com/coleifer/peewee/compare/3.3.4...HEAD) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 4d32797ff..fb6754dd3 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -456,11 +456,11 @@ Database pass -.. py:class:: SqliteDatabase(database[, pragmas=None[, timeout=5000[, **kwargs]]]) +.. py:class:: SqliteDatabase(database[, pragmas=None[, timeout=5[, **kwargs]]]) :param list pragmas: A list of 2-tuples containing pragma key and value to set every time a connection is opened. - :param timeout: Set the busy-timeout on the SQLite driver (in milliseconds). + :param timeout: Set the busy-timeout on the SQLite driver (in seconds). Sqlite database implementation. :py:class:`SqliteDatabase` that provides some advanced features only offered by Sqlite. @@ -534,7 +534,7 @@ Database .. py:attribute:: timeout - Get or set the busy timeout (milliseconds). + Get or set the busy timeout (seconds). .. py:method:: register_aggregate(klass[, name=None[, num_params=-1]]) diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index b9785d32d..bf3fe28da 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -44,11 +44,11 @@ Instantiating a :py:class:`SqliteExtDatabase`: APIs ---- -.. py:class:: SqliteExtDatabase(database[, pragmas=None[, timeout=5000[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False]]]]]]]) +.. py:class:: SqliteExtDatabase(database[, pragmas=None[, timeout=5[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False]]]]]]]) :param list pragmas: A list of 2-tuples containing pragma key and value to set every time a connection is opened. - :param int timeout: Set the busy-timeout on the SQLite driver (in milliseconds). + :param timeout: Set the busy-timeout on the SQLite driver (in seconds). :param bool c_extensions: Declare that C extension speedups must/must-not be used. If set to ``True`` and the extension module is not available, will raise an :py:class:`ImproperlyConfigured` exception. @@ -60,11 +60,11 @@ APIs Extends :py:class:`SqliteDatabase` and inherits methods for declaring user-defined functions, pragmas, etc. -.. py:class:: CSqliteExtDatabase(database[, pragmas=None[, timeout=5000[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False[, replace_busy_handler=False]]]]]]]]) +.. py:class:: CSqliteExtDatabase(database[, pragmas=None[, timeout=5[, c_extensions=None[, rank_functions=True[, hash_functions=False[, regexp_function=False[, bloomfilter=False[, replace_busy_handler=False]]]]]]]]) :param list pragmas: A list of 2-tuples containing pragma key and value to set every time a connection is opened. - :param int timeout: Set the busy-timeout on the SQLite driver (in milliseconds). + :param timeout: Set the busy-timeout on the SQLite driver (in seconds). :param bool c_extensions: Declare that C extension speedups must/must-not be used. If set to ``True`` and the extension module is not available, will raise an :py:class:`ImproperlyConfigured` exception. diff --git a/peewee.py b/peewee.py index 929e1d987..7526b4379 100644 --- a/peewee.py +++ b/peewee.py @@ -2769,7 +2769,7 @@ def __init__(self, database, *args, **kwargs): self.register_function(_sqlite_date_part, 'date_part', 2) self.register_function(_sqlite_date_trunc, 'date_trunc', 2) - def init(self, database, pragmas=None, timeout=5000, **kwargs): + def init(self, database, pragmas=None, timeout=5, **kwargs): if pragmas is not None: self._pragmas = pragmas self._timeout = timeout @@ -2835,13 +2835,15 @@ def timeout(self): return self._timeout @timeout.setter - def timeout(self, milliseconds): - if self._timeout == milliseconds: + def timeout(self, seconds): + if self._timeout == seconds: return - self._timeout = milliseconds + self._timeout = seconds if not self.is_closed(): - self.execute_sql('PRAGMA busy_timeout=%d;' % milliseconds) + # PySQLite multiplies user timeout by 1000, but the unit of the + # timeout PRAGMA is actually milliseconds. + self.execute_sql('PRAGMA busy_timeout=%d;' % (seconds * 1000)) def _load_aggregates(self, conn): for name, (klass, num_params) in self._aggregates.items(): diff --git a/playhouse/_sqlite_ext.pyx b/playhouse/_sqlite_ext.pyx index 3a3c4edac..59f3175b2 100644 --- a/playhouse/_sqlite_ext.pyx +++ b/playhouse/_sqlite_ext.pyx @@ -1375,12 +1375,12 @@ cdef class ConnectionHelper(object): else: sqlite3_update_hook(self.conn.db, _update_callback, fn) - def set_busy_handler(self, timeout=5000): + def set_busy_handler(self, timeout=5): """ Replace the default busy handler with one that introduces some "jitter" into the amount of time delayed between checks. """ - cdef int n = timeout + cdef int n = timeout * 1000 sqlite3_busy_handler(self.conn.db, _aggressive_busy_handler, n) return True diff --git a/playhouse/apsw_ext.py b/playhouse/apsw_ext.py index 2ad2ab5b3..b757a5f53 100644 --- a/playhouse/apsw_ext.py +++ b/playhouse/apsw_ext.py @@ -41,8 +41,8 @@ def unregister_module(self, mod_name): def _connect(self): conn = apsw.Connection(self.database, **self.connect_params) - if self.timeout is not None: - conn.setbusytimeout(self.timeout) + if self._timeout is not None: + conn.setbusytimeout(self._timeout * 1000) try: self._add_conn_hooks(conn) except: diff --git a/playhouse/sqlite_ext.py b/playhouse/sqlite_ext.py index efd3fff89..a98d6a940 100644 --- a/playhouse/sqlite_ext.py +++ b/playhouse/sqlite_ext.py @@ -1049,7 +1049,8 @@ def _add_conn_hooks(self, conn): if self._update_hook is not None: self._conn_helper.set_update_hook(self._update_hook) if self._replace_busy_handler: - self._conn_helper.set_busy_handler(self.timeout or 5000) + timeout = self._timeout or 5 + self._conn_helper.set_busy_handler(timeout * 1000) def on_commit(self, fn): self._commit_hook = fn From 4fc7279343456ad12e33c0577705f3b2f512d8de Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 14 May 2018 14:32:21 -0500 Subject: [PATCH 28/43] Add tests for sqlite timeout semantics. --- tests/database.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/database.py b/tests/database.py index 59a51c70b..ec9530999 100644 --- a/tests/database.py +++ b/tests/database.py @@ -39,6 +39,20 @@ def test_pragmas(self): self.database.foreign_keys = 'off' self.assertEqual(self.database.foreign_keys, 0) + def test_timeout_semantics(self): + self.assertEqual(self.database.timeout, 5) + self.assertEqual(self.database.pragma('busy_timeout'), 5000) + + self.database.timeout = 2.5 + self.assertEqual(self.database.timeout, 2.5) + self.assertEqual(self.database.pragma('busy_timeout'), 2500) + + self.database.close() + self.database.connect() + + self.assertEqual(self.database.timeout, 2.5) + self.assertEqual(self.database.pragma('busy_timeout'), 2500) + def test_pragmas_deferred(self): pragmas = (('journal_mode', 'wal'),) db = SqliteDatabase(None, pragmas=pragmas) From d297d3fe172c806924ccb74a24fa498845353235 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 14 May 2018 15:07:05 -0500 Subject: [PATCH 29/43] Add custom field test. --- tests/fields.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/fields.py b/tests/fields.py index bf442c5f3..e3f12b8f2 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -618,3 +618,33 @@ def test_binary_uuid_field(self): u_db2 = UUIDModel.get(UUIDModel.bdata == uu) self.assertEqual(u_db2.id, u.id) + + +class ListField(TextField): + def db_value(self, value): + return ','.join(value) if value else '' + + def python_value(self, value): + return value.split(',') if value else [] + + +class Todo(TestModel): + content = TextField() + tags = ListField() + + +class TestCustomField(ModelTestCase): + requires = [Todo] + + def test_custom_field(self): + t1 = Todo.create(content='t1', tags=['t1-a', 't1-b']) + t2 = Todo.create(content='t2', tags=[]) + + t1_db = Todo.get(Todo.id == t1.id) + self.assertEqual(t1_db.tags, ['t1-a', 't1-b']) + + t2_db = Todo.get(Todo.id == t2.id) + self.assertEqual(t2_db.tags, []) + + t1_db = Todo.get(Todo.tags == Value(['t1-a', 't1-b'], unpack=False)) + self.assertEqual(t1_db.id, t1.id) From cf9aee906c15624ebe26548601160ee33ab22525 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 14 May 2018 15:27:28 -0500 Subject: [PATCH 30/43] Use AsIs rather than Value in test. --- tests/fields.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/fields.py b/tests/fields.py index e3f12b8f2..1a125198b 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -646,5 +646,8 @@ def test_custom_field(self): t2_db = Todo.get(Todo.id == t2.id) self.assertEqual(t2_db.tags, []) - t1_db = Todo.get(Todo.tags == Value(['t1-a', 't1-b'], unpack=False)) + t1_db = Todo.get(Todo.tags == AsIs(['t1-a', 't1-b'])) self.assertEqual(t1_db.id, t1.id) + + t2_db = Todo.get(Todo.tags == AsIs([])) + self.assertEqual(t2_db.id, t2.id) From 3694cea64f0ef5768fd7de8b227d61e485aba11a Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 15 May 2018 16:10:16 -0500 Subject: [PATCH 31/43] Tests for ATTACH and DETACH DATABASE SQLite functionality. --- docs/peewee/api.rst | 24 +++++++++++++ peewee.py | 23 ++++++++++++ tests/database.py | 87 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index fb6754dd3..68e11d2c0 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -734,6 +734,30 @@ Database db = SqliteExtDatabase('my_app.db') db.load_extension('closure') + .. py:method:: attach(filename, name) + + :param str filename: Database to attach (or ``:memory:`` for in-memory) + :param str name: Schema name for attached database. + :return: boolean indicating success + + Register another database file that will be attached to every database + connection. If the main database is currently connected, the new + database will be attached on the open connection. + + .. note:: + Databases that are attached using this method will be attached + every time a database connection is opened. + + .. py:method:: detach(name) + + :param str name: Schema name for attached database. + :return: boolean indicating success + + Unregister another database file that was attached previously with a + call to :py:meth:`~SqliteDatabase.attach`. If the main database is + currently connected, the attached database will be detached from the + open connection. + .. py:method:: transaction([lock_type=None]) :param str lock_type: Locking strategy: DEFERRED, IMMEDIATE, EXCLUSIVE. diff --git a/peewee.py b/peewee.py index 7526b4379..f9ff86ba2 100644 --- a/peewee.py +++ b/peewee.py @@ -2766,6 +2766,7 @@ def __init__(self, database, *args, **kwargs): self._functions = {} self._table_functions = [] self._extensions = set() + self._attached = {} self.register_function(_sqlite_date_part, 'date_part', 2) self.register_function(_sqlite_date_trunc, 'date_trunc', 2) @@ -2798,6 +2799,8 @@ def _add_conn_hooks(self, conn): table_function.register(conn) if self._extensions: self._load_extensions(conn) + for db_name, filename in self._attached.items(): + conn.execute('ATTACH DATABASE "%s" AS "%s"' % (filename, db_name)) def _set_pragmas(self, conn): if self._pragmas: @@ -2941,6 +2944,26 @@ def load_extension(self, extension): def unload_extension(self, extension): self._extensions.remove(extension) + def attach(self, filename, name): + if name in self._attached: + if self._attached[name] == filename: + return False + raise OperationalError('schema "%s" already attached.' % name) + + self._attached[name] = filename + if not self.is_closed(): + self.execute_sql('ATTACH DATABASE "%s" AS "%s"' % (filename, name)) + return True + + def detach(self, name): + if name not in self._attached: + return False + + del self._attached[name] + if not self.is_closed(): + self.execute_sql('DETACH DATABASE "%s"' % name) + return True + def atomic(self, lock_type=None): return _atomic(self, lock_type=lock_type) diff --git a/tests/database.py b/tests/database.py index ec9530999..ea342902c 100644 --- a/tests/database.py +++ b/tests/database.py @@ -25,6 +25,93 @@ from .base_models import User +class Data(TestModel): + key = TextField() + value = TextField() + + class Meta: + schema = 'main' + + +class TestAttachDatabase(ModelTestCase): + database = db_loader('sqlite3') + requires = [Data] + + def test_attach(self): + database = self.database + Data.create(key='k1', value='v1') + Data.create(key='k2', value='v2') + + # Attach an in-memory cache database. + database.attach(':memory:', 'cache') + + # Clone data into the in-memory cache. + class CacheData(Data): + class Meta: + schema = 'cache' + + CacheData.create_table(safe=False) + (CacheData + .insert_from(Data.select(), fields=[Data.id, Data.key, Data.value]) + .execute()) + + # Update the source data. + query = Data.update({Data.value: Data.value + '-x'}) + self.assertEqual(query.execute(), 2) + + # Verify the source data was updated. + query = Data.select(Data.key, Data.value).order_by(Data.key) + self.assertSQL(query, ( + 'SELECT "t1"."key", "t1"."value" ' + 'FROM "main"."data" AS "t1" ' + 'ORDER BY "t1"."key"'), []) + self.assertEqual([v for k, v in query.tuples()], ['v1-x', 'v2-x']) + + # Verify the cached data reflects the original data, pre-update. + query = (CacheData + .select(CacheData.key, CacheData.value) + .order_by(CacheData.key)) + self.assertSQL(query, ( + 'SELECT "t1"."key", "t1"."value" ' + 'FROM "cache"."cachedata" AS "t1" ' + 'ORDER BY "t1"."key"'), []) + self.assertEqual([v for k, v in query.tuples()], ['v1', 'v2']) + + database.close() + + # On re-connecting, the in-memory database will re-attached. + database.connect() + + # Cache-Data table does not exist. + curs = database.execute_sql('select * from cache.sqlite_master;') + self.assertEqual(curs.fetchall(), []) + + # Because it's in-memory, the table needs to be re-created. + CacheData.create_table(safe=False) + self.assertEqual(CacheData.select().count(), 0) + + # Original data is still there. + self.assertEqual(Data.select().count(), 2) + + def test_attach_detach(self): + database = self.database + Data.create(key='k1', value='v1') + Data.create(key='k2', value='v2') + + # Attach an in-memory cache database. + database.attach(':memory:', 'cache') + curs = database.execute_sql('select * from cache.sqlite_master') + self.assertEqual(curs.fetchall(), []) + + self.assertFalse(database.attach(':memory:', 'cache')) + self.assertRaises(OperationalError, database.attach, 'foo.db', 'cache') + + self.assertTrue(database.detach('cache')) + self.assertFalse(database.detach('cache')) + self.assertRaises(OperationalError, database.execute_sql, + 'select * from cache.sqlite_master') + + class TestDatabase(DatabaseTestCase): database = db_loader('sqlite3') From 1dd013d13dc499119e0df4ea7da296856e0be346 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 15 May 2018 16:20:08 -0500 Subject: [PATCH 32/43] Re-org test order. --- tests/database.py | 174 +++++++++++++++++++++++----------------------- 1 file changed, 87 insertions(+), 87 deletions(-) diff --git a/tests/database.py b/tests/database.py index ea342902c..17a6c8834 100644 --- a/tests/database.py +++ b/tests/database.py @@ -25,93 +25,6 @@ from .base_models import User -class Data(TestModel): - key = TextField() - value = TextField() - - class Meta: - schema = 'main' - - -class TestAttachDatabase(ModelTestCase): - database = db_loader('sqlite3') - requires = [Data] - - def test_attach(self): - database = self.database - Data.create(key='k1', value='v1') - Data.create(key='k2', value='v2') - - # Attach an in-memory cache database. - database.attach(':memory:', 'cache') - - # Clone data into the in-memory cache. - class CacheData(Data): - class Meta: - schema = 'cache' - - CacheData.create_table(safe=False) - (CacheData - .insert_from(Data.select(), fields=[Data.id, Data.key, Data.value]) - .execute()) - - # Update the source data. - query = Data.update({Data.value: Data.value + '-x'}) - self.assertEqual(query.execute(), 2) - - # Verify the source data was updated. - query = Data.select(Data.key, Data.value).order_by(Data.key) - self.assertSQL(query, ( - 'SELECT "t1"."key", "t1"."value" ' - 'FROM "main"."data" AS "t1" ' - 'ORDER BY "t1"."key"'), []) - self.assertEqual([v for k, v in query.tuples()], ['v1-x', 'v2-x']) - - # Verify the cached data reflects the original data, pre-update. - query = (CacheData - .select(CacheData.key, CacheData.value) - .order_by(CacheData.key)) - self.assertSQL(query, ( - 'SELECT "t1"."key", "t1"."value" ' - 'FROM "cache"."cachedata" AS "t1" ' - 'ORDER BY "t1"."key"'), []) - self.assertEqual([v for k, v in query.tuples()], ['v1', 'v2']) - - database.close() - - # On re-connecting, the in-memory database will re-attached. - database.connect() - - # Cache-Data table does not exist. - curs = database.execute_sql('select * from cache.sqlite_master;') - self.assertEqual(curs.fetchall(), []) - - # Because it's in-memory, the table needs to be re-created. - CacheData.create_table(safe=False) - self.assertEqual(CacheData.select().count(), 0) - - # Original data is still there. - self.assertEqual(Data.select().count(), 2) - - def test_attach_detach(self): - database = self.database - Data.create(key='k1', value='v1') - Data.create(key='k2', value='v2') - - # Attach an in-memory cache database. - database.attach(':memory:', 'cache') - curs = database.execute_sql('select * from cache.sqlite_master') - self.assertEqual(curs.fetchall(), []) - - self.assertFalse(database.attach(':memory:', 'cache')) - self.assertRaises(OperationalError, database.attach, 'foo.db', 'cache') - - self.assertTrue(database.detach('cache')) - self.assertFalse(database.detach('cache')) - self.assertRaises(OperationalError, database.execute_sql, - 'select * from cache.sqlite_master') - - class TestDatabase(DatabaseTestCase): database = db_loader('sqlite3') @@ -578,3 +491,90 @@ class Tweet(BaseModel): self.assertFalse(User._meta.database.is_closed()) self.assertFalse(Tweet._meta.database.is_closed()) sqlite_db.close() + + +class Data(TestModel): + key = TextField() + value = TextField() + + class Meta: + schema = 'main' + + +class TestAttachDatabase(ModelTestCase): + database = db_loader('sqlite3') + requires = [Data] + + def test_attach(self): + database = self.database + Data.create(key='k1', value='v1') + Data.create(key='k2', value='v2') + + # Attach an in-memory cache database. + database.attach(':memory:', 'cache') + + # Clone data into the in-memory cache. + class CacheData(Data): + class Meta: + schema = 'cache' + + CacheData.create_table(safe=False) + (CacheData + .insert_from(Data.select(), fields=[Data.id, Data.key, Data.value]) + .execute()) + + # Update the source data. + query = Data.update({Data.value: Data.value + '-x'}) + self.assertEqual(query.execute(), 2) + + # Verify the source data was updated. + query = Data.select(Data.key, Data.value).order_by(Data.key) + self.assertSQL(query, ( + 'SELECT "t1"."key", "t1"."value" ' + 'FROM "main"."data" AS "t1" ' + 'ORDER BY "t1"."key"'), []) + self.assertEqual([v for k, v in query.tuples()], ['v1-x', 'v2-x']) + + # Verify the cached data reflects the original data, pre-update. + query = (CacheData + .select(CacheData.key, CacheData.value) + .order_by(CacheData.key)) + self.assertSQL(query, ( + 'SELECT "t1"."key", "t1"."value" ' + 'FROM "cache"."cachedata" AS "t1" ' + 'ORDER BY "t1"."key"'), []) + self.assertEqual([v for k, v in query.tuples()], ['v1', 'v2']) + + database.close() + + # On re-connecting, the in-memory database will re-attached. + database.connect() + + # Cache-Data table does not exist. + curs = database.execute_sql('select * from cache.sqlite_master;') + self.assertEqual(curs.fetchall(), []) + + # Because it's in-memory, the table needs to be re-created. + CacheData.create_table(safe=False) + self.assertEqual(CacheData.select().count(), 0) + + # Original data is still there. + self.assertEqual(Data.select().count(), 2) + + def test_attach_detach(self): + database = self.database + Data.create(key='k1', value='v1') + Data.create(key='k2', value='v2') + + # Attach an in-memory cache database. + database.attach(':memory:', 'cache') + curs = database.execute_sql('select * from cache.sqlite_master') + self.assertEqual(curs.fetchall(), []) + + self.assertFalse(database.attach(':memory:', 'cache')) + self.assertRaises(OperationalError, database.attach, 'foo.db', 'cache') + + self.assertTrue(database.detach('cache')) + self.assertFalse(database.detach('cache')) + self.assertRaises(OperationalError, database.execute_sql, + 'select * from cache.sqlite_master') From ab32cff3092a4bcfda7f852bd90e8b38ae68bf09 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 17 May 2018 10:26:56 -0500 Subject: [PATCH 33/43] Include schema information when querying sqlite metadata. --- peewee.py | 33 +++++++++++++++++++++------------ tests/database.py | 21 +++++++++++++++++++++ tests/migrations.py | 13 +++++++------ 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/peewee.py b/peewee.py index f9ff86ba2..fab8e04ea 100644 --- a/peewee.py +++ b/peewee.py @@ -2790,6 +2790,8 @@ def _connect(self): return conn def _add_conn_hooks(self, conn): + for db_name, filename in self._attached.items(): + conn.execute('ATTACH DATABASE "%s" AS "%s"' % (filename, db_name)) self._set_pragmas(conn) self._load_aggregates(conn) self._load_collations(conn) @@ -2799,8 +2801,6 @@ def _add_conn_hooks(self, conn): table_function.register(conn) if self._extensions: self._load_extensions(conn) - for db_name, filename in self._attached.items(): - conn.execute('ATTACH DATABASE "%s" AS "%s"' % (filename, db_name)) def _set_pragmas(self, conn): if self._pragmas: @@ -2809,7 +2809,9 @@ def _set_pragmas(self, conn): cursor.execute('PRAGMA %s = %s;' % (pragma, value)) cursor.close() - def pragma(self, key, value=SENTINEL, permanent=False): + def pragma(self, key, value=SENTINEL, permanent=False, schema=None): + if schema is not None: + key = '"%s".%s' % (schema, key) sql = 'PRAGMA %s' % key if value is not SENTINEL: sql += ' = %s' % (value or 0) @@ -2975,19 +2977,22 @@ def begin(self, lock_type=None): self.execute_sql(statement, commit=False) def get_tables(self, schema=None): - cursor = self.execute_sql('SELECT name FROM sqlite_master WHERE ' - 'type = ? ORDER BY name;', ('table',)) + schema = schema or 'main' + cursor = self.execute_sql('SELECT name FROM "%s".sqlite_master WHERE ' + 'type=? ORDER BY name' % schema, ('table',)) return [row for row, in cursor.fetchall()] def get_indexes(self, table, schema=None): - query = ('SELECT name, sql FROM sqlite_master ' - 'WHERE tbl_name = ? AND type = ? ORDER BY name') + schema = schema or 'main' + query = ('SELECT name, sql FROM "%s".sqlite_master ' + 'WHERE tbl_name = ? AND type = ? ORDER BY name') % schema cursor = self.execute_sql(query, (table, 'index')) index_to_sql = dict(cursor.fetchall()) # Determine which indexes have a unique constraint. unique_indexes = set() - cursor = self.execute_sql('PRAGMA index_list("%s")' % table) + cursor = self.execute_sql('PRAGMA "%s".index_list("%s")' % + (schema, table)) for row in cursor.fetchall(): name = row[1] is_unique = int(row[2]) == 1 @@ -2997,7 +3002,8 @@ def get_indexes(self, table, schema=None): # Retrieve the indexed columns. index_columns = {} for index_name in sorted(index_to_sql): - cursor = self.execute_sql('PRAGMA index_info("%s")' % index_name) + cursor = self.execute_sql('PRAGMA "%s".index_info("%s")' % + (schema, index_name)) index_columns[index_name] = [row[2] for row in cursor.fetchall()] return [ @@ -3010,16 +3016,19 @@ def get_indexes(self, table, schema=None): for name in sorted(index_to_sql)] def get_columns(self, table, schema=None): - cursor = self.execute_sql('PRAGMA table_info("%s")' % table) + cursor = self.execute_sql('PRAGMA "%s".table_info("%s")' % + (schema or 'main', table)) return [ColumnMetadata(r[1], r[2], not r[3], bool(r[5]), table, r[4]) for r in cursor.fetchall()] def get_primary_keys(self, table, schema=None): - cursor = self.execute_sql('PRAGMA table_info("%s")' % table) + cursor = self.execute_sql('PRAGMA "%s".table_info("%s")' % + (schema or 'main', table)) return [row[1] for row in filter(lambda r: r[-1], cursor.fetchall())] def get_foreign_keys(self, table, schema=None): - cursor = self.execute_sql('PRAGMA foreign_key_list("%s")' % table) + cursor = self.execute_sql('PRAGMA "%s".foreign_key_list("%s")' % + (schema or 'main', table)) return [ForeignKeyMetadata(row[3], row[2], row[4], table) for row in cursor.fetchall()] diff --git a/tests/database.py b/tests/database.py index 17a6c8834..739ef7908 100644 --- a/tests/database.py +++ b/tests/database.py @@ -518,7 +518,10 @@ class CacheData(Data): class Meta: schema = 'cache' + self.assertFalse(CacheData.table_exists()) CacheData.create_table(safe=False) + self.assertTrue(CacheData.table_exists()) + (CacheData .insert_from(Data.select(), fields=[Data.id, Data.key, Data.value]) .execute()) @@ -551,6 +554,9 @@ class Meta: database.connect() # Cache-Data table does not exist. + self.assertFalse(CacheData.table_exists()) + + # Double-check the sqlite master table. curs = database.execute_sql('select * from cache.sqlite_master;') self.assertEqual(curs.fetchall(), []) @@ -578,3 +584,18 @@ def test_attach_detach(self): self.assertFalse(database.detach('cache')) self.assertRaises(OperationalError, database.execute_sql, 'select * from cache.sqlite_master') + + def test_sqlite_schema_support(self): + class CacheData(Data): + class Meta: + schema = 'cache' + + # Attach an in-memory cache database and create the cache table. + self.database.attach(':memory:', 'cache') + CacheData.create_table() + + tables = self.database.get_tables() + self.assertEqual(tables, ['data']) + + tables = self.database.get_tables(schema='cache') + self.assertEqual(tables, ['cachedata']) diff --git a/tests/migrations.py b/tests/migrations.py index febda9b0b..a8bfa339e 100644 --- a/tests/migrations.py +++ b/tests/migrations.py @@ -597,7 +597,7 @@ def test_index_preservation(self): queries = [x.msg for x in self.history] self.assertEqual(queries, [ # Get all the columns. - ('PRAGMA table_info("indexmodel")', None), + ('PRAGMA "main".table_info("indexmodel")', None), # Get the table definition. ('select name, sql from sqlite_master ' @@ -605,15 +605,16 @@ def test_index_preservation(self): ['table', 'indexmodel']), # Get the indexes and indexed columns for the table. - ('SELECT name, sql FROM sqlite_master ' + ('SELECT name, sql FROM "main".sqlite_master ' 'WHERE tbl_name = ? AND type = ? ORDER BY name', ('indexmodel', 'index')), - ('PRAGMA index_list("indexmodel")', None), - ('PRAGMA index_info("indexmodel_data")', None), - ('PRAGMA index_info("indexmodel_first_name_last_name")', None), + ('PRAGMA "main".index_list("indexmodel")', None), + ('PRAGMA "main".index_info("indexmodel_data")', None), + ('PRAGMA "main".index_info("indexmodel_first_name_last_name")', + None), # Get foreign keys. - ('PRAGMA foreign_key_list("indexmodel")', None), + ('PRAGMA "main".foreign_key_list("indexmodel")', None), # Drop any temporary table, if it exists. ('DROP TABLE IF EXISTS "indexmodel__tmp__"', []), From 1bdc57c683418bce9387e7da692f088cb6558308 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 17 May 2018 10:41:08 -0500 Subject: [PATCH 34/43] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 950b3a785..635284cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ https://github.com/coleifer/peewee/releases brackets, e.g. `[table].[column]`. * Unify timeout interfaces for SQLite databases (use seconds everywhere rather than mixing seconds and milliseconds, which was confusing). +* Added `attach()` and `detach()` methods to SQLite database, making it + possible to attach additional databases (e.g. an in-memory cache db). +* Support the `schema` parameter for SQLite database introspection methods. [View commits](https://github.com/coleifer/peewee/compare/3.3.4...HEAD) From 024faa95c13798b427ba175ae78727ceac9af425 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 18 May 2018 09:04:30 -0500 Subject: [PATCH 35/43] Clean up and normalize some cache filling. --- peewee.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/peewee.py b/peewee.py index fab8e04ea..c9543107e 100644 --- a/peewee.py +++ b/peewee.py @@ -3501,8 +3501,6 @@ def __iter__(self): def __getitem__(self, item): if isinstance(item, slice): - # TODO: getslice - start = item.start stop = item.stop if stop is None or stop < 0: self.fill_cache() @@ -3546,7 +3544,7 @@ def iterator(self): while True: yield self.iterate(False) - def fill_cache(self, n=0.): + def fill_cache(self, n=0): n = n or float('Inf') if n < 0: raise ValueError('Negative values are not supported.') From 909ed5ff80db0888fa263f7a4521896e27ff8044 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 18 May 2018 11:31:08 -0500 Subject: [PATCH 36/43] Make sqlite regexp function case-sensitive. --- playhouse/sqlite_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playhouse/sqlite_ext.py b/playhouse/sqlite_ext.py index a98d6a940..868cda630 100644 --- a/playhouse/sqlite_ext.py +++ b/playhouse/sqlite_ext.py @@ -924,7 +924,7 @@ def delete_by_id(cls, pk): OP.MATCH = 'MATCH' def _sqlite_regexp(regex, value): - return re.search(regex, value, re.I) is not None + return re.search(regex, value) is not None class SqliteExtDatabase(SqliteDatabase): From f631f222fe6b0c5cb68de99dede98d03454d3e3f Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 18 May 2018 11:51:50 -0500 Subject: [PATCH 37/43] Add tests and ops for IREGEXP (supported only in mysql/postgresql) --- peewee.py | 9 ++++++++- tests/__init__.py | 1 + tests/expressions.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 tests/expressions.py diff --git a/peewee.py b/peewee.py index c9543107e..8ea9a13d0 100644 --- a/peewee.py +++ b/peewee.py @@ -266,6 +266,7 @@ def __add__(self, rhs): d = attrdict(self); d.update(rhs); return d ILIKE='ILIKE', BETWEEN='BETWEEN', REGEXP='REGEXP', + IREGEXP='IREGEXP', CONCAT='||', BITWISE_NEGATION='~') @@ -1026,6 +1027,10 @@ def between(self, lo, hi): return Expression(self, OP.BETWEEN, NodeList((lo, SQL('AND'), hi))) def concat(self, rhs): return StringExpression(self, OP.CONCAT, rhs) + def regexp(self, rhs): + return Expression(self, OP.REGEXP, rhs) + def iregexp(self, rhs): + return Expression(self, OP.IREGEXP, rhs) def __getitem__(self, item): if isinstance(item, slice): if item.start is None or item.stop is None: @@ -3083,7 +3088,7 @@ class PostgresqlDatabase(Database): 'DOUBLE': 'DOUBLE PRECISION', 'UUID': 'UUID', 'UUIDB': 'BYTEA'} - operations = {'REGEXP': '~'} + operations = {'REGEXP': '~', 'IREGEXP': '~*'} param = '%s' commit_select = True @@ -3250,6 +3255,8 @@ class MySQLDatabase(Database): operations = { 'LIKE': 'LIKE BINARY', 'ILIKE': 'LIKE', + 'REGEXP': 'REGEXP BINARY', + 'IREGEXP': 'REGEXP', 'XOR': 'XOR'} param = '%s' quote = '``' diff --git a/tests/__init__.py b/tests/__init__.py index 285193043..6e447669c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,7 @@ # Core modules. from .database import * +from .expressions import * from .fields import * from .keys import * from .manytomany import * diff --git a/tests/expressions.py b/tests/expressions.py new file mode 100644 index 000000000..49769de2a --- /dev/null +++ b/tests/expressions.py @@ -0,0 +1,44 @@ +from peewee import * + +from .base import skip_if +from .base import IS_SQLITE +from .base import ModelTestCase +from .base import TestModel + + +class Person(TestModel): + name = CharField() + + +class BaseNamesTest(ModelTestCase): + requires = [Person] + + def assertNames(self, exp, x): + query = Person.select().where(exp).order_by(Person.name) + self.assertEqual([p.name for p in query], x) + + + +class TestRegexp(BaseNamesTest): + @skip_if(IS_SQLITE) + def test_regexp_iregexp(self): + people = [Person.create(name=name) for name in ('n1', 'n2', 'n3')] + + self.assertNames(Person.name.regexp('n[1,3]'), ['n1', 'n3']) + self.assertNames(Person.name.regexp('N[1,3]'), []) + self.assertNames(Person.name.iregexp('n[1,3]'), ['n1', 'n3']) + self.assertNames(Person.name.iregexp('N[1,3]'), ['n1', 'n3']) + + +class TestContains(BaseNamesTest): + def test_contains_startswith_endswith(self): + people = [Person.create(name=n) for n in ('huey', 'mickey', 'zaizee')] + + self.assertNames(Person.name.contains('ey'), ['huey', 'mickey']) + self.assertNames(Person.name.contains('EY'), ['huey', 'mickey']) + + self.assertNames(Person.name.startswith('m'), ['mickey']) + self.assertNames(Person.name.startswith('M'), ['mickey']) + + self.assertNames(Person.name.endswith('ey'), ['huey', 'mickey']) + self.assertNames(Person.name.endswith('EY'), ['huey', 'mickey']) From d4b3429755a6467c8eabce34853122aea6cc369e Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 18 May 2018 11:54:56 -0500 Subject: [PATCH 38/43] Docs for iregexp. --- docs/peewee/querying.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 7f614b9df..2088239b4 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -1152,7 +1152,8 @@ Method Meaning ``.startswith(prefix)`` Search for values beginning with ``prefix``. ``.endswith(suffix)`` Search for values ending with ``suffix``. ``.between(low, high)`` Search for values between ``low`` and ``high``. -``.regexp(exp)`` Regular expression match. +``.regexp(exp)`` Regular expression match (case-sensitive). +``.iregexp(exp)`` Regular expression match (case-insensitive). ``.bin_and(value)`` Binary AND. ``.bin_or(value)`` Binary OR. ``.in_(value)`` IN lookup (identical to ``<<``). From 243f6d69deaeb71a3d0986ab15529e5066c64dc8 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 18 May 2018 15:10:06 -0500 Subject: [PATCH 39/43] Add test and change BareField DDL to support constraints --- peewee.py | 7 ++++-- tests/sqlite.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/peewee.py b/peewee.py index 8ea9a13d0..34e25685a 100644 --- a/peewee.py +++ b/peewee.py @@ -1042,6 +1042,9 @@ def __getitem__(self, item): def distinct(self): return NodeList((SQL('DISTINCT'), self)) + def collate(self, collation): + return NodeList((self, SQL('COLLATE %s' % collation))) + def get_sort_key(self, ctx): return () @@ -4246,8 +4249,8 @@ def __init__(self, adapt=None, *args, **kwargs): if adapt is not None: self.adapt = adapt - def ddl(self, ctx): - return Entity(self.column_name) + def ddl_datatype(self, ctx): + return class ForeignKeyField(Field): diff --git a/tests/sqlite.py b/tests/sqlite.py index 30a672136..f337049e2 100644 --- a/tests/sqlite.py +++ b/tests/sqlite.py @@ -10,6 +10,7 @@ from .base import IS_SQLITE_9 from .base import ModelTestCase from .base import TestModel +from .base import get_in_memory_db from .base import requires_models from .base import skip_if from .base import skip_unless @@ -1702,3 +1703,67 @@ def assertValues(query, expected): fq2 = fq1.where(CalendarDay.value < 30) assertValues(fq2, range(25, 29)) + + +class Datum(TestModel): + a = BareField() + b = BareField(collation='BINARY') + c = BareField(collation='RTRIM') + d = BareField(collation='NOCASE') + + +class TestCollatedFieldDefinitions(ModelTestCase): + database = get_in_memory_db() + requires = [Datum] + + def test_collated_fields(self): + rows = ( + (1, 'abc', 'abc', 'abc ', 'abc'), + (2, 'abc', 'abc', 'abc', 'ABC'), + (3, 'abc', 'abc', 'abc ', 'Abc'), + (4, 'abc', 'abc ', 'ABC', 'abc')) + for pk, a, b, c, d in rows: + Datum.create(id=pk, a=a, b=b, c=c, d=d) + + def assertC(query, expected): + self.assertEqual([r.id for r in query], expected) + + base = Datum.select().order_by(Datum.id) + + # Text comparison a=b is performed using binary collating sequence. + assertC(base.where(Datum.a == Datum.b), [1, 2, 3]) + + # Text comparison a=b is performed using the RTRIM collating sequence. + assertC(base.where(Datum.a == Datum.b.collate('RTRIM')), [1, 2, 3, 4]) + + # Text comparison d=a is performed using the NOCASE collating sequence. + assertC(base.where(Datum.d == Datum.a), [1, 2, 3, 4]) + + # Text comparison a=d is performed using the BINARY collating sequence. + assertC(base.where(Datum.a == Datum.d), [1, 4]) + + # Text comparison 'abc'=c is performed using RTRIM collating sequence. + assertC(base.where('abc' == Datum.c), [1, 2, 3]) + + # Text comparison c='abc' is performed using RTRIM collating sequence. + assertC(base.where(Datum.c == 'abc'), [1, 2, 3]) + + # Grouping is performed using the NOCASE collating sequence (Values + # 'abc', 'ABC', and 'Abc' are placed in the same group). + query = Datum.select(fn.COUNT(Datum.id)).group_by(Datum.d) + self.assertEqual(query.scalar(), 4) + + # Grouping is performed using the BINARY collating sequence. 'abc' and + # 'ABC' and 'Abc' form different groups. + query = Datum.select(fn.COUNT(Datum.id)).group_by(Datum.d.concat('')) + self.assertEqual([r[0] for r in query.tuples()], [1, 1, 2]) + + # Sorting or column c is performed using the RTRIM collating sequence. + assertC(base.order_by(Datum.c, Datum.id), [4, 1, 2, 3]) + + # Sorting of (c||'') is performed using the BINARY collating sequence. + assertC(base.order_by(Datum.c.concat(''), Datum.id), [4, 2, 3, 1]) + + # Sorting of column c is performed using the NOCASE collating sequence. + assertC(base.order_by(Datum.c.collate('NOCASE'), Datum.id), + [2, 4, 3, 1]) From c16740d56528222944f3c0a683895092fadfb985 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 18 May 2018 15:13:44 -0500 Subject: [PATCH 40/43] changes --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 635284cc5..5ed882e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,21 @@ https://github.com/coleifer/peewee/releases ## master +**Backwards-incompatible changes** + +* The `regexp()` operation is now case-sensitive for MySQL and Postgres. To + perform case-insensitive regexp operations, use `iregexp()`. +* The SQLite `BareField()` field-type now supports all column constraints + *except* specifying the data-type. Previously it silently ignored any column + constraints. * LIMIT and OFFSET parameters are now treated as parameterized values instead of literals. +* The `schema` parameter for SQLite database introspection methods is no longer + ignored by default. The schema corresponds to the name given to an attached + database. + +**New features and other changes** + * SQLite backup interface supports specifying page-counts and a user-defined progress handler. * GIL is released when doing backups or during SQLite busy timeouts (when using @@ -21,7 +34,6 @@ https://github.com/coleifer/peewee/releases than mixing seconds and milliseconds, which was confusing). * Added `attach()` and `detach()` methods to SQLite database, making it possible to attach additional databases (e.g. an in-memory cache db). -* Support the `schema` parameter for SQLite database introspection methods. [View commits](https://github.com/coleifer/peewee/compare/3.3.4...HEAD) From e0c750b2c5200a88d4a78f3480f4b88f6b8e0b40 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sat, 19 May 2018 09:24:19 -0500 Subject: [PATCH 41/43] Fix doc reference. --- docs/peewee/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 2088239b4..94568be94 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -273,7 +273,7 @@ prior to 3.24.0 and MySQL, Peewee offers the :py:meth:`~Model.replace`, which allows you to insert a record or, in the event of a constraint violation, replace the existing record. -Example of using `~Model.replace` and :py:meth:`~Insert.on_conflict_replace`: +Example of using :py:meth:`~Model.replace` and :py:meth:`~Insert.on_conflict_replace`: .. code-block:: python From ad94047fa0a6cd8569bd1f725d50b606dd8f4b21 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sat, 19 May 2018 22:48:01 -0500 Subject: [PATCH 42/43] Add `field_kwargs` parameter to ArrayField initializer. Fixes #1608. --- CHANGELOG.md | 2 ++ docs/peewee/playhouse.rst | 3 ++- playhouse/postgres_ext.py | 6 +++--- tests/postgres.py | 16 ++++++++++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed882e90..ea36661ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ https://github.com/coleifer/peewee/releases * The `schema` parameter for SQLite database introspection methods is no longer ignored by default. The schema corresponds to the name given to an attached database. +* `ArrayField` now accepts a new parameter `field_kwargs`, which is used to + pass information to the array field's `field_class` initializer. **New features and other changes** diff --git a/docs/peewee/playhouse.rst b/docs/peewee/playhouse.rst index cb17fcf3f..82b5d30e1 100644 --- a/docs/peewee/playhouse.rst +++ b/docs/peewee/playhouse.rst @@ -1090,9 +1090,10 @@ postgres_ext API notes .. _pgarrays: -.. py:class:: ArrayField([field_class=IntegerField[, dimensions=1[, convert_values=False]]]) +.. py:class:: ArrayField([field_class=IntegerField[, field_kwargs=None[, dimensions=1[, convert_values=False]]]]) :param field_class: a subclass of :py:class:`Field`, e.g. :py:class:`IntegerField`. + :param dict field_kwargs: arguments to initialize ``field_class``. :param int dimensions: dimensions of array. :param bool convert_values: apply ``field_class`` value conversion to array data. diff --git a/playhouse/postgres_ext.py b/playhouse/postgres_ext.py index 89c998ab2..a65135f7d 100644 --- a/playhouse/postgres_ext.py +++ b/playhouse/postgres_ext.py @@ -147,9 +147,9 @@ class ArrayField(IndexedFieldMixin, Field): default_index_type = 'GIN' passthrough = True - def __init__(self, field_class=IntegerField, dimensions=1, - convert_values=False, *args, **kwargs): - self.__field = field_class(*args, **kwargs) + def __init__(self, field_class=IntegerField, field_kwargs=None, + dimensions=1, convert_values=False, *args, **kwargs): + self.__field = field_class(**(field_kwargs or {})) self.dimensions = dimensions self.convert_values = convert_values self.field_type = self.__field.field_type diff --git a/tests/postgres.py b/tests/postgres.py index 3bfdbdffb..462ba79cf 100644 --- a/tests/postgres.py +++ b/tests/postgres.py @@ -1,5 +1,6 @@ #coding:utf-8 import datetime +from decimal import Decimal as Dc from types import MethodType from peewee import * @@ -9,6 +10,7 @@ from .base import ModelTestCase from .base import TestModel from .base import db_loader +from .base import requires_models from .base import skip_unless from .base_models import Register @@ -32,6 +34,10 @@ class ArrayTSModel(TestModel): timestamps = ArrayField(TimestampField, convert_values=True) +class DecimalArray(TestModel): + values = ArrayField(DecimalField, field_kwargs={'decimal_places': 1}) + + class FTSModel(TestModel): title = CharField() data = TextField() @@ -306,6 +312,16 @@ def test_array_index_slice(self): row = AM.select(I[1:2][0].alias('ints')).dicts().get() self.assertEqual(row['ints'], [[3], [5]]) + @requires_models(DecimalArray) + def test_field_kwargs(self): + vl1, vl2 = [Dc('3.1'), Dc('1.3')], [Dc('3.14'), Dc('1')] + da1, da2 = [DecimalArray.create(values=vl) for vl in (vl1, vl2)] + + da1_db = DecimalArray.get(DecimalArray.id == da1.id) + da2_db = DecimalArray.get(DecimalArray.id == da2.id) + self.assertEqual(da1_db.values, [Dc('3.1'), Dc('1.3')]) + self.assertEqual(da2_db.values, [Dc('3.1'), Dc('1.0')]) + class TestArrayFieldConvertValues(ModelTestCase): database = db From e582871f4257dbbed3dc23d052fbeee144193458 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sat, 19 May 2018 22:54:51 -0500 Subject: [PATCH 43/43] 3.4.0 --- CHANGELOG.md | 6 +++++- peewee.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea36661ec..e6aa86cd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ https://github.com/coleifer/peewee/releases ## master +[View commits](https://github.com/coleifer/peewee/compare/3.4.0...HEAD) + +## 3.4.0 + **Backwards-incompatible changes** * The `regexp()` operation is now case-sensitive for MySQL and Postgres. To @@ -37,7 +41,7 @@ https://github.com/coleifer/peewee/releases * Added `attach()` and `detach()` methods to SQLite database, making it possible to attach additional databases (e.g. an in-memory cache db). -[View commits](https://github.com/coleifer/peewee/compare/3.3.4...HEAD) +[View commits](https://github.com/coleifer/peewee/compare/3.3.4...3.4.0) ## 3.3.4 diff --git a/peewee.py b/peewee.py index 34e25685a..befa99da4 100644 --- a/peewee.py +++ b/peewee.py @@ -57,7 +57,7 @@ mysql = None -__version__ = '3.3.4' +__version__ = '3.4.0' __all__ = [ 'AsIs', 'AutoField',