From 53bf97f7e5cd2e3978ae91dfa4779f669742e58e Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 29 Jan 2018 13:01:50 -0600 Subject: [PATCH 1/9] Update changelog. --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da93999da..7379f9d3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ releases, visit GitHub: https://github.com/coleifer/peewee/releases +## 3.0.2 + +Ensures that the pysqlite headers are included in the source distribution so +that certain C extensions can be compiled. + ## 3.0.0 * Complete rewrite of SQL AST and code-generation. From 670aa160bd7f616964e55362040fc890f7061572 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Jan 2018 13:42:10 -0600 Subject: [PATCH 2/9] Add base class rename --- docs/peewee/changes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/peewee/changes.rst b/docs/peewee/changes.rst index 259ba5c97..b1918add9 100644 --- a/docs/peewee/changes.rst +++ b/docs/peewee/changes.rst @@ -37,6 +37,7 @@ Model Meta options Models ^^^^^^ +* :py:class:`BaseModel` has been renamed to :py:class:`ModelBase` * Accessing raw model data is now done using ``__data__`` instead of ``_data`` Fields From 54c8fde6177d91e77ef5ceb8e500c25453be0107 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 29 Jan 2018 15:07:39 -0600 Subject: [PATCH 3/9] Docs on fn helper. --- docs/peewee/api.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 6108fbd5a..35c1f113c 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -1175,6 +1175,37 @@ Query-builder :param bool coerce: Whether to coerce function-call result. +.. py:function:: fn() + + The :py:func:`fn` helper is actually an instance of :py:class:`Function` + that implements a ``__getattr__`` hook to provide a nice API for calling + SQL functions. + + To create a node representative of a SQL function call, use the function + name as an attribute on ``fn`` and then provide the arguments as you would + if calling a Python function: + + .. code-block:: python + + # List users and the number of tweets they have authored, + # from highest-to-lowest: + sql_count = fn.COUNT(Tweet.id) + query = (User + .select(User, sql_count.alias('count')) + .join(Tweet, JOIN.LEFT_OUTER) + .group_by(User) + .order_by(sql_count.desc())) + + # Get the timestamp of the most recent tweet: + query = Tweet.select(fn.MAX(Tweet.timestamp)) + max_timestamp = query.scalar() # Retrieve scalar result from query. + + Function calls can, like anything else, be composed and nested: + + .. code-block:: python + + # Get users whose username begins with "A" or "a": + a_users = User.select().where(fn.LOWER(fn.SUBSTR(User.username, 1, 1)) == 'a') .. py:class:: Window([partition_by=None[, order_by=None[, start=None[, end=None[, alias=None]]]]]) From 57c8199d850190e3940c63890e00468e0d58bc04 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 29 Jan 2018 15:37:25 -0600 Subject: [PATCH 4/9] More docs, bitfields and plain select queries. --- docs/peewee/models.rst | 103 +++++++++++++++++++++++++++++++++- docs/peewee/query_builder.rst | 73 ++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/docs/peewee/models.rst b/docs/peewee/models.rst index 100c9e90a..de87668f6 100644 --- a/docs/peewee/models.rst +++ b/docs/peewee/models.rst @@ -396,6 +396,89 @@ have an event attached: be formatted so they are sorted lexicographically. That is why they are stored, by default, as ``YYYY-MM-DD HH:MM:SS``. +BitField and BigBitField +^^^^^^^^^^^^^^^^^^^^^^^^ + +The :py:class:`BitField` and :py:class:`BigBitField` are new as of 3.0.0. The +former provides a subclass of :py:class:`IntegerField` that is suitable for +storing feature toggles as an integer bitmask. The latter is suitable for +storing a bitmap for a large data-set, e.g. expressing membership or +bitmap-type data. + +As an example of using :py:class:`BitField`, let's say we have a *Post* model +and we wish to store certain True/False flags about how the post. We could +store all these feature toggles in their own :py:class:`BooleanField` objects, +or we could use :py:class:`BitField` instead: + +.. code-block:: python + + class Post(Model): + content = TextField() + flags = BitField() + + is_favorite = flags.flag(1) + is_sticky = flags.flag(2) + is_minimized = flags.flag(4) + is_deleted = flags.flag(8) + +Using these flags is quite simple: + +.. code-block:: pycon + + >>> p = Post() + >>> p.is_sticky = True + >>> p.is_minimized = True + >>> print(p.flags) # Prints 4 | 2 --> "6" + 6 + >>> p.is_favorite + False + >>> p.is_sticky + True + +We can also use the flags on the Post class to build expressions in queries: + +.. code-block:: python + + # Generates a WHERE clause that looks like: + # WHERE (post.flags & 1 != 0) + favorites = Post.select().where(Post.is_favorite) + + # Query for sticky + favorite posts: + sticky_faves = Post.select().where(Post.is_sticky & Post.is_favorite) + +Since the :py:class:`BitField` is stored in an integer, there is a maximum of +64 flags you can represent (64-bits is common size of integer column). For +storing arbitrarily large bitmaps, you can instead use :py:class:`BigBitField`, +which uses an automatically managed buffer of bytes, stored in a +:py:class:`BlobField`. + +Example usage: + +.. code-block:: python + + class Bitmap(Model): + data = BigBitField() + + bitmap = Bitmap() + + # Sets the ith bit, e.g. the 1st bit, the 11th bit, the 63rd, etc. + bits_to_set = (1, 11, 63, 31, 55, 48, 100, 99) + for bit_idx in bits_to_set: + bitmap.data.set_bit(bit_idx) + + # We can test whether a bit is set using "is_set": + assert bitmap.data.is_set(11) + assert not bitmap.data.is_set(12) + + # We can clear a bit: + bitmap.data.clear_bit(11) + assert not bitmap.data.is_set(11) + + # We can also "toggle" a bit. Recall that the 63rd bit was set earlier. + assert bitmap.data.toggle_bit(63) is False + assert bitmap.data.toggle_bit(63) is True + assert bitmap.data.is_set(63) + BareField ^^^^^^^^^ @@ -404,7 +487,8 @@ SQLite uses dynamic typing and data-types are not enforced, it can be perfectly fine to declare fields without *any* data-type. In those cases you can use :py:class:`BareField`. It is also common for SQLite virtual tables to use meta-columns or untyped columns, so for those cases as well you may wish to use -an untyped field. +an untyped field (although for full-text search, you should use +:py:class:`SearchField` instead!). :py:class:`BareField` accepts a special parameter ``coerce``. This parameter is a function that takes a value coming from the database and converts it into the @@ -412,6 +496,23 @@ appropriate Python type. For instance, if you have a virtual table with an un-typed column but you know that it will return ``int`` objects, you can specify ``coerce=int``. +Example: + +.. code-block:: python + + db = SqliteDatabase(':memory:') + + class Junk(Model): + anything = BareField() + + class Meta: + database = db + + # Store multiple data-types in the Junk.anything column: + Junk.create(anything='a string') + Junk.create(anything=12345) + Junk.create(anything=3.14159) + .. _custom-fields: Creating a custom field diff --git a/docs/peewee/query_builder.rst b/docs/peewee/query_builder.rst index a97b8067c..d8c175003 100644 --- a/docs/peewee/query_builder.rst +++ b/docs/peewee/query_builder.rst @@ -274,6 +274,79 @@ would delete all notes by anyone whose last name is "Foo": # Delete all notes by any person whose ID is in the previous query. Note.delete().where(Note.person_id.in_(foo_people)).execute() +Query Objects +------------- + +One of the fundamental limitations of the abstractions provided by Peewee 2.x +was the absence of a class that represented a structured query with no relation +to a given model class. + +An example of this might be computing aggregate values over a subquery. For +example, the :py:meth:`~SelectBase.count` method, which returns the count of +rows in an arbitrary query, is implemented by wrapping the query: + +.. code-block:: sql + + SELECT COUNT(1) FROM (...) + +To accomplish this with Peewee, the implementation is written in this way: + +.. code-block:: python + + def count(query): + # Select([source1, ... sourcen], [column1, ...columnn]) + wrapped = Select(from_list=[query], columns=[fn.COUNT(SQL('1'))]) + curs = wrapped.tuples().execute(db) + return curs[0][0] # Return first column from first row of result. + +We can actually express this more concisely using the +:py:meth:`~SelectBase.scalar` method, which is suitable for returning values +from aggregate queries: + +.. code-block:: python + + def count(query): + wrapped = Select(from_list=[query], columns=[fn.COUNT(SQL('1'))]) + return wrapped.scalar(db) + +The :ref:`query_examples` document has a more complex example, in which we +write a query for a facility with the highest number of available slots booked: + +The SQL we wish to express is: + +.. code-block:: sql + + SELECT facid, total FROM ( + SELECT facid, SUM(slots) AS total, + rank() OVER (order by SUM(slots) DESC) AS rank + FROM bookings + GROUP BY facid + ) AS ranked + WHERE rank = 1 + +We can express this fairly elegantly by using a plain :py:class:`Select` for +the outer query: + +.. code-block:: python + + # Store rank expression in variable for readability. + rank_expr = fn.rank().over(order_by=[fn.SUM(Booking.slots).desc()]) + + subq = (Booking + .select(Booking.facility, fn.SUM(Booking.slots).alias('total'), + rank_expr.alias('rank')) + .group_by(Booking.facility)) + + # Use a plain "Select" to create outer query. + query = (Select(columns=[subq.c.facid, subq.c.total]) + .from_(subq) + .where(subq.c.rank == 1) + .tuples()) + + # Iterate over the resulting facility ID(s) and total(s): + for facid, total in query.execute(db): + print(facid, total) + More ---- From 20cbb2c14b4f4638ae5e86e4f97063f60405d3ba Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 29 Jan 2018 16:47:30 -0600 Subject: [PATCH 5/9] Update logo! --- README.rst | 2 +- docs/index.rst | 2 +- docs/peewee3-logo.png | Bin 0 -> 1655 bytes 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 docs/peewee3-logo.png diff --git a/README.rst b/README.rst index 31ec40ad9..cac9feb0d 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ Peewee 3.0a has been merged into master and pushed to PyPI. -.. image:: http://media.charlesleifer.com/blog/photos/p1423749536.32.png +.. image:: http://media.charlesleifer.com/blog/photos/peewee3-logo.png peewee ====== diff --git a/docs/index.rst b/docs/index.rst index e188ff854..0b4aa301d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ peewee ====== -.. image:: peewee-logo.png +.. image:: peewee3-logo.png .. attention:: Peewee 3.0 has been released (you are looking at the 3.0 documentation). To diff --git a/docs/peewee3-logo.png b/docs/peewee3-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..287dd83b7c27ed31d39e87c5b6b1c8a5dc42d25d GIT binary patch literal 1655 zcmeAS@N?(olHy`uVBq!ia0y~yU{+vYVASDY1BzJdUbg~LY)RhkE)4%caKYZ?lYt_f z1s;*b3=G`DAk4@xYmNj^kiEpy*OmPdqpX-7*KeavZJA?KVLm`FT>#-#@wfwddG?hRkpS*^OZwTm;oPkd+6T zA*S5Y`>f-c_d8D-=z#<&MkXY$>^bZGf`MT}gLV9$xK3W6%?b`cOF&9686f|IrQi&k z3<;oLY}UUuIzQ>jw)fi@fxKmcK${UBxxJ@_4;T|WJ|BHQ^+p5G08%^vHpb7O2dHh{ z^=}>fFN4AX*4Nh?SFkUq#wl1I**|at^w{cOa|D;pwLHU z5E~eHk_TK0o;-F-Slib`tN+mM{9F`w?)8JuOy667AFAAWzdVr{82E4vm<+fPC=5_o zfP?nNMFXZZ&m@j>+y(Iu_E^jzIZiQcgo~iq=+FSQ5pFdK12w=wQm7*EX7^-3;pM;@5;ZoetyJZ22&4LhQdINAb6C)VyZ!QDKOI*80s=G?6`Qe{_OTN%EFWw zkD<5^p0;o}6w}4AJ#*#vtrccrXy_#`myl#XETB*`Iw+n|^EZkJIKts+0nQ*RQ5>*0 ySXZ}UZ6>h1klzR@HQ*tF$sonCB)QSz4u64wbFNYNmaRaKGkCiCxvX Date: Mon, 29 Jan 2018 16:54:45 -0600 Subject: [PATCH 6/9] Update front matter. --- README.rst | 8 ++++---- docs/index.rst | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index cac9feb0d..91aec8377 100644 --- a/README.rst +++ b/README.rst @@ -10,12 +10,12 @@ peewee Peewee is a simple and small ORM. It has few (but expressive) concepts, making it easy to learn and intuitive to use. * A small, expressive ORM -* Written in python with support for versions 2.6+ and 3.2+. +* Written in python with support for versions 2.7+ and 3.4+ (developed with 3.6) * built-in support for sqlite, mysql and postgresql * tons of extensions available in the `playhouse `_ * `Postgresql HStore, JSON, arrays and more `_ - * `SQLite full-text search, user-defined functions, virtual tables and more `_ + * `SQLite full-text search, user-defined functions, virtual tables and more `_ * `Schema migrations `_ and `model code generator `_ * `Connection pool `_ * `Encryption `_ @@ -28,6 +28,7 @@ New to peewee? Here is a list of documents you might find most helpful when gett started: * `Quickstart guide `_ -- this guide covers all the essentials. It will take you between 5 and 10 minutes to go through it. +* `Example queries `_ taken from the `PostgreSQL Exercises website `_. * `Guide to the various query operators `_ describes how to construct queries and combine expressions. * `Field types table `_ lists the various field types peewee supports and the parameters they accept. @@ -116,8 +117,7 @@ Queries are expressive and composable: .order_by(tweet_ct.desc())) # Do an atomic update - Counter.update(count=Counter.count + 1).where( - Counter.url == request.url) + Counter.update(count=Counter.count + 1).where(Counter.url == request.url) Check out the `example app `_ for a working Twitter-clone website written with Flask. diff --git a/docs/index.rst b/docs/index.rst index 0b4aa301d..4b4ec9764 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ Peewee is a simple and small ORM. It has few (but expressive) concepts, making it easy to learn and intuitive to use. * A small, expressive ORM -* Written in python with support for versions 2.7+ and 3.2+. +* Written in python with support for versions 2.7+ and 3.4+ (developed with 3.6) * Built-in support for SQLite, MySQL and Postgresql. * :ref:`numerous extensions available ` (:ref:`postgres hstore/json/arrays `, :ref:`sqlite full-text-search `, :ref:`schema migrations `, and much more). @@ -38,6 +38,7 @@ New to peewee? Here is a list of documents you might find most helpful when gett started: * :ref:`Quickstart guide ` -- this guide covers all the bare essentials. It will take you between 5 and 10 minutes to go through it. +* :ref:`Example queries ` taken from the `PostgreSQL exercises website `_. * :ref:`Guide to the various query operators ` describes how to construct queries and combine expressions. * :ref:`Field types table ` lists the various field types peewee supports and the parameters they accept. There is also an :ref:`extension module ` that contains :ref:`special/custom field types `. From bd3ee2ab7a0680d0b5181a13e02622761e169e5c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 30 Jan 2018 09:14:06 -0600 Subject: [PATCH 7/9] Fix #1439, document naive->objects --- docs/peewee/changes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/peewee/changes.rst b/docs/peewee/changes.rst index b1918add9..74f2873bd 100644 --- a/docs/peewee/changes.rst +++ b/docs/peewee/changes.rst @@ -69,6 +69,9 @@ code and it's somewhat limited usefulness convinced me to scrap it. You can instead use :py:func:`prefetch` to achieve the same result. * :py:class:`Select` query attribute ``_select`` has changed to ``_returning`` +* The ``naive()`` method is now :py:meth:`~BaseQuery.objects`, which defaults + to using the model class as the constructor, but accepts any callable to use + as an alternate constructor. The :py:func:`Case` helper has moved from the ``playhouse.shortcuts`` module into the main peewee module. From 20c2a9fe9f604c9ef8fbef98bd8dba44405a98a6 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 30 Jan 2018 09:45:17 -0600 Subject: [PATCH 8/9] Fix column->field name translation. Fixes #1437. --- peewee.py | 20 ++++++++++++++++---- tests/models.py | 23 +++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/peewee.py b/peewee.py index 399e2d68a..68bca28b1 100644 --- a/peewee.py +++ b/peewee.py @@ -5761,6 +5761,7 @@ def _initialize_columns(self): self.columns = [] self.converters = converters = [None] * self.ncols self.fields = fields = [None] * self.ncols + self.translation = {} for idx, description_item in enumerate(description): column = description_item[0] @@ -5814,12 +5815,14 @@ class ModelDictCursorWrapper(BaseModelCursorWrapper): def process_row(self, row): result = {} columns, converters = self.columns, self.converters + fields = self.fields for i in range(self.ncols): + attr = fields[i].name if fields[i] is not None else columns[i] if converters[i] is not None: - result[columns[i]] = converters[i](row[i]) + result[attr] = converters[i](row[i]) else: - result[columns[i]] = row[i] + result[attr] = row[i] return result @@ -5837,7 +5840,13 @@ def process_row(self, row): class ModelNamedTupleCursorWrapper(ModelTupleCursorWrapper): def initialize(self): self._initialize_columns() - self.tuple_class = namedtuple('Row', self.columns) + attributes = [] + for i in range(self.ncols): + if self.fields[i] is not None: + attributes.append(self.fields[i].name) + else: + attributes.append(self.columns[i]) + self.tuple_class = namedtuple('Row', attributes) self.constructor = lambda row: self.tuple_class(*row) @@ -5915,7 +5924,10 @@ def process_row(self, row): set_keys = set() for idx, key in enumerate(self.column_keys): instance = objects[key] - column = self.columns[idx] + if self.fields[idx] is not None: + column = self.fields[idx].name + else: + column = self.columns[idx] value = row[idx] if value is not None: set_keys.add(key) diff --git a/tests/models.py b/tests/models.py index 604a93c03..a4736bbb6 100644 --- a/tests/models.py +++ b/tests/models.py @@ -31,6 +31,12 @@ class Color(TestModel): is_neutral = BooleanField(default=False) +class Post(TestModel): + content = TextField(column_name='Content') + timestamp = DateTimeField(column_name='TimeStamp', + default=datetime.datetime.now) + + class TestModelAPIs(ModelTestCase): def add_user(self, username): return User.create(username=username) @@ -51,6 +57,23 @@ def do_test(n): do_test(4) self.assertRaises(AssertionError, do_test, 5) + @requires_models(Post) + def test_column_field_translation(self): + ts = datetime.datetime(2017, 2, 1, 13, 37) + ts2 = datetime.datetime(2017, 2, 2, 13, 37) + p = Post.create(content='p1', timestamp=ts) + p2 = Post.create(content='p2', timestamp=ts2) + + p_db = Post.get(Post.content == 'p1') + self.assertEqual(p_db.content, 'p1') + self.assertEqual(p_db.timestamp, ts) + + pd1, pd2 = Post.select().order_by(Post.id).dicts() + self.assertEqual(pd1['content'], 'p1') + self.assertEqual(pd1['timestamp'], ts) + self.assertEqual(pd2['content'], 'p2') + self.assertEqual(pd2['timestamp'], ts2) + @requires_models(User, Tweet) def test_create(self): with self.assertQueryCount(1): From ae2da5b1750f7e9ae2bcfceef4e544bc8be05f27 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 30 Jan 2018 09:45:57 -0600 Subject: [PATCH 9/9] 3.0.3 --- peewee.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peewee.py b/peewee.py index 68bca28b1..456e0329b 100644 --- a/peewee.py +++ b/peewee.py @@ -53,7 +53,7 @@ mysql = None -__version__ = '3.0.2' +__version__ = '3.0.3' __all__ = [ 'AutoField', 'BareField',