diff --git a/.travis.yml b/.travis.yml index 48bac04b2..262290d14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ addons: services: - postgresql - mysql -install: "pip install psycopg2 Cython pymysql apsw" +install: "pip install psycopg2 Cython pymysql apsw mysql-connector" before_script: - python setup.py build_ext -i - psql -c 'drop database if exists peewee_test;' -U postgres diff --git a/CHANGELOG.md b/CHANGELOG.md index 466688c32..c29e5b411 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ releases, visit GitHub: https://github.com/coleifer/peewee/releases +## 3.2.1 + +**Notice:** the default mysql driver for Peewee has changed to [pymysql](https://github.com/PyMySQL/PyMySQL) +in version 3.2.1. In previous versions, if both *mysql-python* and *pymysql* +were installed, Peewee would use *mysql-python*. As of 3.2.1, if both libraries +are installed Peewee will use *pymysql*. + +* Added new module `playhouse.mysql_ext` which includes + `MySQLConnectorDatabase`, a database implementation that works with the + [mysql-connector](https://dev.mysql.com/doc/connector-python/en/) driver. +* Added new field to `ColumnMetadata` class which captures a database column's + default value. `ColumnMetadata` is returned by `Database.get_columns()`. +* Added [documentation on making Peewee async](http://docs.peewee-orm.com/en/latest/peewee/database.html#async-with-gevent). + +[View commits](https://github.com/coleifer/peewee/compare/3.2.0...3.2.1) + ## 3.2.0 The 3.2.0 release introduces a potentially backwards-incompatible change. The diff --git a/bench.py b/bench.py index 745804636..1f21f2e02 100644 --- a/bench.py +++ b/bench.py @@ -2,6 +2,8 @@ db = SqliteDatabase(':memory:') +#db = PostgresqlDatabase('peewee_test', host='127.0.0.1', port=26257, user='root') +#db = PostgresqlDatabase('peewee_test', host='127.0.0.1', user='postgres') class Base(Model): class Meta: @@ -47,6 +49,20 @@ def insert(): with db.atomic(): populate_register(1000) +@timed +def batch_insert(): + with db.atomic(): + it = range(1000) + for i in db.batch_commit(it, 100): + Register.insert(value=i + 100000).execute() + +@timed +def bulk_insert(): + with db.atomic(): + for i in range(0, 1000, 100): + data = [(j,) for j in range(i, i + 100)] + Register.insert_many(data, fields=[Register.value]).execute() + @timed def select(): query = Register.select() @@ -81,7 +97,12 @@ def select_related_dicts(): db.create_tables([Register, Collection, Item]) insert() insert_related() + Register.delete().execute() + batch_insert() + Register.delete().execute() + bulk_insert() select() select_related() select_related_left() select_related_dicts() + db.drop_tables([Register, Collection, Item]) diff --git a/docs/peewee/database.rst b/docs/peewee/database.rst index 2efac2af2..02e4de2d5 100644 --- a/docs/peewee/database.rst +++ b/docs/peewee/database.rst @@ -809,6 +809,56 @@ potential compatibility issues. If you'd like to see some more examples of how to run tests using Peewee, check out Peewee's own `test-suite `_. +Async with Gevent +----------------- + +`gevent ` is recommended for doing asynchronous i/o +with Postgresql or MySQL. Reasons I prefer gevent: + +* No need for special-purpose "loop-aware" re-implementations of *everything*. + Third-party libraries using asyncio usually have to re-implement layers and + layers of code as well as re-implementing the protocols themselves. +* Gevent allows you to write your application in normal, clean, idiomatic + Python. No need to litter every line with "async", "await" and other noise. + No callbacks. No cruft. +* Gevent works with both Python 2 *and* Python 3. +* Gevent is *Pythonic*. Asyncio is an un-pythonic abomination. + +Besides monkey-patching socket, no special steps are required if you are using +**MySQL** with a pure Python driver like `pymysql `_ +or are using `mysql-connector `_ +in pure-python mode. MySQL drivers written in C will require special +configuration which is beyond the scope of this document. + +For **Postgres** and `psycopg2 `_, which is a C +extension, you can use the following code snippet to register event hooks that +will make your connection async: + +.. code-block:: python + + from gevent.socket import wait_read, wait_write + from psycopg2 import extensions + + # Call this function after monkey-patching socket (etc). + def patch_psycopg2(): + extensions.set_wait_callback(_psycopg2_gevent_callback) + + def _psycopg2_gevent_callback(conn, timeout=None): + while True: + state = conn.poll() + if state == extensions.POLL_OK: + break + elif state == extensions.POLL_READ: + wait_read(conn.fileno(), timeout=timeout) + elif state == extensions.POLL_WRITE: + wait_write(conn.fileno(), timeout=timeout) + else: + raise ValueError('poll() returned unexpected result') + +**SQLite**, because it is embedded in the Python application itself, does not +do any socket operations that would be a candidate for non-blocking. Async has +no effect one way or the other on SQLite databases. + .. _framework-integration: Framework Integration diff --git a/docs/peewee/playhouse.rst b/docs/peewee/playhouse.rst index 69a585792..cb17fcf3f 100644 --- a/docs/peewee/playhouse.rst +++ b/docs/peewee/playhouse.rst @@ -19,6 +19,7 @@ make up the ``playhouse``. * :ref:`apsw` * :ref:`sqlcipher_ext` * :ref:`postgres_ext` +* :ref:`mysql_ext` **High-level features** @@ -1541,6 +1542,24 @@ postgres_ext API notes search_content=fn.to_tsvector(content)) # Note `to_tsvector()`. +.. _mysql_ext: + +MySQL Extensions +---------------- + +Peewee provides an alternate database implementation for using the +`mysql-connector `_ driver. The +implementation can be found in ``playhouse.mysql_ext``. + +Example usage: + +.. code-block:: python + + from playhouse.mysql_ext import MySQLConnectorDatabase + + # MySQL database implementation that utilizes mysql-connector driver. + db = MySQLConnectorDatabase('my_database', host='1.2.3.4', user='mysql') + .. _dataset: DataSet diff --git a/peewee.py b/peewee.py index b8e1b00a5..e546c53b9 100644 --- a/peewee.py +++ b/peewee.py @@ -45,16 +45,19 @@ from psycopg2 import extensions as pg_extensions except ImportError: psycopg2 = None + +mysql_passwd = False try: - import MySQLdb as mysql # prefer the C module. + import pymysql as mysql except ImportError: try: - import pymysql as mysql + import MySQLdb as mysql # prefer the C module. + mysql_passwd = True except ImportError: mysql = None -__version__ = '3.2.0' +__version__ = '3.2.1' __all__ = [ 'AsIs', 'AutoField', @@ -2338,7 +2341,7 @@ def __exit__(self, exc_type, exc_value, traceback): ('name', 'sql', 'columns', 'unique', 'table')) ColumnMetadata = collections.namedtuple( 'ColumnMetadata', - ('name', 'data_type', 'null', 'primary_key', 'table')) + ('name', 'data_type', 'null', 'primary_key', 'table', 'default')) ForeignKeyMetadata = collections.namedtuple( 'ForeignKeyMetadata', ('column', 'dest_table', 'dest_column', 'table')) @@ -2902,8 +2905,8 @@ def get_indexes(self, table, schema=None): def get_columns(self, table, schema=None): cursor = self.execute_sql('PRAGMA table_info("%s")' % table) - return [ColumnMetadata(row[1], row[2], not row[3], bool(row[5]), table) - for row in cursor.fetchall()] + return [ColumnMetadata(r[1], r[2], not r[3], bool(r[5]), table, r[4]) + for r in cursor.fetchall()] def get_primary_keys(self, table, schema=None): cursor = self.execute_sql('PRAGMA table_info("%s")' % table) @@ -3011,14 +3014,14 @@ def get_indexes(self, table, schema=None): def get_columns(self, table, schema=None): query = """ - SELECT column_name, is_nullable, data_type + SELECT column_name, is_nullable, data_type, column_default FROM information_schema.columns WHERE table_name = %s AND table_schema = %s ORDER BY ordinal_position""" cursor = self.execute_sql(query, (table, schema or 'public')) pks = set(self.get_primary_keys(table, schema)) - return [ColumnMetadata(name, dt, null == 'YES', name in pks, table) - for name, null, dt in cursor.fetchall()] + return [ColumnMetadata(name, dt, null == 'YES', name in pks, table, df) + for name, null, dt, df in cursor.fetchall()] def get_primary_keys(self, table, schema=None): query = """ @@ -3152,7 +3155,7 @@ class MySQLDatabase(Database): def init(self, database, **kwargs): params = {'charset': 'utf8', 'use_unicode': True} params.update(kwargs) - if 'password' in params: + if 'password' in params and mysql_passwd: params['passwd'] = params.pop('password') super(MySQLDatabase, self).init(database, **params) @@ -3181,13 +3184,13 @@ def get_indexes(self, table, schema=None): def get_columns(self, table, schema=None): sql = """ - SELECT column_name, is_nullable, data_type + SELECT column_name, is_nullable, data_type, column_default FROM information_schema.columns WHERE table_name = %s AND table_schema = DATABASE()""" cursor = self.execute_sql(sql, (table,)) pks = set(self.get_primary_keys(table)) - return [ColumnMetadata(name, dt, null == 'YES', name in pks, table) - for name, null, dt in cursor.fetchall()] + return [ColumnMetadata(name, dt, null == 'YES', name in pks, table, df) + for name, null, dt, df in cursor.fetchall()] def get_primary_keys(self, table, schema=None): cursor = self.execute_sql('SHOW INDEX FROM `%s`' % table) diff --git a/playhouse/mysql_ext.py b/playhouse/mysql_ext.py new file mode 100644 index 000000000..dbccc9541 --- /dev/null +++ b/playhouse/mysql_ext.py @@ -0,0 +1,19 @@ +try: + import mysql.connector as mysql_connector +except ImportError: + mysql_connector = None + +from peewee import ImproperlyConfigured +from peewee import MySQLDatabase + + +class MySQLConnectorDatabase(MySQLDatabase): + def _connect(self): + if mysql_connector is None: + raise ImproperlyConfigured('MySQL connector not installed!') + return mysql_connector.connect(db=self.database, **self.connect_params) + + def cursor(self, commit=None): + if self.is_closed(): + self.connect() + return self._state.conn.cursor(buffered=True) diff --git a/playhouse/reflection.py b/playhouse/reflection.py index 7e43bd66d..1edca980d 100644 --- a/playhouse/reflection.py +++ b/playhouse/reflection.py @@ -8,10 +8,10 @@ from peewee import * try: - from MySQLdb.constants import FIELD_TYPE + from pymysql.constants import FIELD_TYPE except ImportError: try: - from pymysql.constants import FIELD_TYPE + from MySQLdb.constants import FIELD_TYPE except ImportError: FIELD_TYPE = None try: diff --git a/runtests.py b/runtests.py index df5c7d7ae..6e5c6aba9 100755 --- a/runtests.py +++ b/runtests.py @@ -26,7 +26,7 @@ def get_option_parser(): '--engine', dest='engine', help=('Database engine to test, one of ' - '[sqlite, postgres, mysql, apsw, sqlcipher]')) + '[sqlite, postgres, mysql, mysqlconnector, apsw, sqlcipher]')) basic.add_option('-v', '--verbosity', dest='verbosity', default=1, type='int', help='Verbosity of output') parser.add_option_group(basic) diff --git a/tests/base.py b/tests/base.py index ad440e3c9..c518abdea 100644 --- a/tests/base.py +++ b/tests/base.py @@ -10,6 +10,7 @@ from .libs import mock from peewee import * +from playhouse.mysql_ext import MySQLConnectorDatabase logger = logging.getLogger('peewee') @@ -21,6 +22,7 @@ def db_loader(engine, name='peewee_test', db_class=None, **params): SqliteDatabase: ['sqlite', 'sqlite3'], MySQLDatabase: ['mysql'], PostgresqlDatabase: ['postgres', 'postgresql'], + MySQLConnectorDatabase: ['mysqlconnector'], } engine_map = dict((alias, db) for db, aliases in engine_aliases.items() for alias in aliases) @@ -44,7 +46,7 @@ def get_in_memory_db(**params): VERBOSITY = int(os.environ.get('PEEWEE_TEST_VERBOSITY') or 1) IS_SQLITE = BACKEND in ('sqlite', 'sqlite3') -IS_MYSQL = BACKEND == 'mysql' +IS_MYSQL = BACKEND in ('mysql', 'mysqlconnector') IS_POSTGRESQL = BACKEND in ('postgres', 'postgresql') diff --git a/tests/postgres.py b/tests/postgres.py index 20afb743f..d2e271552 100644 --- a/tests/postgres.py +++ b/tests/postgres.py @@ -13,8 +13,7 @@ from .base_models import Register -db = db_loader('postgres', db_class=PostgresqlExtDatabase, - register_hstore=True) +db = db_loader('postgres', db_class=PostgresqlExtDatabase) class HStoreModel(TestModel): @@ -83,7 +82,8 @@ def test_tz_field(self): class TestHStoreField(ModelTestCase): - database = db + database = db_loader('postgres', db_class=PostgresqlExtDatabase, + register_hstore=True) requires = [HStoreModel] def setUp(self):