Enable building sqlite3 into cpython (pyodide#352)

* Enable building sqlite3 into cpython
iCodeIN · Mar 26, 2019 · ac76220 · ac76220
1 parent b95b669
commit ac76220
Show file tree

Hide file tree

Showing 8 changed files with 199 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## Unreleased
+
+**User improvements:**
+
+- The built-in `sqlite3` module of Python is now enabled.
+
 ## Version 0.10.0
 
 **User improvements:**

diff --git a/Makefile b/Makefile
@@ -35,6 +35,7 @@ LDFLAGS=\
 	-s USE_FREETYPE=1 \
 	-s USE_LIBPNG=1 \
 	-std=c++14 \
+  -L$(wildcard $(CPYTHONROOT)/build/sqlite*/.libs) -lsqlite3 \
   -lstdc++ \
   --memory-init-file 0 \
   -s "BINARYEN_TRAP_MODE='clamp'" \

diff --git a/cpython/Makefile b/cpython/Makefile
@@ -20,6 +20,10 @@ ZLIBTARBALL=$(ROOT)/downloads/zlib-$(ZLIBVERSION).tar.gz
 ZLIBBUILD=$(ROOT)/build/zlib-$(ZLIBVERSION)
 ZLIBURL=https://zlib.net/zlib-1.2.11.tar.gz
 
+SQLITETARBALL=$(ROOT)/downloads/sqlite-autoconf-3270200.tar.gz
+SQLITEBUILD=$(ROOT)/build/sqlite-autoconf-3270200
+SQLITEURL=https://www.sqlite.org/2019/sqlite-autoconf-3270200.tar.gz
+
 
 all: $(INSTALL)/lib/$(LIB)
 
@@ -53,6 +57,11 @@ $(ZLIBTARBALL):
 	wget -q -O $@ $(ZLIBURL)
 
 
+$(SQLITETARBALL):
+	[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
+	wget -q -O $@ $(SQLITEURL)
+
+
 $(HOSTPYTHON) $(HOSTPGEN): $(TARBALL)
 	mkdir -p $(HOSTINSTALL)
 	[ -d $(HOSTBUILD) ] || tar -C $(HOSTINSTALL) -xf $(TARBALL)
@@ -79,11 +88,31 @@ $(ZLIBBUILD)/.patched: $(ZLIBTARBALL)
 	touch $@
 
 
-$(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.patched
+$(SQLITEBUILD)/libsqlite3.la: $(SQLITETARBALL)
+	[ -d $(ROOT)/build ] || (mkdir $(ROOT)/build)
+	tar -C $(ROOT)/build/ -xf $(SQLITETARBALL)
+	( \
+		cd $(SQLITEBUILD); \
+		emconfigure ./configure; \
+		emmake make; \
+	)
+
+
+$(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.patched $(SQLITEBUILD)/libsqlite3.la
 	cp config.site $(BUILD)/
 	( \
 		cd $(BUILD); \
-		CONFIG_SITE=./config.site READELF=true emconfigure ./configure --without-pymalloc --disable-shared --disable-ipv6 --without-gcc --host=asmjs-unknown-emscripten --build=$(shell $(BUILD)/config.guess) --prefix=$(INSTALL) ; \
+		CONFIG_SITE=./config.site READELF=true LD_RUN_PATH=$(SQLITEBUILD) emconfigure \
+		  ./configure \
+			  CPPFLAGS="-I$(SQLITEBUILD)" \
+				LDFLAGS="-L$(SQLITEBUILD)" \
+				--without-pymalloc \
+				--disable-shared \
+				--disable-ipv6 \
+				--without-gcc \
+				--host=asmjs-unknown-emscripten \
+				--build=$(shell $(BUILD)/config.guess) \
+				--prefix=$(INSTALL) ; \
 	)
 
 

diff --git a/cpython/Setup.local b/cpython/Setup.local
@@ -35,6 +35,9 @@ _sha3 _sha3/sha3module.c
 _md5 md5module.c
 _blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.7.0/Modules/_blake2/blake2s_impl.c
 
+_sqlite3 _sqlite/cache.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c -I$(SQLITEBUILD) -L$(SQLITEBUILD) -lsqlite3
+
+
 _queue _queuemodule.c
 
 #future_builtins future_builtins.c

diff --git a/cpython/patches/sqlite-MODULE_NAME.patch b/cpython/patches/sqlite-MODULE_NAME.patch
@@ -0,0 +1,135 @@
+diff --git a/Modules/_sqlite/Modules/_sqlite/cache.c b/Modules/_sqlite/cache.c
+index 72b1f2c..2190bd4 100644
+--- a/Modules/_sqlite/cache.c
++++ b/Modules/_sqlite/cache.c
+@@ -24,6 +24,10 @@
+ #include "cache.h"
+ #include <limits.h>
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ /* only used internally */
+ pysqlite_Node* pysqlite_new_node(PyObject* key, PyObject* data)
+ {
+diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
+index 6e05761..2d1ad34 100644
+--- a/Modules/_sqlite/connection.c
++++ b/Modules/_sqlite/connection.c
+@@ -45,6 +45,10 @@
+ #define HAVE_BACKUP_API
+ #endif
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ _Py_IDENTIFIER(cursor);
+
+ static const char * const begin_statements[] = {
+diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c
+index 4ecb5b4..24b03c1 100644
+--- a/Modules/_sqlite/cursor.c
++++ b/Modules/_sqlite/cursor.c
+@@ -25,6 +25,10 @@
+ #include "module.h"
+ #include "util.h"
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ PyObject* pysqlite_cursor_iternext(pysqlite_Cursor* self);
+
+ static const char errmsg_fetch_across_rollback[] = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from.";
+diff --git a/Modules/_sqlite/microprotocols.c b/Modules/_sqlite/microprotocols.c
+index 3d01872..7f98c2f 100644
+--- a/Modules/_sqlite/microprotocols.c
++++ b/Modules/_sqlite/microprotocols.c
+@@ -30,6 +30,10 @@
+ #include "microprotocols.h"
+ #include "prepare_protocol.h"
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+
+ /** the adapters registry **/
+
+diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c
+index 6befa07..4611574 100644
+--- a/Modules/_sqlite/module.c
++++ b/Modules/_sqlite/module.c
+@@ -33,6 +33,10 @@
+ #define HAVE_SHARED_CACHE
+ #endif
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ /* static objects at module-level */
+
+ PyObject *pysqlite_Error = NULL;
+diff --git a/Modules/_sqlite/prepare_protocol.c b/Modules/_sqlite/prepare_protocol.c
+index f2c85f9..14969bb 100644
+--- a/Modules/_sqlite/prepare_protocol.c
++++ b/Modules/_sqlite/prepare_protocol.c
+@@ -23,6 +23,10 @@
+
+ #include "prepare_protocol.h"
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ int pysqlite_prepare_protocol_init(pysqlite_PrepareProtocol* self, PyObject* args, PyObject* kwargs)
+ {
+     return 0;
+diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
+index ec2c788..ce65e62 100644
+--- a/Modules/_sqlite/row.c
++++ b/Modules/_sqlite/row.c
+@@ -24,6 +24,10 @@
+ #include "row.h"
+ #include "cursor.h"
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ void pysqlite_row_dealloc(pysqlite_Row* self)
+ {
+     Py_XDECREF(self->data);
+diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c
+index 3869088..6d16049 100644
+--- a/Modules/_sqlite/statement.c
++++ b/Modules/_sqlite/statement.c
+@@ -28,6 +28,10 @@
+ #include "prepare_protocol.h"
+ #include "util.h"
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ /* prototypes */
+ static int pysqlite_check_remaining_sql(const char* tail);
+
+diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c
+index 3fa671d..bd12f51 100644
+--- a/Modules/_sqlite/util.c
++++ b/Modules/_sqlite/util.c
+@@ -24,6 +24,10 @@
+ #include "module.h"
+ #include "connection.h"
+
++#ifndef MODULE_NAME
++#define MODULE_NAME "sqlite"
++#endif
++
+ int pysqlite_step(sqlite3_stmt* statement, pysqlite_Connection* connection)
+ {
+     int rc;
diff --git a/remove_modules.txt b/remove_modules.txt
@@ -6,7 +6,6 @@ ensurepip
 idlelib
 lib2to3
 multiprocessing
-sqlite3
 tkinter
 turtle.py
 turtledemo

diff --git a/test/python_tests.txt b/test/python_tests.txt
@@ -416,7 +416,7 @@ test_socketserver
 test_sort
 test_source_encoding  subprocess,
 test_spwd
-test_sqlite
+test_sqlite threading
 test_ssl
 test_startfile
 test_stat

diff --git a/test/test_sqlite3.py b/test/test_sqlite3.py
@@ -0,0 +1,22 @@
+def test_sqlite3(selenium):
+    content = selenium.run("""
+        import sqlite3
+
+        with sqlite3.connect(':memory:') as conn:
+            c = conn.cursor()
+            c.execute('''
+                CREATE TABLE people (
+                    first_name VARCHAR,
+                    last_name VARCHAR
+                )
+            ''')
+            c.execute("INSERT INTO people VALUES ('John', 'Doe')")
+            c.execute("INSERT INTO people VALUES ('Jane', 'Smith')")
+            c.execute("INSERT INTO people VALUES ('Michael', 'Jordan')")
+            c.execute("SELECT * FROM people")
+    """)
+    content = selenium.run("c.fetchall()")
+    assert len(content) == 3
+    assert content[0][0] == 'John'
+    assert content[1][0] == 'Jane'
+    assert content[2][0] == 'Michael'
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,7 +6,6 @@ ensurepip @@
     idlelib
     lib2to3
     multiprocessing
-    sqlite3
     tkinter
     turtle.py
     turtledemo
@@ Expand Down @@