Skip to content

Commit

Permalink
Update to Python 3.8.2 (pyodide#712)
Browse files Browse the repository at this point in the history 
* Updating to CPython 3.8

* More fixes

* Skip python headers check in tools/dependency-check.sh

* Fix checksum

* Fix packages

* Fix syntax error

* Fix benchmark path

* Update CPython tests

* More test fixes

* Fix PyStone

* More Cpython test fixes

* Update docs

* Fix issue in conftest

* Add documentation

* Upload updated Docker image
  • Loading branch information
@rth
rth authored Jul 7, 2020
1 parent 3032dcc commit fc5495f
Show file tree
Hide file tree
Showing 35 changed files with 610 additions and 392 deletions.
12 changes: 6 additions & 6 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ version: 2
defaults: &defaults
working_directory: ~/repo
docker:
- image: iodide/pyodide-env:0.15.1
- image: iodide/pyodide-env:0.16.0
environment:
- EMSDK_NUM_CORES: 4
EMCC_CORES: 4
Expand All @@ -27,7 +27,7 @@ jobs:

- restore_cache:
keys:
- v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20200606-
- v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20200706-

- run:
name: build
Expand All @@ -45,13 +45,13 @@ jobs:
paths:
- ./emsdk/emsdk
- ~/.ccache
key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20200606-{{ .BuildNum }}
key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20200706-{{ .BuildNum }}

- persist_to_workspace:
root: .
paths:
- ./build
- ./cpython/build/3.7.4/host
- ./cpython/build/3.8.2/host

- store_artifacts:
path: /home/circleci/repo/build/
Expand All @@ -64,7 +64,7 @@ jobs:
# this cache is generated by the main build job, we never store it here
- restore_cache:
keys:
- v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20200606-
- v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20200706-

- run:
name: build
Expand Down Expand Up @@ -132,7 +132,7 @@ jobs:
- run:
name: benchmark
command: |
python benchmark/benchmark.py cpython/build/3.7.4/host/bin/python3 build/benchmarks.json
python benchmark/benchmark.py cpython/build/3.8.2/host/bin/python3 build/benchmarks.json
- store_artifacts:
path: /home/circleci/repo/build/benchmarks.json

Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM circleci/python:3.7.7-buster
FROM circleci/python:3.8.2-buster

RUN sudo apt-get update \
# bzip2 and libgconf-2-4 are necessary for extracting firefox and running chrome, respectively
Expand Down
6 changes: 5 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ clean:
make -C zlib clean
echo "The Emsdk, CPython and CLAPACK are not cleaned. cd into those directories to do so."

clean-all: clean
make -C emsdk clean
make -C cpython clean
rm -fr cpython/build

%.bc: %.c $(CPYTHONLIB) $(LZ4LIB)
$(CC) -o $@ -c $< $(CFLAGS)
Expand All @@ -176,7 +180,7 @@ build/test.data: $(CPYTHONLIB)
)
( \
cd build; \
python $(FILEPACKAGER) test.data --abi=$(PYODIDE_PACKAGE_ABI) --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude __pycache__ \
python $(FILEPACKAGER) test.data --abi=$(PYODIDE_PACKAGE_ABI) --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.8/test --js-output=test.js --export-name=pyodide._module --exclude __pycache__ \
)
uglifyjs build/test.js -o build/test.js

Expand Down
2 changes: 1 addition & 1 deletion Makefile.envs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ export EM_CACHE = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten_cache
export EMSCRIPTEN = $(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-$(EMSCRIPTEN_VERSION)
export BINARYEN_ROOT = $(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-$(EMSCRIPTEN_VERSION)_64bit_binaryen

export PYVERSION=3.7.4
export PYVERSION=3.8.2
export PYMINOR=$(basename $(PYVERSION))
export HOSTPYTHONROOT=$(PYODIDE_ROOT)/cpython/build/$(PYVERSION)/host
export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ The Python scientific stack, compiled to WebAssembly.

## What is Pyodide?

**Pyodide** brings the Python runtime to the browser via WebAssembly, along with the Python scientific stack including NumPy, Pandas, Matplotlib, parts of SciPy, and NetworkX. The [`packages` directory](https://github.com/iodide-project/pyodide/tree/master/packages) lists over 35 packages which are currently available.
**Pyodide** brings the Python 3.8 runtime to the browser via WebAssembly, along with the Python scientific stack including NumPy, Pandas, Matplotlib, parts of SciPy, and NetworkX. The [`packages` directory](https://github.com/iodide-project/pyodide/tree/master/packages) lists over 35 packages which are currently available.

**Pyodide** provides transparent conversion of objects between Javascript and Python.
When used inside a browser, Python has full access to the Web APIs.
Expand Down
4 changes: 2 additions & 2 deletions conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,8 +398,8 @@ def run_web_server(q, log_filepath, build_dir):

class Handler(http.server.CGIHTTPRequestHandler):
def translate_path(self, path):
if path.startswith("/test/"):
return TEST_PATH / path[6:]
if str(path).startswith("/test/"):
return str(TEST_PATH / path[6:])
return super(Handler, self).translate_path(path)

def is_cgi(self):
Expand Down
1 change: 0 additions & 1 deletion cpython/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ $(HOSTPYTHON) $(HOSTPGEN): $(TARBALL)
PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig" ./configure --prefix=$(HOSTINSTALL) || cat config.log && \
make regen-grammar -j $${PYODIDE_JOBS:-3} && \
make install -j $${PYODIDE_JOBS:-3} && \
cp Parser/pgen$(EXE) $(HOSTINSTALL)/bin/ && \
make distclean \
)

Expand Down
2 changes: 1 addition & 1 deletion cpython/Setup.local
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ _sha256 sha256module.c
_sha512 sha512module.c
_sha3 _sha3/sha3module.c
_md5 md5module.c
_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.7.4/Modules/_blake2/blake2s_impl.c
_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.8.2/Modules/_blake2/blake2s_impl.c

_sqlite3 _sqlite/cache.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c -I$(SQLITEBUILD) -L$(SQLITEBUILD) -lsqlite3
_crypt _cryptmodule.c
Expand Down
2 changes: 1 addition & 1 deletion cpython/checksums
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
68111671e5b2db4aef7b9ab01bf0f9be downloads/Python-3.7.4.tgz
f9f3768f757e34b342dbc06b41cbc844 downloads/Python-3.8.2.tgz
1 change: 1 addition & 0 deletions cpython/config.site
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ ac_cv_func_dlopen=yes
ac_cv_lib_socket_socket=no
ac_cv_header_linux_vm_sockets_h=yes
ac_cv_little_endian_double=yes
ax_cv_c_float_words_bigendian=no
15 changes: 12 additions & 3 deletions ...hon/patches/disable-set_inheritable.patch → ...atches/0001-disable-set_inheritable.patch
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
From f78bb95d3c95a4fb20745eb70ad876495de26e94 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Sun, 5 Jul 2020 17:37:43 +0200
Subject: [PATCH] disable-set_inheritable


diff --git a/Python/fileutils.c b/Python/fileutils.c
index 35869c81ac..1af4ead7f5 100644
index e79e732d1f..864dc78a57 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -964,6 +964,9 @@ _Py_get_inheritable(int fd)
@@ -1091,6 +1091,9 @@ _Py_get_inheritable(int fd)
static int
set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
{
Expand All @@ -12,11 +18,14 @@ index 35869c81ac..1af4ead7f5 100644
#ifdef MS_WINDOWS
HANDLE handle;
DWORD flags;
@@ -1080,6 +1083,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
@@ -1214,6 +1217,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
}
return 0;
#endif
+#endif
}

/* Make the file descriptor non-inheritable.
--
2.25.1

13 changes: 11 additions & 2 deletions ...hes/dont-test-undecodable-filenames.patch → ...002-dont-test-undecodable-filenames.patch
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
From ba3bbdfc19c8d1d778c36af0424c56e193460170 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Sun, 5 Jul 2020 17:38:21 +0200
Subject: [PATCH] dont-test-undecodable-filenames


diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index ddcd2cc387..f422f2feb6 100644
index d6a7819cb0..17a13cf49d 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -880,6 +880,8 @@ if os.name == 'nt':
@@ -952,6 +952,8 @@ if os.name == 'nt':
'Unicode filename tests may not be effective'
% (TESTFN_UNENCODABLE, TESTFN_ENCODING))
TESTFN_UNENCODABLE = None
Expand All @@ -11,3 +17,6 @@ index ddcd2cc387..f422f2feb6 100644
# Mac OS X denies unencodable filenames (invalid utf-8)
elif sys.platform != 'darwin':
try:
--
2.25.1

13 changes: 11 additions & 2 deletions cpython/patches/no_af_vsock.patch → cpython/patches/0003-no_af_vsock.patch
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
From 9597e7b2ec30968fe9aec0c5a3bc1187e45bc870 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Sun, 5 Jul 2020 17:39:02 +0200
Subject: [PATCH] no_af_vsock


diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h
index fdb4e871ce..c990003ff3 100644
index dff1f8f4e9..6014853ac8 100644
--- a/Modules/socketmodule.h
+++ b/Modules/socketmodule.h
@@ -144,6 +144,8 @@ typedef int socklen_t;
@@ -154,6 +154,8 @@ typedef int socklen_t;
extern "C" {
#endif

Expand All @@ -11,3 +17,6 @@ index fdb4e871ce..c990003ff3 100644
/* Python module and C API name */
#define PySocket_MODULE_NAME "_socket"
#define PySocket_CAPI_NAME "CAPI"
--
2.25.1

29 changes: 19 additions & 10 deletions cpython/patches/sqlite-MODULE_NAME.patch → ...hon/patches/0004-sqlite-MODULE_NAME.patch
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
diff --git a/Modules/_sqlite/Modules/_sqlite/cache.c b/Modules/_sqlite/cache.c
index 72b1f2c..2190bd4 100644
From c1b59989f88ba4911134539573e90d56da922e75 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Sun, 5 Jul 2020 19:55:57 +0200
Subject: [PATCH] sqlite-MODULE_NAME


diff --git a/Modules/_sqlite/cache.c b/Modules/_sqlite/cache.c
index 4d41804218..ff29a935c4 100644
--- a/Modules/_sqlite/cache.c
+++ b/Modules/_sqlite/cache.c
@@ -24,6 +24,10 @@
Expand All @@ -14,7 +20,7 @@ index 72b1f2c..2190bd4 100644
pysqlite_Node* pysqlite_new_node(PyObject* key, PyObject* data)
{
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 6e05761..2d1ad34 100644
index ebe073f644..daa3aad784 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -45,6 +45,10 @@
Expand All @@ -29,7 +35,7 @@ index 6e05761..2d1ad34 100644

static const char * const begin_statements[] = {
diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c
index 4ecb5b4..24b03c1 100644
index 01b9dc44cb..955da65f50 100644
--- a/Modules/_sqlite/cursor.c
+++ b/Modules/_sqlite/cursor.c
@@ -25,6 +25,10 @@
Expand All @@ -44,7 +50,7 @@ index 4ecb5b4..24b03c1 100644

static const char errmsg_fetch_across_rollback[] = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from.";
diff --git a/Modules/_sqlite/microprotocols.c b/Modules/_sqlite/microprotocols.c
index 3d01872..7f98c2f 100644
index c23b09f56b..48bb8c1290 100644
--- a/Modules/_sqlite/microprotocols.c
+++ b/Modules/_sqlite/microprotocols.c
@@ -30,6 +30,10 @@
Expand All @@ -59,7 +65,7 @@ index 3d01872..7f98c2f 100644
/** the adapters registry **/

diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c
index 6befa07..4611574 100644
index 9fe0dc952f..b325c56284 100644
--- a/Modules/_sqlite/module.c
+++ b/Modules/_sqlite/module.c
@@ -33,6 +33,10 @@
Expand All @@ -74,7 +80,7 @@ index 6befa07..4611574 100644

PyObject *pysqlite_Error = NULL;
diff --git a/Modules/_sqlite/prepare_protocol.c b/Modules/_sqlite/prepare_protocol.c
index f2c85f9..14969bb 100644
index 181c7edf96..643d776ae1 100644
--- a/Modules/_sqlite/prepare_protocol.c
+++ b/Modules/_sqlite/prepare_protocol.c
@@ -23,6 +23,10 @@
Expand All @@ -89,7 +95,7 @@ index f2c85f9..14969bb 100644
{
return 0;
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index ec2c788..ce65e62 100644
index 4b47108278..4a60a5f622 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -24,6 +24,10 @@
Expand All @@ -104,7 +110,7 @@ index ec2c788..ce65e62 100644
{
Py_XDECREF(self->data);
diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c
index 3869088..6d16049 100644
index 491294b020..2c48308486 100644
--- a/Modules/_sqlite/statement.c
+++ b/Modules/_sqlite/statement.c
@@ -28,6 +28,10 @@
Expand All @@ -119,7 +125,7 @@ index 3869088..6d16049 100644
static int pysqlite_check_remaining_sql(const char* tail);

diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c
index 3fa671d..bd12f51 100644
index 3fa671d052..bd12f51df1 100644
--- a/Modules/_sqlite/util.c
+++ b/Modules/_sqlite/util.c
@@ -24,6 +24,10 @@
Expand All @@ -133,3 +139,6 @@ index 3fa671d..bd12f51 100644
int pysqlite_step(sqlite3_stmt* statement, pysqlite_Connection* connection)
{
int rc;
--
2.25.1

48 changes: 28 additions & 20 deletions cpython/patches/add-emscripten-host.patch → ...on/patches/0005-add-emscripten-host.patch
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
From a6cf085d48b4783c997544f68636d0c4beb96401 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Sun, 5 Jul 2020 17:36:12 +0200
Subject: [PATCH] add-emscripten-host


diff --git a/config.sub b/config.sub
index ba37cf99e2..d772f33065 100755
index ba37cf99e2..c99e28a272 100755
--- a/config.sub
+++ b/config.sub
@@ -118,7 +118,8 @@ case $maybe_os in
Expand All @@ -16,7 +22,7 @@ index ba37cf99e2..d772f33065 100755
| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
| arm-* | armbe-* | armle-* | armeb-* | armv*-* \
+ | asmjs \
+ | asmjs \
| avr-* | avr32-* \
| ba-* \
| be32-* | be64-* \
Expand All @@ -30,22 +36,22 @@ index ba37cf99e2..d772f33065 100755
;;
-ios)
diff --git a/configure b/configure
index e0389649ae..e6ab1613cf 100755
index a979363acf..c4e0b866c3 100755
--- a/configure
+++ b/configure
@@ -3257,6 +3257,9 @@ then
*-*-cygwin*)
ac_sys_system=Cygwin
;;
@@ -3274,6 +3274,9 @@ then
*-*-vxworks*)
ac_sys_system=VxWorks
;;
+ asmjs-*-*)
+ ac_sys_system=Emscripten
+ ;;
*)
# for now, limit cross builds to known configurations
MACHDEP="unknown"
@@ -3301,6 +3304,9 @@ if test "$cross_compiling" = yes; then
*-*-cygwin*)
_host_cpu=
@@ -3324,6 +3327,9 @@ if test "$cross_compiling" = yes; then
*-*-vxworks*)
_host_cpu=$host_cpu
;;
+ asmjs-*-*)
+ _host_cpu=
Expand All @@ -54,26 +60,28 @@ index e0389649ae..e6ab1613cf 100755
# for now, limit cross builds to known configurations
MACHDEP="unknown"
diff --git a/configure.ac b/configure.ac
index b5beb08574..8051e5a8ed 100644
index e57ef7c38b..c1977f3a6f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -379,6 +379,9 @@ then
*-*-cygwin*)
ac_sys_system=Cygwin
;;
@@ -382,6 +382,9 @@ then
*-*-vxworks*)
ac_sys_system=VxWorks
;;
+ asmjs-*-*)
+ ac_sys_system=Emscripten
+ ;;
*)
# for now, limit cross builds to known configurations
MACHDEP="unknown"
@@ -423,6 +426,9 @@ if test "$cross_compiling" = yes; then
*-*-cygwin*)
_host_cpu=
@@ -430,6 +433,8 @@ if test "$cross_compiling" = yes; then
;;
*-*-vxworks*)
_host_cpu=$host_cpu
+ asmjs-*-*)
+ _host_cpu=
+ ;;
;;
*)
# for now, limit cross builds to known configurations
MACHDEP="unknown"
--
2.25.1

Loading

0 comments on commit fc5495f

Please sign in to comment.