Python3.11 (pyodide#3252)

rsohlot · Jan 24, 2023 · 3b2952f · 3b2952f
1 parent 3f845c8
commit 3b2952f
Show file tree

Hide file tree

Showing 54 changed files with 256 additions and 4,922 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,7 +6,7 @@ defaults: &defaults
     # Note: when updating the docker image version,
     #       make sure there are no extra old versions lying around.
     #       (e.g. `rg -F --hidden <old_tag>`)
-    - image: pyodide/pyodide-env:20230110-chrome108-firefox107
+    - image: pyodide/pyodide-env:20230123-chrome107-firefox109-py311
   environment:
     - EMSDK_NUM_CORES: 3
       EMCC_CORES: 3
@@ -293,12 +293,12 @@ jobs:
           command: |
             curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
             bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh
-            ~/miniforge3/bin/conda create -n pyodide python=3.10 -y
+            ~/miniforge3/bin/conda create -n pyodide python=3.11 -y
       - run:
           name: install dependencies
           command: |
             export PATH="$HOME/miniforge3/bin:$PATH"
-            conda create -n pyodide python=3.10 -y
+            conda create -n pyodide python=3.11 -y
             source activate pyodide
             python -m pip install -r requirements.txt
             pip install -e ./pyodide-build
@@ -334,7 +334,7 @@ jobs:
             make npm-link
             pip install -r requirements.txt
             npm install -g node-fetch@2
-            pytest -s benchmark/stack_usage.py  | sed -n 's/## //pg'
+            pytest -s benchmark/stack_usage.py --rt node chrome firefox | sed -n 's/## //pg'
 
   test-js:
     <<: *defaults
@@ -382,7 +382,7 @@ jobs:
             git clone https://github.com/python-attrs/attrs --depth 1 --branch 22.1.0
             git clone https://github.com/zopefoundation/zope.interface.git --depth 1 --branch 5.4.0
 
-            python3.10 -m venv .venv-host
+            python -m venv .venv-host
             source .venv-host/bin/activate
             pip install $PYODIDE_ROOT/pyodide-build
             pyodide venv .venv-pyodide

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 concurrency:
   group: main-${{ github.head_ref }}
@@ -17,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.10"
+          python-version: 3.11.1
       - uses: pre-commit/action@v2.0.3
 
   test-docs:
@@ -26,7 +25,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.10"
+          python-version: 3.11.1
       - name: Install requirements
         shell: bash -l {0}
         run: |
@@ -49,7 +48,7 @@ jobs:
 
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.10"
+          python-version: 3.11.1
 
       - name: Install requirements
         shell: bash -l {0}
@@ -91,7 +90,7 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: pyodide-env
-          python-version: 3.10.2
+          python-version: "3.11"
           channels: conda-forge
 
       - name: Check Python versions
@@ -157,7 +156,7 @@ jobs:
       fail-fast: false
       matrix:
         runner: [playwright]
-        browser: [firefox, chrome]
+        browser: [chrome]
 
     steps:
       - uses: actions/checkout@v2
@@ -171,16 +170,15 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: pyodide-env
-          python-version: 3.10.2
+          python-version: "3.11"
           channels: conda-forge
 
       - name: install test requirements
         shell: bash -l {0}
         run: |
           pip install -r requirements.txt
           pip install -e ./pyodide-build
-          # FIXME: playwright 1.23.0 has unknown performance issue on firefox
-          pip install "playwright<1.23.0" && python -m playwright install
+          python -m playwright install
 
       - name: run core tests
         env:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 exclude: (^.*patches|.*\.cgi$|^packages/micropip/src/micropip/externals|^benchmark/benchmarks$)
 default_language_version:
-  python: "3.10"
+  python: "3.11"
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: "v4.4.0"

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -17,5 +17,5 @@ python:
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.10"
+    python: "3.11"
     nodejs: "14"
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 FROM node:14.16.1-buster-slim AS node-image
-FROM python:3.10.2-slim-buster
+FROM python:3.11.1-slim-buster
 
 # Requirements for building packages
 RUN apt-get update \

diff --git a/Makefile b/Makefile
@@ -115,7 +115,8 @@ src/js/pyproxy.gen.ts : src/core/pyproxy.* src/core/*.h
 	echo "// Do not edit it directly!" >> $@
 	cat src/core/pyproxy.ts | \
 		sed '/^\/\/\s*pyodide-skip/,/^\/\/\s*end-pyodide-skip/d' | \
-		$(CC) -E -C -P -imacros src/core/pyproxy.c $(MAIN_MODULE_CFLAGS) - \
+		$(CC) -E -C -P -imacros src/core/pyproxy.c $(MAIN_MODULE_CFLAGS) - | \
+		sed 's/^#pragma clang.*//g' \
 		>> $@
 
 dist/test.html: src/templates/test.html

diff --git a/Makefile.envs b/Makefile.envs
@@ -1,4 +1,4 @@
-export PYVERSION ?= 3.10.2
+export PYVERSION ?= 3.11.1
 export PYODIDE_EMSCRIPTEN_VERSION ?= 3.1.30
 
 export PLATFORM_TRIPLET=wasm32-emscripten

diff --git a/benchmark/stack_usage.py b/benchmark/stack_usage.py
@@ -39,10 +39,16 @@ def test_stack_usage(selenium, print_info):
             return depth;
         };
         let py_limit = pyodide.runPython("import sys; sys.getrecursionlimit()");
+        self.jsrecurse = function(f, g) {
+            return (n) => (n > 0) ? f(n-1) : g();
+        }
         let py_usage = pyodide.runPython(`
-            from js import measure_available_js_stack_depth
+            from js import measure_available_js_stack_depth, jsrecurse
+            from pyodide.ffi import create_proxy
+            recurse_proxy = None
             def recurse(n):
-                return measure_available_js_stack_depth() if n==0 else recurse(n-1)
+                return jsrecurse(recurse_proxy, measure_available_js_stack_depth)(n)
+            recurse_proxy = create_proxy(recurse)
             (recurse(0)-recurse(100))/100
         `);
         let js_depth = measure_available_js_stack_depth();

diff --git a/cpython/Makefile b/cpython/Makefile
@@ -3,7 +3,7 @@ include ../Makefile.envs
 
 ROOT=$(abspath .)
 
-PYTHON_CFLAGS=$(CFLAGS_BASE)
+PYTHON_CFLAGS=$(CFLAGS_BASE) -DPY_CALL_TRAMPOLINE
 
 BUILD=$(CPYTHONROOT)/build/Python-$(PYVERSION)
 INSTALL=$(CPYTHONINSTALL)
@@ -22,6 +22,7 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) remove_modules.txt
 	( \
 		cd $(BUILD); \
 		sed -i -e 's/libinstall:.*/libinstall:/' Makefile; \
+		sed -i '/MODOBJS=/s/$$/ $$(LIBMPDEC_OBJS) $$(LIBEXPAT_OBJS) /' Makefile; \
 		touch $(BUILD)/$(LIB) ; \
 		emmake make PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) -j $${PYODIDE_JOBS:-3} && \
 		cp $(LIB) $(INSTALL)/lib/ \
@@ -59,7 +60,6 @@ $(BUILD)/.patched: $(TARBALL)
 	cat patches/*.patch | (cd $(BUILD) ; patch -p1)
 	touch $@
 
-
 $(INSTALL)/lib/libffi.a :
 	rm -rf $(FFIBUILD)
 	git clone $(LIBFFIREPO) --depth 1 --branch $(LIBFFI_TAG) $(FFIBUILD)
@@ -76,10 +76,9 @@ $(BUILD)/Makefile: $(BUILD)/.patched
 	#   30 on systems with 16 bit pointers. WASM has 32 bit pointers so Python
 	#   will default to the size 15 digits but WASM has native 64 bit arithmetic
 	#   so it is more efficient to use 30 bit digits.
-	cp config.site $(BUILD)/
 	( \
 		cd $(BUILD); \
-		CONFIG_SITE=./config.site READELF=true emconfigure \
+		CONFIG_SITE=./Tools/wasm/config.site-wasm32-emscripten READELF=true emconfigure \
 		  ./configure \
 			  CFLAGS="${PYTHON_CFLAGS}" \
 			  CPPFLAGS="-sUSE_BZIP2=1 -sUSE_ZLIB=1" \
@@ -92,15 +91,15 @@ $(BUILD)/Makefile: $(BUILD)/.patched
 			  --host=wasm32-unknown-emscripten\
 			  --build=$(shell $(BUILD)/config.guess) \
 			  --prefix=$(INSTALL)  \
+			  --with-build-python=$$(which python3.11) \
 	)
 
 
-$(BUILD)/Modules/Setup.local : Setup.local
+$(BUILD)/Modules/Setup.local: Setup.local
 	cp Setup.local $(BUILD)/Modules/
 
 $(BUILD)/$(LIB): $(BUILD)/Makefile $(BUILD)/pyconfig.h $(BUILD)/Modules/Setup.local $(INSTALL)/lib/libffi.a
 	cp Setup.local $(BUILD)/Modules/
-	cat pyconfig.undefs.h >> $(BUILD)/pyconfig.h
 	( \
 		cd $(BUILD); \
 		emmake make CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \

diff --git a/cpython/Setup.local b/cpython/Setup.local
@@ -6,59 +6,21 @@
 
 *static*
 
-array arraymodule.c	# array objects
-audioop audioop.c	# Operations on audio samples
-math mathmodule.c _math.c # -lm # math library functions, e.g. sin()
-cmath cmathmodule.c # complex math functions
-_contextvars _contextvarsmodule.c
-_struct _struct.c	# binary structure packing/unpacking
-_random _randommodule.c	# Random number generator
-_bisect _bisectmodule.c	# Bisection algorithms
-_datetime _datetimemodule.c
-_heapq _heapqmodule.c
-_json _json.c
-_csv _csv.c
-
-CTYPES_FLAGS=-DHAVE_FFI_PREP_CIF_VAR=1 -DHAVE_FFI_PREP_CLOSURE_LOC=1 -DHAVE_FFI_CLOSURE_ALLOC=1
+CTYPES_FLAGS=-DHAVE_FFI_PREP_CIF_VAR=1 -DHAVE_FFI_PREP_CLOSURE_LOC=1 -DHAVE_FFI_CLOSURE_ALLOC=1 -I ../libffi/include
 _ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/cfield.c _ctypes/stgdict.c $(CTYPES_FLAGS)
 
-unicodedata unicodedata.c
-_pickle _pickle.c
-
-_socket socketmodule.c
-select selectmodule.c
 _posixsubprocess _posixsubprocess.c
-binascii binascii.c
-
-zlib zlibmodule.c
-
-pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI -DXML_POOR_ENTROPY
-
-_sha1 sha1module.c
-_sha256 sha256module.c
-_sha512 sha512module.c
-_sha3 _sha3/sha3module.c
-_md5 md5module.c
-_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
 
-_crypt _cryptmodule.c
 _bz2 _bz2module.c -lbz2
+zlib zlibmodule.c
 
-_queue _queuemodule.c
-
-#future_builtins future_builtins.c
-
-_multibytecodec cjkcodecs/multibytecodec.c
-
-_codecs_cn cjkcodecs/_codecs_cn.c
-_codecs_hk cjkcodecs/_codecs_hk.c
-_codecs_iso2022 cjkcodecs/_codecs_iso2022.c
-_codecs_jp cjkcodecs/_codecs_jp.c
-_codecs_kr cjkcodecs/_codecs_kr.c
-_codecs_tw cjkcodecs/_codecs_tw.c
-
-_lsprof _lsprof.c rotatingtree.c
 
-_decimal _decimal/_decimal.c _decimal/libmpdec/basearith.c _decimal/libmpdec/constants.c _decimal/libmpdec/context.c _decimal/libmpdec/convolute.c _decimal/libmpdec/crt.c _decimal/libmpdec/difradix2.c _decimal/libmpdec/fnt.c _decimal/libmpdec/fourstep.c _decimal/libmpdec/io.c _decimal/libmpdec/mpalloc.c _decimal/libmpdec/mpdecimal.c _decimal/libmpdec/numbertheory.c _decimal/libmpdec/sixstep.c _decimal/libmpdec/transpose.c -I$(srcdir)/Modules/_decimal/libmpdec
-mmap mmapmodule.c
 _xxsubinterpreters _xxsubinterpretersmodule.c
+
+*disabled*
+pwd
+_sqlite3
+_ssl
+_lzma
+_hashlib
+_uuid
diff --git a/cpython/checksums b/cpython/checksums
@@ -1 +1 @@
-3c0ede893011319f9b0a56b44953a3d52c7abf9657c23fb4bc9ced93b86e9c97  downloads/Python-3.10.2.tgz
+baed518e26b337d4d8105679caf68c5c32630d702614fc174e98cb95c46bdfa4  downloads/Python-3.11.1.tgz
diff --git a/cpython/config.site b/cpython/config.site
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		3c0ede893011319f9b0a56b44953a3d52c7abf9657c23fb4bc9ced93b86e9c97 downloads/Python-3.10.2.tgz
		baed518e26b337d4d8105679caf68c5c32630d702614fc174e98cb95c46bdfa4 downloads/Python-3.11.1.tgz