Merge https://github.com/mypyc/mypyc into merge-mypyc

The mypyc history was rewritten before this merge to fix up PR and issue references to be cross-repo references. The command was `git filter-branch -f --msg-filter 'sed "s@\(^\|[ (]\)\(#[0-9]\+\)@\1mypyc/mypyc\2@g"'`
elazarg · Aug 21, 2019 · 43461c1 · 43461c1
2 parents afb1284 + 72eaed6
commit 43461c1
Show file tree

Hide file tree

Showing 142 changed files with 70,851 additions and 32 deletions.
diff --git a/.gitignore b/.gitignore
@@ -43,3 +43,12 @@ include/
 
 .tox
 pip-wheel-metadata
+
+
+test_capi
+*.o
+*.a
+test_capi
+/.mypyc-flake8-cache.json
+/mypyc/lib-rt/build/
+/mypyc/lib-rt/*.so
diff --git a/.travis.yml b/.travis.yml
@@ -4,13 +4,22 @@ if: tag IS present OR type = pull_request OR ((branch = master OR branch =~ rele
 language: python
 # cache package wheels (1 cache per python version)
 cache: pip
+# also cache the directories where we set up our custom pythons in some builds
+cache:
+  directories:
+    - $HOME/python-debug
+    # I ran into some issues with this but will investigate again later.
+    # - $HOME/.pyenv/versions
+    - $HOME/Library/Caches/pip
+
 # newer python versions are available only on xenial (while some older only on trusty) Ubuntu distribution
 dist: xenial
 
 env:
   TOXENV=py
   EXTRA_ARGS="-n 12"
   TEST_MYPYC=0
+  PYTHON_DEBUG_BUILD=0
 
 jobs:
   include:
@@ -22,6 +31,20 @@ jobs:
     python: 3.6    # 3.6.3  pip  9.0.1
   - name: "run test suite with python 3.7"
     python: 3.7    # 3.7.0  pip 10.0.1
+  - name: "run mypyc runtime tests with python 3.6 debug build"
+    language: generic
+    env:
+      - TOXENV=py36
+      - PYTHONVERSION=3.6.8
+      - PYTHON_DEBUG_BUILD=1
+      - EXTRA_ARGS="-n 12 mypyc/test/test_run.py mypyc/test/test_external.py"
+  - name: "run mypyc runtime tests with python 3.6 on OS X"
+    os: osx
+    osx_image: xcode8.3
+    language: generic
+    env:
+      - PYTHONVERSION=3.6.3
+      - EXTRA_ARGS="-n 12 mypyc/test/test_run.py mypyc/test/test_external.py"
   - name: "run test suite with python 3.7 (compiled with mypyc)"
     python: 3.7
     env:
@@ -58,11 +81,33 @@ install:
 - pip install -U pip setuptools
 - pip install -U tox==3.9.0
 - tox --notest
+
 # This is a big hack and only works because the layout of our directories
 # means that tox picks up the mypy from the source directories instead of
 # the version it installed into a venv. This is also *why* we need to do this,
 # since if we arranged for tox to build with mypyc, pytest wouldn't use it.
-- if [[ $TEST_MYPYC == 1 ]]; then pip install -r mypyc-requirements.txt; CC=clang MYPYC_OPT_LEVEL=0 python3 setup.py --use-mypyc build_ext --inplace; fi
+- if [[ $TEST_MYPYC == 1 ]]; then pip install -r test-requirements.txt; CC=clang MYPYC_OPT_LEVEL=0 python3 setup.py --use-mypyc build_ext --inplace; fi
 
 script:
 - tox -- $EXTRA_ARGS
+
+# Getting our hands on a debug build or the right OS X build is
+# annoying, unfortunately.
+before_install: |
+  set -e
+  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    if [[ $PYTHON_DEBUG_BUILD == 1 ]]; then
+      PYTHONDIR=~/python-debug/python-$PYTHONVERSION
+      VENV=$PYTHONDIR/env
+      misc/build-debug-python.sh $PYTHONVERSION $PYTHONDIR $VENV
+      source $VENV/bin/activate
+    fi
+  elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    # Attempt to install, skipping if version already exists.
+    pyenv install $PYTHONVERSION -s
+    # Regenerate shims
+    pyenv rehash
+    # Manually set pyenv variables per https://pythonhosted.org/CodeChat/.travis.yml.html
+    export PYENV_VERSION=$PYTHONVERSION
+    export PATH="/Users/travis/.pyenv/shims:${PATH}"
+  fi
diff --git a/LICENSE b/LICENSE
@@ -1,10 +1,10 @@
-Mypy is licensed under the terms of the MIT license, reproduced below.
+Mypy (and mypyc) are licensed under the terms of the MIT license, reproduced below.
 
 = = = = =
 
 The MIT License
 
-Copyright (c) 2015-2016 Jukka Lehtosalo and contributors
+Copyright (c) 2015-2019 Jukka Lehtosalo and contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the "Software"),
@@ -26,8 +26,10 @@ DEALINGS IN THE SOFTWARE.
 
 = = = = =
 
-Portions of mypy are licensed under different licenses.  The files
-under stdlib-samples are licensed under the PSF 2 License, reproduced below.
+Portions of mypy and mypyc are licensed under different licenses.  The
+files under stdlib-samples as well as the files
+mypyc/lib-rt/pythonsupport.h and mypyc/lib-rt/getargs.c are licensed
+under the PSF 2 License, reproduced below.
 
 = = = = =
 
@@ -223,5 +225,3 @@ FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
 OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-= = = = =
diff --git a/README.md b/README.md
@@ -265,9 +265,10 @@ Compiled version of mypy
 ------------------------
 
 We have built an compiled version of mypy using the [mypyc
-compiler](https://github.com/mypyc/mypyc) for mypy-annotated Python
-code. It is approximately 4 times faster than interpreted mypy and is
-available (and the default) for 64-bit Windows, macOS, and Linux.
+compiler](https://github.com/python/mypy/files/mypyc) for
+mypy-annotated Python code. It is approximately 4 times faster than
+interpreted mypy and is available (and the default) for 64-bit
+Windows, macOS, and Linux.
 
 To install an interpreted mypy instead, use:
 

diff --git a/appveyor.yml b/appveyor.yml
@@ -3,10 +3,14 @@ cache:
 
 environment:
   matrix:
-
     - PYTHON: "C:\\Python37-x64"
       PYTHON_VERSION: "3.7.x"
       PYTHON_ARCH: "64"
+      EXTRA_ARGS:
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7.x"
+      PYTHON_ARCH: "32"
+      EXTRA_ARGS: "mypyc/test/test_run.py mypyc/test/test_external.py"
 
 install:
     - "git submodule update --init mypy/typeshed"
@@ -16,7 +20,7 @@ install:
 build: off
 
 test_script:
-    - "%PYTHON%\\python.exe -m tox -e py37"
+    - "%PYTHON%\\python.exe -m tox -e py37 %EXTRA_ARGS%"
 
 skip_commits:
   files:

diff --git a/conftest.py b/conftest.py
@@ -9,3 +9,10 @@ def pytest_configure(config):
     mypy_source_root = os.path.dirname(os.path.abspath(__file__))
     if os.getcwd() != mypy_source_root:
         os.chdir(mypy_source_root)
+
+
+# This function name is special to pytest.  See
+# http://doc.pytest.org/en/latest/writing_plugins.html#initialization-command-line-and-configuration-hooks
+def pytest_addoption(parser) -> None:
+    parser.addoption('--bench', action='store_true', default=False,
+                     help='Enable the benchmark test runs')
diff --git a/misc/build-debug-python.sh b/misc/build-debug-python.sh
@@ -0,0 +1,34 @@
+#!/bin/bash -eux
+
+# Build a debug build of python, install it, and create a venv for it
+# This is mainly intended for use in our travis builds but it can work
+# locally. (Though it unfortunately uses brew on OS X to deal with openssl
+# nonsense.)
+# Usage: build-debug-python.sh <version> <install prefix> <venv location>
+#
+# Running it locally might look something like: mkdir -p ~/tmp/cpython-debug && cd ~/tmp/cpython-debug && ~/src/mypy/misc/build-debug-python.sh 3.6.6 ~/tmp/cpython-debug ~/src/mypy/env-debug
+
+VERSION=$1
+PREFIX=$2
+VENV=$3
+if [[ -f $PREFIX/bin/python3 ]]; then
+    exit
+fi
+
+CPPFLAGS=""
+LDFLAGS=""
+if [[ $(uname) == Darwin ]]; then
+    brew install openssl xz
+    CPPFLAGS="-I$(brew --prefix openssl)/include"
+    LDFLAGS="-L$(brew --prefix openssl)/lib"
+fi
+
+curl -O https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tgz
+tar zxf Python-$VERSION.tgz
+cd Python-$VERSION
+CPPFLAGS="$CPPFLAGS" LDFLAGS="$LDFLAGS" ./configure CFLAGS="-DPy_DEBUG -DPy_TRACE_REFS -DPYMALLOC_DEBUG" --with-pydebug --prefix=$PREFIX
+make -j4
+make install
+$PREFIX/bin/python3 -m pip install virtualenv
+$PREFIX/bin/python3 -m virtualenv $VENV
+ln -s python3-config $PREFIX/bin/python-config
diff --git a/misc/clean-mypyc.sh b/misc/clean-mypyc.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "Cleaning C/C++ build artifacts..."
+(cd mypyc/lib-rt; make clean)
+(cd mypyc/external/googletest/make; make clean)
diff --git a/mypyc-requirements.txt b/mypyc-requirements.txt
diff --git a/mypyc/README.md b/mypyc/README.md
@@ -0,0 +1,134 @@
+mypyc: Mypy to Python C Extension Compiler
+==========================================
+
+*Mypyc is (mostly) not yet useful for general Python development.*
+
+Mypyc is a compiler that compiles mypy-annotated, statically typed
+Python modules into CPython C extensions. Currently our primary focus
+is on making mypy faster through compilation -- the default mypy wheels
+are compiled with mypyc.  Compiled mypy is about 4x faster than
+without compilation.
+
+Mypyc compiles what is essentially a Python language variant using "strict"
+semantics. This means (among some other things):
+
+ * Most type annotations are enforced at runtime (raising ``TypeError`` on mismatch)
+
+ * Classes are compiled into extension classes without ``__dict__``
+   (much, but not quite, like if they used ``__slots__``)
+
+ * Monkey patching doesn't work
+
+ * Instance attributes won't fall back to class attributes if undefined
+
+ * Metaclasses not supported
+
+ * Also there are still a bunch of bad bugs and unsupported features :)
+
+Compiled modules can import arbitrary Python modules, and compiled modules
+can be used from other Python modules.  Typically mypyc is used to only
+compile modules that contain performance bottlenecks.
+
+You can run compiled modules also as normal, interpreted Python
+modules, since mypyc targets valid Python code. This means that
+all Python developer tools and debuggers can be used.
+
+macOS Requirements
+------------------
+
+* macOS Sierra or later
+
+* Xcode command line tools
+
+* Python 3.5+ from python.org (other versions are untested)
+
+Linux Requirements
+------------------
+
+* A recent enough C/C++ build environment
+
+* Python 3.5+
+
+Windows Requirements
+--------------------
+
+* Windows has been tested with Windows 10 and MSVC 2017.
+
+* Python 3.5+
+
+Quick Start for Contributors
+----------------------------
+
+First clone the mypy git repository *and git submodules*:
+
+    $ git clone --recurse-submodules https://github.com/mypyc/mypy.git
+    $ cd mypy
+
+Optionally create a virtualenv (recommended):
+
+    $ virtualenv -p python3 <directory>
+    $ source <directory>/bin/activate
+
+Then install the dependencies:
+
+    $ python3 -m pip install -r test-requirements.txt
+
+Now you can run the tests:
+
+    $ pytest mypyc
+
+Look at the [issue tracker](https://github.com/mypyc/mypyc/issues)
+for things to work on. Please express your interest in working on an
+issue by adding a comment before doing any significant work, since
+development is currently very active and there is real risk of duplicate
+work.
+
+Note that the issue tracker is still hosted on the mypyc project, not
+with mypy itself.
+
+Documentation
+-------------
+
+We have some [developer documentation](doc/dev-intro.md).
+
+Development Status and Roadmap
+------------------------------
+
+These are the current planned major milestones:
+
+1. [DONE] Support a smallish but useful Python subset. Focus on compiling
+   single modules, while the rest of the program is interpreted and does not
+   need to be type checked.
+
+2. [DONE] Support compiling multiple modules as a single compilation unit (or
+   dynamic linking of compiled modules).  Without this inter-module
+   calls will use slower Python-level objects, wrapper functions and
+   Python namespaces.
+
+3. [DONE] Mypyc can compile mypy.
+
+4. [DONE] Optimize some important performance bottlenecks.
+
+5. [PARTIALLY DONE] Generate useful errors for code that uses unsupported Python
+   features instead of crashing or generating bad code.
+
+6. [DONE] Release a version of mypy that includes a compiled mypy.
+
+7a. More feature/compatibility work. (100% compatibility with Python is distinctly
+    an anti-goal, but more than we have now is a good idea.)
+
+7b. Support compiling Black, which is a prominent tool that could benefit
+    and has maintainer buy-in.
+    (Let us know if you maintain another Python tool or library and are
+     interested in working with us on this!)
+
+7c. More optimization! Code size reductions in particular are likely to
+    be valuable and will speed up mypyc compilation.
+
+8.  We'll see! Adventure is out there!
+
+Future
+------
+
+We have some ideas for
+[future improvements and optimizations](doc/future.md).
diff --git a/mypyc/__init__.py b/mypyc/__init__.py
@@ -0,0 +1 @@
+