Experimental new test system (pyodide#1047)

jmsmdy · Jan 11, 2021 · 65a9da0 · 65a9da0
1 parent 381997a
commit 65a9da0
Show file tree

Hide file tree

Showing 8 changed files with 165 additions and 47 deletions.
diff --git a/Makefile b/Makefile
@@ -113,10 +113,10 @@ test: all
 
 lint:
 	# check for unused imports, the rest is done by black
-	flake8 --select=F401 src tools pyodide_build benchmark
+	flake8 --select=F401 src tools pyodide_build benchmark conftest.py
 	clang-format-6.0 -output-replacements-xml `find src -type f -regex ".*\.\(c\|h\|js\)"` | (! grep '<replacement ')
 	black --check .
-	mypy --ignore-missing-imports pyodide_build/ src/ packages/micropip/micropip/ packages/*/test*
+	mypy --ignore-missing-imports pyodide_build/ src/ packages/micropip/micropip/ packages/*/test* conftest.py
 
 
 apply-lint:

diff --git a/conftest.py b/conftest.py
@@ -45,7 +45,7 @@ def pytest_addoption(parser):
 
 
 except ImportError:
-    pytest = None
+    pytest = None  # type: ignore
 
 
 class JavascriptException(Exception):
@@ -55,7 +55,7 @@ def __init__(self, msg, stack):
 
     def __str__(self):
         if self.stack:
-            return self.msg + "\n" + self.stack
+            return self.msg + "\n\n" + self.stack
         else:
             return self.msg
 

diff --git a/docs/testing.md b/docs/testing.md
@@ -71,3 +71,36 @@ To lint the code, run:
 ```bash
 make lint
 ```
+
+## Testing framework
+
+### run_in_pyodide
+Many tests simply involve running a chunk of code in pyodide and ensuring it
+doesn't error. In this case, one can use the `run_in_pyodide` decorate from
+`pyodide_build/testing.py`, e.g.
+
+```python
+from pyodide_build.testing import run_in_pyodide
+
+@run_in_pyodide
+def test_add():
+    assert 1 + 1 == 2
+```
+In this case, the body of the function will automatically be run in pyodide.
+The decorator can also be called with arguments. It has two configuration
+options --- standalone and packages.
+
+Setting `standalone = True` starts a standalone browser session to run the test
+(the session is shared between tests by default). This is useful for testing
+things like package loading.
+
+The `packages` option lists packages to load before running the test. For
+example,
+```python
+from pyodide_build.testing import run_in_pyodide
+
+@run_in_pyodide(standalone = True, packages = ["regex"])
+def test_regex():
+    import regex
+    assert regex.search("o", "foo").end() == 2
+```
diff --git a/packages/jedi/test_jedi.py b/packages/jedi/test_jedi.py
@@ -1,11 +1,10 @@
-def test_jedi(selenium_standalone):
-    selenium_standalone.load_package("jedi")
-    result = selenium_standalone.run(
-        """
-        import jedi
-        script = jedi.Script("import json\\njson.lo", path='example.py')
-        completions = script.complete(2, len('json.lo'))
-        [el.name for el in completions]
-        """
-    )
-    assert result == ["load", "loads"]
+from pyodide_build.testing import run_in_pyodide
+
+
+@run_in_pyodide(standalone=True, packages=["jedi"])
+def test_jedi():
+    import jedi
+
+    script = jedi.Script("import json\njson.lo", path="example.py")
+    completions = script.complete(2, len("json.lo"))
+    assert [el.name for el in completions] == ["load", "loads"]
diff --git a/packages/regex/test_regex.py b/packages/regex/test_regex.py
@@ -1,3 +1,8 @@
-def test_regex(selenium, request):
-    selenium.load_package("regex")
-    assert selenium.run("import regex\nregex.search('o', 'foo').end()") == 2
+from pyodide_build.testing import run_in_pyodide
+
+
+@run_in_pyodide(packages=["regex"])
+def test_regex():
+    import regex
+
+    assert regex.search("o", "foo").end() == 2
diff --git a/pyodide_build/testing.py b/pyodide_build/testing.py
@@ -0,0 +1,78 @@
+import pytest
+import inspect
+from typing import Optional, List, Callable
+
+
+def run_in_pyodide(
+    _function: Optional[Callable] = None,
+    standalone: bool = False,
+    packages: List[str] = [],
+) -> Callable:
+    """
+    This decorator can be called in two ways --- with arguments and without
+    arguments. If it is called without arguments, then the `_function` kwarg
+    catches the function the decorator is applied to. Otherewise, standalone
+    and packages are the actual arguments to the decorator.
+
+    See docs/testing.md for details on how to use this.
+
+    Parameters
+    ----------
+    standalone : bool, default=False
+        Whether to use a standalone selenium instance to run the test or not
+    packages : List[str]
+        List of packages to load before running the test
+    """
+
+    def decorator(f):
+        def inner(selenium):
+            if len(packages) > 0:
+                selenium.load_package(packages)
+            lines, start_line = inspect.getsourcelines(f)
+            # Remove first line, which is the decorator. Then pad with empty lines to fix line number.
+            lines = ["\n"] * start_line + lines[1:]
+            source = "".join(lines)
+
+            err = None
+            try:
+                # When writing the function, we set the filename to the file
+                # containing the source. This results in a more helpful
+                # traceback
+                selenium.run_js(
+                    """pyodide._module.pyodide_py.eval_code({!r}, pyodide._module.globals, "last_expr", true, {!r})""".format(
+                        source, inspect.getsourcefile(f)
+                    )
+                )
+                # When invoking the function, use the default filename <eval>
+                selenium.run_js(
+                    """pyodide._module.pyodide_py.eval_code("{}()", pyodide._module.globals)""".format(
+                        f.__name__
+                    )
+                )
+            except selenium.JavascriptException as e:
+                err = e
+
+            if err is not None:
+                pytest.fail(
+                    "Error running function in pyodide\n\n" + str(err),
+                    pytrace=False,
+                )
+
+        if standalone:
+
+            def wrapped_standalone(selenium_standalone):
+                inner(selenium_standalone)
+
+            return wrapped_standalone
+
+        else:
+
+            def wrapped(selenium):
+                inner(selenium)
+
+            return wrapped
+
+    if _function is not None:
+        return decorator(_function)
+    else:
+        return decorator
diff --git a/src/tests/test_bz2.py b/src/tests/test_bz2.py
@@ -1,12 +1,12 @@
-def test_bz2(selenium):
-    selenium.run(
-        """
-        import bz2
+from pyodide_build.testing import run_in_pyodide
 
-        text = "Hello test test test test this is a test test test"
-        some_compressed_bytes = bz2.compress(text.encode('utf-8'))
-        assert some_compressed_bytes != text
-        decompressed_bytes = bz2.decompress(some_compressed_bytes)
-        assert decompressed_bytes.decode('utf-8') == text
-    """
-    )
+
+@run_in_pyodide
+def test_bz2():
+    import bz2
+
+    text = "Hello test test test test this is a test test test"
+    some_compressed_bytes = bz2.compress(text.encode("utf-8"))
+    assert some_compressed_bytes != text
+    decompressed_bytes = bz2.decompress(some_compressed_bytes)
+    assert decompressed_bytes.decode("utf-8") == text
diff --git a/src/tests/test_sqlite3.py b/src/tests/test_sqlite3.py
@@ -1,23 +1,26 @@
-def test_sqlite3(selenium):
-    content = selenium.run(
+from pyodide_build.testing import run_in_pyodide
+
+
+@run_in_pyodide
+def test_sqlite3():
+    import sqlite3
+
+    with sqlite3.connect(":memory:") as conn:
+        c = conn.cursor()
+        c.execute(
+            """
+            CREATE TABLE people (
+                first_name VARCHAR,
+                last_name VARCHAR
+            )
         """
-        import sqlite3
+        )
+        c.execute("INSERT INTO people VALUES ('John', 'Doe')")
+        c.execute("INSERT INTO people VALUES ('Jane', 'Smith')")
+        c.execute("INSERT INTO people VALUES ('Michael', 'Jordan')")
+        c.execute("SELECT * FROM people")
 
-        with sqlite3.connect(':memory:') as conn:
-            c = conn.cursor()
-            c.execute('''
-                CREATE TABLE people (
-                    first_name VARCHAR,
-                    last_name VARCHAR
-                )
-            ''')
-            c.execute("INSERT INTO people VALUES ('John', 'Doe')")
-            c.execute("INSERT INTO people VALUES ('Jane', 'Smith')")
-            c.execute("INSERT INTO people VALUES ('Michael', 'Jordan')")
-            c.execute("SELECT * FROM people")
-    """
-    )
-    content = selenium.run("c.fetchall()")
+    content = c.fetchall()
     assert len(content) == 3
     assert content[0][0] == "John"
     assert content[1][0] == "Jane"