Use package_data to ship typeshed and xml files (python#5517)

Data files have caused many issues where mypy cannot find typeshed. Using package_data allows us to find typeshed relative to the mypy package itself, which avoids a lot of fragile path computation. Fixes python#5307 Fixes python#5319
elazarg · Aug 31, 2018 · 5e8815e · 5e8815e
1 parent 2b15be3
commit 5e8815e
Show file tree

Hide file tree

Showing 11 changed files with 19 additions and 79 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "typeshed"]
-	path = typeshed
+	path = mypy/typeshed
 	url = https://github.com/python/typeshed
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -2,5 +2,7 @@ recursive-include scripts *
 recursive-include test-data *
 recursive-include extensions *
 recursive-include docs *
+recursive-include mypy/typeshed *.py *.pyi
+recursive-include mypy/xml *.xsd *.xslt *.css
 include mypy_self_check.ini
 include LICENSE
diff --git a/appveyor.yml b/appveyor.yml
@@ -9,10 +9,7 @@ environment:
       PYTHON_ARCH: "64"
 
 install:
-    - "git config core.symlinks true"
-    - "git reset --hard"
-    - "git submodule update --init typeshed"
-    - "cd typeshed && git config core.symlinks true && git reset --hard && cd .."
+    - "git submodule update --init mypy/typeshed"
     - "%PYTHON%\\python.exe -m pip install -U setuptools tox"
     - "%PYTHON%\\python.exe -m tox -e py37 --notest"
 

diff --git a/mypy/build.py b/mypy/build.py
@@ -20,6 +20,7 @@
 import hashlib
 import json
 import os.path
+import pathlib
 import re
 import site
 import stat
@@ -326,7 +327,7 @@ def _build(sources: List[BuildSource],
     # This seems the most reasonable place to tune garbage collection.
     gc.set_threshold(50000)
 
-    data_dir = default_data_dir(bin_dir)
+    data_dir = default_data_dir()
     fscache = fscache or FileSystemCache()
 
     search_paths = compute_search_paths(sources, options, data_dir, fscache, alt_lib_path)
@@ -365,68 +366,9 @@ def _build(sources: List[BuildSource],
         reports.finish()
 
 
-def default_data_dir(bin_dir: Optional[str]) -> str:
-    """Returns directory containing typeshed directory
-
-    Args:
-      bin_dir: directory containing the mypy script
-    """
-    if not bin_dir:
-        if os.name == 'nt':
-            prefixes = [os.path.join(sys.prefix, 'Lib')]
-            try:
-                prefixes.append(os.path.join(site.getuserbase(), 'lib'))
-            except AttributeError:
-                # getuserbase in not available in virtualenvs
-                prefixes.append(os.path.join(get_python_lib(), 'lib'))
-            for parent in prefixes:
-                    data_dir = os.path.join(parent, 'mypy')
-                    if os.path.exists(data_dir):
-                        return data_dir
-        mypy_package = os.path.dirname(__file__)
-        parent = os.path.dirname(mypy_package)
-        if (os.path.basename(parent) == 'site-packages' or
-                os.path.basename(parent) == 'dist-packages'):
-            # Installed in site-packages or dist-packages, but invoked with python3 -m mypy;
-            # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py
-            # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian)
-            # or .../blah/lib64/python3.N/dist-packages/mypy/build.py (Gentoo)
-            # or .../blah/lib/site-packages/mypy/build.py (Windows)
-            # blah may be a virtualenv or /usr/local.  We want .../blah/lib/mypy.
-            lib = parent
-            for i in range(2):
-                lib = os.path.dirname(lib)
-                if os.path.basename(lib) in ('lib', 'lib32', 'lib64'):
-                    return os.path.join(os.path.dirname(lib), 'lib/mypy')
-        subdir = os.path.join(parent, 'lib', 'mypy')
-        if os.path.isdir(subdir):
-            # If installed via buildout, the __file__ is
-            # somewhere/mypy/__init__.py and what we want is
-            # somewhere/lib/mypy.
-            return subdir
-        # Default to directory containing this file's parent.
-        return parent
-    base = os.path.basename(bin_dir)
-    dir = os.path.dirname(bin_dir)
-    if (sys.platform == 'win32' and base.lower() == 'scripts'
-            and not os.path.isdir(os.path.join(dir, 'typeshed'))):
-        # Installed, on Windows.
-        return os.path.join(dir, 'Lib', 'mypy')
-    elif base == 'scripts':
-        # Assume that we have a repo check out or unpacked source tarball.
-        return dir
-    elif base == 'bin':
-        # Installed to somewhere (can be under /usr/local or anywhere).
-        return os.path.join(dir, 'lib', 'mypy')
-    elif base == 'python3':
-        # Assume we installed python3 with brew on os x
-        return os.path.join(os.path.dirname(dir), 'lib', 'mypy')
-    elif dir.endswith('python-exec'):
-        # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink.
-        return os.path.join(os.path.dirname(dir), 'mypy')
-    else:
-        # Don't know where to find the data files!
-        raise RuntimeError("Broken installation: can't determine base dir")
+def default_data_dir() -> str:
+    """Returns directory containing typeshed directory."""
+    return os.path.dirname(__file__)
 
 
 def mypy_path() -> List[str]:

diff --git a/typeshed → mypy/typeshed b/typeshed → mypy/typeshed
diff --git a/xml/mypy-html.css → mypy/xml/mypy-html.css b/xml/mypy-html.css → mypy/xml/mypy-html.css
diff --git a/xml/mypy-html.xslt → mypy/xml/mypy-html.xslt b/xml/mypy-html.xslt → mypy/xml/mypy-html.xslt
diff --git a/xml/mypy-txt.xslt → mypy/xml/mypy-txt.xslt b/xml/mypy-txt.xslt → mypy/xml/mypy-txt.xslt
diff --git a/xml/mypy.xsd → mypy/xml/mypy.xsd b/xml/mypy.xsd → mypy/xml/mypy.xsd
diff --git a/setup.cfg b/setup.cfg
@@ -22,7 +22,7 @@ exclude =
   # fixtures have their own .pyi-specific configuration
   test-data/*,
   # typeshed has its own .pyi-specific configuration
-  typeshed/*,
+  mypy/typeshed/*,
   # flake8 might be started when there's still examples in the temp dir
   tmp-test-dirs/*
   .tox

diff --git a/setup.py b/setup.py
@@ -33,8 +33,8 @@
 '''.lstrip()
 
 
-def find_data_files(base, globs):
-    """Find all interesting data files, for setup(data_files=)
+def find_package_data(base, globs):
+    """Find all interesting data files, for setup(package_data=)
 
     Arguments:
       root:  The directory to search in.
@@ -49,9 +49,7 @@ def find_data_files(base, globs):
             files += glob.glob(os.path.join(rv_dir, pat))
         if not files:
             continue
-        target = os.path.join('lib', 'mypy', rv_dir)
-        rv.append((target, files))
-
+        rv.extend([f[5:] for f in files])
     return rv
 
 
@@ -67,11 +65,12 @@ def run(self):
         build_py.run(self)
 
 
-data_files = []
+package_data = ['py.typed']
+
+package_data += find_package_data(os.path.join('mypy', 'typeshed'), ['*.py', '*.pyi'])
 
-data_files += find_data_files('typeshed', ['*.py', '*.pyi'])
+package_data += find_package_data(os.path.join('mypy', 'xml'), ['*.xsd', '*.xslt', '*.css'])
 
-data_files += find_data_files('xml', ['*.xsd', '*.xslt', '*.css'])
 
 classifiers = [
     'Development Status :: 3 - Alpha',
@@ -95,12 +94,11 @@ def run(self):
       license='MIT License',
       py_modules=[],
       packages=['mypy', 'mypy.test', 'mypy.server', 'mypy.plugins'],
-      package_data={'mypy': ['py.typed']},
+      package_data={'mypy': package_data},
       entry_points={'console_scripts': ['mypy=mypy.__main__:console_entry',
                                         'stubgen=mypy.stubgen:main',
                                         'dmypy=mypy.dmypy:main',
                                         ]},
-      data_files=data_files,
       classifiers=classifiers,
       cmdclass={'build_py': CustomPythonBuild},
       install_requires = ['typed-ast >= 1.1.0, < 1.2.0',
@@ -110,4 +108,5 @@ def run(self):
           ':python_version < "3.5"': 'typing >= 3.5.3',
           'dmypy': 'psutil >= 5.4.0, < 5.5.0; sys_platform!="win32"',
       },
+      include_package_data=True,
       )