Skip to content

Commit

Permalink
Check for module presence of least used external modules
Browse files Browse the repository at this point in the history
  • Loading branch information
@MiguelMarcelino
MiguelMarcelino committed Sep 22, 2022
1 parent ac11b64 commit a8df62e
Show file tree
Hide file tree
Showing 9 changed files with 158 additions and 124 deletions.
83 changes: 46 additions & 37 deletions pyjl/external/modules/ctypes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,36 +108,7 @@ def visit_Libdl(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[str,s
"LPCWSTR": lambda node, vargs, kwargs: f"isa({vargs[0]}, String) ? Cwstring(pointer(transcode(Cwchar_t, {vargs[0]}))) : Cwstring(Ptr{{Cwchar_t}}({vargs[0]}))"
}

if sys.platform.startswith('win32'):
# Import windows ctypes modules
from builtins import WindowsError
from ctypes import wintypes

WIN_SMALL_DISPATCH_MAP = {
"GetLastError": lambda node, vargs, kwargs: "Base.Libc.GetLastError"
if getattr(node, "is_attr", None)
else "Base.Libc.GetLastError()",
"ctypes.GetLastError": lambda node, vargs, kwargs: "Base.Libc.GetLastError"
if getattr(node, "is_attr", None)
else "Base.Libc.GetLastError()",
}

SMALL_DISPATCH_MAP = GENERIC_SMALL_DISPATCH_MAP | WIN_SMALL_DISPATCH_MAP

WIN_DISPATCH_TABLE = {
ctypes.WinDLL: (JuliaExternalModulePlugins.visit_load_library, True),
ctypes.windll.LoadLibrary: (JuliaExternalModulePlugins.visit_load_library, True),
# ctypes.WinDLL: (JuliaExternalModulePlugins.visit_windll, True),
# ctypes.GetLastError: (lambda self, node, vargs, kwargs: "Base.Libc.GetLastError", True),
ctypes.FormatError: (lambda self, node, vargs, kwargs: f"Base.Libc.FormatMessage({', '.join(vargs)})", True),
wintypes: (JuliaExternalModulePlugins.visit_wintypes, True),
}
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = GENERIC_DISPATCH_TABLE | WIN_DISPATCH_TABLE
else:
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = GENERIC_DISPATCH_TABLE


EXTERNAL_TYPE_MAP = {
GENERIC_EXTERNAL_TYPE_MAP = {
ctypes.c_int: lambda self: "Cint",
ctypes.c_int8: lambda self: "Cint",
ctypes.c_int16: lambda self: "Cint",
Expand Down Expand Up @@ -168,26 +139,64 @@ def visit_Libdl(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[str,s
ctypes.c_wchar_p: lambda self: "Cwstring", # Ptr{Cwchar_t}
ctypes.c_void_p: lambda self: "Ptr{Cvoid}",
ctypes.CDLL: lambda self: "", # TODO: Temporary
ctypes.WinDLL: lambda self: "",
ctypes.py_object: lambda self: "Ptr{Cvoid}",
}


FUNC_TYPE_MAP = {
GENERIC_FUNC_TYPE_MAP = {
ctypes.cdll.LoadLibrary: lambda self, node, vargs, kwargs: "ctypes.CDLL",
ctypes.windll.LoadLibrary: lambda self, node, vargs, kwargs: "ctypes.WinDLL",
ctypes.CDLL: lambda self, node, vargs, kwargs: "ctypes.CDLL",
ctypes.WinDLL: lambda self, node, vargs, kwargs: "ctypes.WinDLL",
ctypes.PyDLL: lambda self, node, vargs, kwargs: "ctypes.CDLL", # Hack, for now
# Why invalid syntax???
# ctypes.cast: lambda self, node, vargs, kwargs: ast.unparse(vargs[1]) if vargs else "ctypes.cast",
ctypes.cast: lambda self, node, vargs, kwargs: "ctypes._SimpleCData",
ctypes.WINFUNCTYPE: lambda self, node, vargs, kwargs: "PyObject",
ctypes.POINTER: lambda self, node, vargs, kwargs: "ctypes.POINTER",
WindowsError: lambda self, node, vargs, kwargs: "OSError",
}

IGNORED_MODULE_SET = set([
"ctypes",
"ctypes.wintypes"
])

if sys.platform.startswith('win32'):
# Import windows ctypes modules
from builtins import WindowsError
from ctypes import wintypes

WIN_SMALL_DISPATCH_MAP = {
"GetLastError": lambda node, vargs, kwargs: "Base.Libc.GetLastError"
if getattr(node, "is_attr", None)
else "Base.Libc.GetLastError()",
"ctypes.GetLastError": lambda node, vargs, kwargs: "Base.Libc.GetLastError"
if getattr(node, "is_attr", None)
else "Base.Libc.GetLastError()",
}

SMALL_DISPATCH_MAP = GENERIC_SMALL_DISPATCH_MAP | WIN_SMALL_DISPATCH_MAP

WIN_DISPATCH_TABLE = {
ctypes.WinDLL: (JuliaExternalModulePlugins.visit_load_library, True),
ctypes.windll.LoadLibrary: (JuliaExternalModulePlugins.visit_load_library, True),
# ctypes.WinDLL: (JuliaExternalModulePlugins.visit_windll, True),
# ctypes.GetLastError: (lambda self, node, vargs, kwargs: "Base.Libc.GetLastError", True),
ctypes.FormatError: (lambda self, node, vargs, kwargs: f"Base.Libc.FormatMessage({', '.join(vargs)})", True),
wintypes: (JuliaExternalModulePlugins.visit_wintypes, True),
}

WIN_EXTERNAL_TYPE_MAP = {
ctypes.WinDLL: lambda self: "",
}

WIN_FUNC_TYPE_MAP = {
ctypes.windll.LoadLibrary: lambda self, node, vargs, kwargs: "ctypes.WinDLL",
ctypes.WinDLL: lambda self, node, vargs, kwargs: "ctypes.WinDLL",
ctypes.WINFUNCTYPE: lambda self, node, vargs, kwargs: "PyObject",
WindowsError: lambda self, node, vargs, kwargs: "OSError",
}

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = GENERIC_DISPATCH_TABLE | WIN_DISPATCH_TABLE
EXTERNAL_TYPE_MAP = WIN_EXTERNAL_TYPE_MAP | GENERIC_EXTERNAL_TYPE_MAP
FUNC_TYPE_MAP = WIN_FUNC_TYPE_MAP + GENERIC_FUNC_TYPE_MAP
else:
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = GENERIC_DISPATCH_TABLE
EXTERNAL_TYPE_MAP = GENERIC_EXTERNAL_TYPE_MAP
FUNC_TYPE_MAP = GENERIC_FUNC_TYPE_MAP
22 changes: 13 additions & 9 deletions pyjl/external/modules/gzip.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import ast
import gzip
from typing import Callable, Dict, Tuple, Union

try:
import gzip
except ImportError:
gzip = None


class JuliaExternalModulePlugins:
def visit_gzipopen(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[str,str]]):
Expand Down Expand Up @@ -30,14 +34,14 @@ def visit_gzipBadGzipFile(self, node: ast.Call, vargs: list[str], kwargs: list[t
def _generic_gzip_visit(self):
self._usings.add("GZip")

if gzip:
FuncType = Union[Callable, str]

FuncType = Union[Callable, str]

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
gzip.open: (JuliaExternalModulePlugins.visit_gzipopen, True),
gzip.compress: (JuliaExternalModulePlugins.visit_gzipcompress, True),
gzip.decompress: (JuliaExternalModulePlugins.visit_gzipdecompress, True),
gzip.BadGzipFile: (JuliaExternalModulePlugins.visit_gzipBadGzipFile, True),
}
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
gzip.open: (JuliaExternalModulePlugins.visit_gzipopen, True),
gzip.compress: (JuliaExternalModulePlugins.visit_gzipcompress, True),
gzip.decompress: (JuliaExternalModulePlugins.visit_gzipdecompress, True),
gzip.BadGzipFile: (JuliaExternalModulePlugins.visit_gzipBadGzipFile, True),
}

IGNORED_MODULE_SET = set(["gzip"])
31 changes: 18 additions & 13 deletions pyjl/external/modules/pandas.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@

import ast
from typing import BinaryIO, Callable, Dict, Tuple, Union
import pandas
from typing import Callable, Dict, Tuple, Union

try:
import pandas
except:
pandas = None

class JuliaExternalModulePlugins:
def visit_pandas_readcsv(t_self, node: ast.Call, vargs: list[str]):
Expand All @@ -23,17 +27,18 @@ def visit_pandas_dataframe_sum(t_self, node: ast.Call, vargs: list[str]):
def _visit_pandas(t_self):
t_self._usings.add("Pandas")

FuncType = Union[Callable, str]

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
pandas.read_csv: (JuliaExternalModulePlugins.visit_pandas_readcsv, False),
pandas.DataFrame.groupby: (JuliaExternalModulePlugins.visit_pandas_groupby, False),
pandas.DataFrame.to_excel: (JuliaExternalModulePlugins.visit_pandas_toexcel, False),
pandas.core.groupby.generic.DataFrameGroupBy.sum: (
JuliaExternalModulePlugins.visit_pandas_dataframe_sum,
False,
),
}
if pandas:
FuncType = Union[Callable, str]

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
pandas.read_csv: (JuliaExternalModulePlugins.visit_pandas_readcsv, False),
pandas.DataFrame.groupby: (JuliaExternalModulePlugins.visit_pandas_groupby, False),
pandas.DataFrame.to_excel: (JuliaExternalModulePlugins.visit_pandas_toexcel, False),
pandas.core.groupby.generic.DataFrameGroupBy.sum: (
JuliaExternalModulePlugins.visit_pandas_dataframe_sum,
False,
),
}

IGNORED_MODULE_SET = set([
"pandas"
Expand Down
27 changes: 14 additions & 13 deletions pyjl/external/modules/pyproj.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,17 @@ def visit_transform(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[s
return f"pyproj.transform({', '.join(vargs)})" \
if vargs else "pyproj.transform"

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
pyproj.Proj: (JuliaExternalModulePlugins.visit_proj, True),
pyproj.transform: (JuliaExternalModulePlugins.visit_transform, True),
}

EXTERNAL_TYPE_MAP = {
pyproj.Proj: lambda self: JuliaExternalModulePlugins.visit_proj(self, None, [], []),
pyproj.transform: lambda self: JuliaExternalModulePlugins.visit_transform(self, None, [], []),
}

IGNORED_MODULE_SET = set([
"pyproj"
])
if pyproj:
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
pyproj.Proj: (JuliaExternalModulePlugins.visit_proj, True),
pyproj.transform: (JuliaExternalModulePlugins.visit_transform, True),
}

EXTERNAL_TYPE_MAP = {
pyproj.Proj: lambda self: JuliaExternalModulePlugins.visit_proj(self, None, [], []),
pyproj.transform: lambda self: JuliaExternalModulePlugins.visit_transform(self, None, [], []),
}

IGNORED_MODULE_SET = set([
"pyproj"
])
29 changes: 17 additions & 12 deletions pyjl/external/modules/requests_mock.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import ast
from pyjl.helpers import pycall_import
import requests_mock
from typing import Callable, Dict, Tuple, Union

FuncType = Union[Callable, str]

try:
import requests_mock
except ImportError:
requests_mock = None

class JuliaExternalModulePlugins():
def visit_mock(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[str,str]]):
pycall_import(self, node, "requests_mock")
Expand All @@ -18,16 +22,17 @@ def visit_mocker_get(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[
kwargs_str = [f"{kw[0]}={kw[1]}" for kw in kwargs]
return f"{func_name}({', '.join(vargs[1:])}, {', '.join(kwargs_str)})"

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
requests_mock.mock: (JuliaExternalModulePlugins.visit_mock, True),
requests_mock.Mocker.get: (JuliaExternalModulePlugins.visit_mocker_get, True),
# requests_mock.mock.get
}
if requests_mock:
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
requests_mock.mock: (JuliaExternalModulePlugins.visit_mock, True),
requests_mock.Mocker.get: (JuliaExternalModulePlugins.visit_mocker_get, True),
# requests_mock.mock.get
}

IGNORED_MODULE_SET = {
"requests_mock"
}
IGNORED_MODULE_SET = {
"requests_mock"
}

FUNC_TYPE_MAP = {
requests_mock.mock: lambda self, node, vargs, kwargs: "requests_mock.Mocker",
}
FUNC_TYPE_MAP = {
requests_mock.mock: lambda self, node, vargs, kwargs: "requests_mock.Mocker",
}
35 changes: 18 additions & 17 deletions pyjl/external/modules/shapely.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,21 @@ def visit_point(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[str,s
return f"shapely_geo.Point({', '.join(vargs)})" \
if vargs else "shapely_geo.Point"

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
BaseGeometry: (JuliaExternalModulePlugins.visit_basegeometry, True),
transform: (JuliaExternalModulePlugins.visit_transform, True),
Point: (JuliaExternalModulePlugins.visit_point, True),
}

IGNORED_MODULE_SET = set([
"shapely",
"shapely.geometry",
"shapely.geometry.base",
"shapely.ops",
])

EXTERNAL_TYPE_MAP = {
BaseGeometry: lambda self: JuliaExternalModulePlugins.visit_basegeometry(self, None, [], []),
Point: lambda self: JuliaExternalModulePlugins.visit_point(self, None, [], []),
}
if shapely:
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
BaseGeometry: (JuliaExternalModulePlugins.visit_basegeometry, True),
transform: (JuliaExternalModulePlugins.visit_transform, True),
Point: (JuliaExternalModulePlugins.visit_point, True),
}

IGNORED_MODULE_SET = set([
"shapely",
"shapely.geometry",
"shapely.geometry.base",
"shapely.ops",
])

EXTERNAL_TYPE_MAP = {
BaseGeometry: lambda self: JuliaExternalModulePlugins.visit_basegeometry(self, None, [], []),
Point: lambda self: JuliaExternalModulePlugins.visit_point(self, None, [], []),
}
25 changes: 14 additions & 11 deletions pyjl/external/modules/shutil.py
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
import shutil
from typing import Callable, Dict, Tuple, Union
try:
import shutil
except ImportError:
shutil = None

FuncType = Union[Callable, str]

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
shutil.rmtree: (lambda self, node, vargs, kwargs: f"Base.Filesystem.rm({vargs[0]}, recursive=True)", True),
shutil.copy: (lambda self, node, vargs, kwargs: f"Base.Filesystem.cp({vargs[0]}, {vargs[1]}, follow_symlinks={vargs[2]}, force=true)"
if len(vargs) == 3 else f"Base.Filesystem.cp({vargs[0]}, {vargs[1]}, force=true)", True),
shutil.move: (lambda self, node, vargs, kwargs: f"Base.Filesystem.mv({vargs[0]}, {vargs[1]})", True),
shutil.chown: (lambda self, node, vargs, kwargs: f"Base.Filesystem.chown({vargs[0]}, {vargs[1]}, {vargs[2]})"
if len(vargs) == 3 else "Base.Filesystem.chown", True),
}
if shutil:
FuncType = Union[Callable, str]
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
shutil.rmtree: (lambda self, node, vargs, kwargs: f"Base.Filesystem.rm({vargs[0]}, recursive=True)", True),
shutil.copy: (lambda self, node, vargs, kwargs: f"Base.Filesystem.cp({vargs[0]}, {vargs[1]}, follow_symlinks={vargs[2]}, force=true)"
if len(vargs) == 3 else f"Base.Filesystem.cp({vargs[0]}, {vargs[1]}, force=true)", True),
shutil.move: (lambda self, node, vargs, kwargs: f"Base.Filesystem.mv({vargs[0]}, {vargs[1]})", True),
shutil.chown: (lambda self, node, vargs, kwargs: f"Base.Filesystem.chown({vargs[0]}, {vargs[1]}, {vargs[2]})"
if len(vargs) == 3 else "Base.Filesystem.chown", True),
}

IGNORED_MODULE_SET = set([
"shutil"
Expand Down
15 changes: 9 additions & 6 deletions pyjl/external/modules/tqdm.py
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@

import ast
import tqdm
from typing import Callable, Dict, Tuple, Union
try:
import tqdm
except ImportError:
tqdm = None

class JuliaExternalModulePlugins():
def visit_tqdm(self, node: ast.Call, vargs: list[str], kwargs: list[tuple[str,str]]):
self._usings.add("ProgressBars")
# Using tqdm alias (Identical to using ProgressBar)
return f"tqdm({', '.join(vargs)})"

if tqdm:
FuncType = Union[Callable, str]

FuncType = Union[Callable, str]

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
tqdm.tqdm: (JuliaExternalModulePlugins.visit_tqdm, True),
}
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
tqdm.tqdm: (JuliaExternalModulePlugins.visit_tqdm, True),
}

IGNORED_MODULE_SET = set(["tqdm"])
15 changes: 9 additions & 6 deletions pyjl/external/modules/zipfile.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
import ast
import zipfile

from typing import Callable, Dict, Tuple, Union
try:
import zipfile
except ImportError:
zipfile = None

class JuliaExternalModulePlugins:
def visit_zipfile(t_self, node: ast.Call, vargs: list[str]):
t_self._usings.add("ZipFile")
return f"ZipFile.Reader({vargs[0]})"

FuncType = Union[Callable, str]
if zipfile:
FuncType = Union[Callable, str]

FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
zipfile.ZipFile: (JuliaExternalModulePlugins.visit_zipfile, False),
}
FUNC_DISPATCH_TABLE: Dict[FuncType, Tuple[Callable, bool]] = {
zipfile.ZipFile: (JuliaExternalModulePlugins.visit_zipfile, False),
}

0 comments on commit a8df62e

Please sign in to comment.