Core Infrastructure (MetroRobots#2)

edu-rossrobotics · Dec 7, 2023 · 9de27c0 · 9de27c0
1 parent 4b5581d
commit 9de27c0
Show file tree

Hide file tree

Showing 10 changed files with 314 additions and 1 deletion.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,11 @@
+[report]
+omit =
+    *_version.py
+    *finder.py
+    *terminal.py
+    *diff.py
+    *config.py
+    */package.py
+    */ros_resources.py
+    */util.py
+    */main.py
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -27,4 +27,12 @@ jobs:
         allow-prereleases: true
 
     - name: Install package
-      run: python -m pip install -e .
+      run: python -m pip install -e .[test]
+
+    - name: Test package
+      run: python -m pytest --cov=ros_introspect --cov ros_glint
+
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v3.1.4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/pyproject.toml b/pyproject.toml
@@ -13,6 +13,13 @@ maintainers = [
   { name = "David V. Lu!!", email = "davidvlu@gmail.com" },
 ]
 
+dependencies = [
+  "betsy-ros",
+  "colorama",
+  "ros_introspect",
+  "PyYAML",
+]
+
 requires-python = ">=3.8"
 
 dynamic = ["version"]
@@ -29,6 +36,13 @@ classifiers = [
 Homepage = "https://github.com/MetroRobots/ros_glint"
 "Bug Tracker" = "https://github.com/MetroRobots/ros_glint/issues"
 
+[project.optional-dependencies]
+test = [
+  "pytest",
+  "pytest-cov",
+  "pooch",
+]
+
 [tool.hatch]
 version.source = "vcs"
 build.hooks.vcs.version-file = "src/ros_glint/_version.py"
diff --git a/src/ros_glint/__init__.py b/src/ros_glint/__init__.py
@@ -0,0 +1,3 @@
+from .core import get_linters
+
+__all__ = ['get_linters']
diff --git a/src/ros_glint/core.py b/src/ros_glint/core.py
@@ -0,0 +1,17 @@
+import collections
+import pathlib
+
+
+_glinter_functions = collections.OrderedDict()
+
+root = pathlib.Path(__file__).parent.parent.parent
+
+
+# Decorator function for gathering all the functions
+def glinter(f):
+    _glinter_functions[f.__name__] = f
+    return f
+
+
+def get_linters():
+    return _glinter_functions
diff --git a/src/ros_glint/diff.py b/src/ros_glint/diff.py
@@ -0,0 +1,10 @@
+import difflib
+from .terminal import color_diff
+
+
+def get_diff_string(contents0, contents1, filename):
+    s = '=' * 5 + str(filename) + '=' * 45 + '\n'
+    d = difflib.Differ()
+    result = d.compare(contents0.split('\n'), contents1.split('\n'))
+    s += '\n'.join(color_diff(result))
+    return s
diff --git a/src/ros_glint/terminal.py b/src/ros_glint/terminal.py
@@ -0,0 +1,42 @@
+from colorama import Fore, Style, init
+
+init()
+
+
+def color_diff(diff, buffer_size=5):
+    # Buffer size is the number of lines before and after diffs to print
+    buffer = []
+    started = False
+
+    for line in diff:
+        color = ''
+        if line.startswith('+'):
+            color = Fore.GREEN
+        elif line.startswith('-'):
+            color = Fore.RED
+        elif line.startswith('^'):
+            color = Fore.BLUE
+        elif line.startswith('?'):
+            continue
+
+        if color:
+            if buffer:
+                if len(buffer) > buffer_size:
+                    yield '...'
+                yield from buffer[-buffer_size:]
+            started = True
+            yield color + line + Fore.RESET
+            buffer = []
+        else:
+            if started and len(buffer) >= buffer_size:
+                yield from buffer[:buffer_size]
+                buffer = []
+                started = False
+            buffer.append(line)
+    if buffer:
+        yield from buffer[-buffer_size:]
+        yield '...'
+
+
+def color_text(s, fore='YELLOW', bright=False):
+    return (Style.BRIGHT if bright else '') + getattr(Fore, fore) + s + Style.RESET_ALL
diff --git a/src/ros_glint/util.py b/src/ros_glint/util.py
@@ -0,0 +1,11 @@
+import os
+import stat
+
+
+def set_executable(fn, state):
+    existing_permissions = stat.S_IMODE(os.lstat(fn).st_mode)
+    if state:
+        flags = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    else:
+        flags = ~stat.S_IXUSR | ~stat.S_IXGRP | ~stat.S_IXOTH
+    os.chmod(fn, existing_permissions | flags)
diff --git a/test/test_from_zip.py b/test/test_from_zip.py
@@ -0,0 +1,92 @@
+import inspect
+import pooch
+import pytest
+from ros_glint import get_linters
+from ros_glint.diff import get_diff_string
+from ros_glint.terminal import color_text
+from zip_interface import get_test_cases
+from betsy_ros import ROSInterface
+
+from ros_introspect import Package, ROSResources
+
+URL_TEMPLATE = 'https://github.com/DLu/roscompile_test_data/raw/{}/test_data.zip'
+TEST_DATA = [
+    # (branch, known_hash)
+    ('ros1', '2100c19c912c6044194e7a77dad3d002e3b22f5206b171c3f127069b87dbc662'),
+    ('ros2', '97fa59fdc742a60e57cf17643b70eb21aa26a6967cc64d364371480665fb5635'),
+]
+
+linters = get_linters()
+
+
+def files_match(pkg_in, pkg_out, filename):
+    """Return true if the contents of the given file are the same in each package. Otherwise maybe show the diff."""
+    generated_contents = pkg_in.get_contents(filename).rstrip()
+    canonical_contents = pkg_out.get_contents(filename).rstrip()
+    ret = generated_contents == canonical_contents
+    if not ret:
+        print(get_diff_string(generated_contents, canonical_contents, filename))
+    return ret
+
+
+def run_case(test_config, cases):
+    resources = ROSResources.get()
+
+    with cases[test_config['in']] as pkg_in:
+        pkg_out = cases[test_config['out']]
+        root = pkg_in.root
+        pkg_obj = Package(root)
+        local_config = test_config.get('config', {})
+
+        # Initialize ROS Resources
+        resources.packages = set(test_config.get('pkgs', []))
+        resources.messages = set()
+        for msg in test_config.get('msgs', []):
+            parts = msg.split('/')
+            resources.messages.add(ROSInterface(parts[0], 'msg', parts[1]))
+
+        # Run Functions
+        for function_name in test_config['functions']:
+            if function_name not in linters:
+                pytest.skip(f'Missing linter: {function_name}')
+
+            fne = linters[function_name]
+            if 'config' in inspect.getfullargspec(fne).args:
+                fne(pkg_obj, config=local_config)
+            else:
+                fne(pkg_obj)
+        pkg_obj.save()
+
+        folder_diff = pkg_in.compare_filesets(pkg_out)
+
+        s = '{:25} >> {:25} {}'.format(test_config['in'], test_config['out'],
+                                       ','.join(test_config['functions']))
+        print(color_text(s, 'BLUE', bright=True))
+
+        def jp(paths):
+            # Join Paths
+            return ', '.join(map(str, paths))
+
+        assert len(folder_diff['deleted']) == 0, \
+            f'These files should have been deleted but weren\'t: {jp(folder_diff["deleted"])}'
+        assert len(folder_diff['added']) == 0, \
+            f'These files should have been generated but weren\'t: {jp(folder_diff["added"])}'
+        for filename in folder_diff['matches']:
+            assert files_match(pkg_in, pkg_out, filename), 'The contents of {} do not match!'.format(filename)
+
+
+parameters = []
+test_ids = []
+
+for branch, known_hash in TEST_DATA:
+    file_path = pooch.retrieve(URL_TEMPLATE.format(branch), known_hash=known_hash)
+    config, test_data = get_test_cases(file_path)
+
+    for i, test_config in enumerate(config):
+        parameters.append((test_config, test_data))
+        test_ids.append(f'{branch}_test_{i:03d}')
+
+
+@pytest.mark.parametrize('test_config, test_data', parameters, ids=test_ids)
+def test_from_zip(test_config, test_data):
+    run_case(test_config, test_data)
diff --git a/test/zip_interface.py b/test/zip_interface.py
@@ -0,0 +1,105 @@
+import collections
+import os
+import pathlib
+import shutil
+import tempfile
+import yaml
+import zipfile
+
+from ros_introspect.finder import walk
+from ros_glint.util import set_executable
+
+
+class ROSCompilePackageFiles:
+    def __init__(self, package_name, pkg_files, executables):
+        self.package_name = package_name
+        self.is_written = False
+        self.root = self.get_input_root()
+        self.pkg_files = pkg_files
+        self.executables = executables
+
+    def copy(self):
+        return ROSCompilePackageFiles(self.package_name, self.pkg_files, self.executables)
+
+    def get_input_root(self):
+        return pathlib.Path(tempfile.gettempdir()) / self.package_name
+
+    def __enter__(self):
+        self.write()
+        self.is_written = True
+        return self
+
+    def __exit__(self, a_type, value, traceback):
+        self.is_written = False
+        self.clear()
+
+    def get_filenames(self):
+        if self.is_written:
+            the_files = []
+            for subpath in walk(self.root):
+                the_files.append(subpath)
+            return set(the_files)
+        else:
+            return set(self.pkg_files.keys())
+
+    def get_contents(self, filename):
+        if self.is_written:
+            full_path = self.root / filename
+            if full_path.exists():
+                return open(full_path).read().replace('\r\n', '\n')
+        elif filename in self.pkg_files:
+            return self.pkg_files[filename].replace('\r\n', '\n')
+
+    def compare_filesets(self, other_package):
+        in_keys = self.get_filenames()
+        out_keys = other_package.get_filenames()
+        matches = in_keys.intersection(out_keys)
+        missed_deletes = in_keys - out_keys
+        missed_generations = out_keys - in_keys
+        return {'matches': sorted(matches), 'deleted': sorted(missed_deletes), 'added': sorted(missed_generations)}
+
+    def write(self):
+        self.clear()
+        self.root.mkdir()
+        for fn, contents in self.pkg_files.items():
+            outfile = self.root / fn
+            outfile.parent.mkdir(exist_ok=True, parents=True)
+            with open(outfile, 'w') as f:
+                f.write(contents)
+            if fn in self.executables:
+                set_executable(outfile, True)
+
+    def clear(self):
+        if self.root.exists():
+            shutil.rmtree(self.root)
+
+    def __repr__(self):
+        return self.package_name
+
+
+def get_test_cases(zip_filename):
+    file_data = collections.defaultdict(dict)
+    zf = zipfile.ZipFile(zip_filename)
+    config = None
+    executables = set()
+    for file in zf.filelist:
+        if file.filename[-1] == '/':
+            continue
+        if file.filename == 'list_o_tests.yaml':
+            config = yaml.safe_load(zf.read(file))
+            continue
+        parts = file.filename.split(os.path.sep)
+        package = parts[0]
+        path = pathlib.Path(os.path.join(*parts[1:]))
+        file_data[package][path] = zf.read(file).decode()
+        if (file.external_attr >> 16) & 0o111:
+            executables.add(path)
+
+    test_data = {}
+    for package, pkg_data in file_data.items():
+        test_data[package] = ROSCompilePackageFiles(package, pkg_data, executables)
+    for pkg_data in config:
+        if 'function' in pkg_data:
+            pkg_data['functions'] = [pkg_data.pop('function')]
+
+    return config, test_data
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		from .core import get_linters

		__all__ = ['get_linters']