Package XML with Data Files (MetroRobots#4)

* Package XML with Data Files * Missing dependency
edu-rossrobotics · Dec 8, 2023 · d390021 · d390021
1 parent 8546559
commit d390021
Show file tree

Hide file tree

Showing 9 changed files with 343 additions and 5 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -21,6 +21,7 @@ repos:
   - id: codespell
     args:
     - --write-changes
+    - --skip=src/ros_glint/data/*ignore
   rev: v2.2.6
 - repo: https://github.com/adrienverge/yamllint
   hooks:

diff --git a/pyproject.toml b/pyproject.toml
@@ -16,6 +16,7 @@ maintainers = [
 dependencies = [
   "betsy-ros",
   "colorama",
+  "importlib-resources",
   "ros_introspect",
   "PyYAML",
 ]

diff --git a/src/ros_glint/__init__.py b/src/ros_glint/__init__.py
@@ -1,5 +1,5 @@
 from .core import get_linters
 
-from .glinters import ros_interfaces
+from .glinters import package_xml, ros_interfaces
 
-__all__ = ['get_linters', 'ros_interfaces']
+__all__ = ['get_linters', 'package_xml', 'ros_interfaces']
diff --git a/src/ros_glint/config.py b/src/ros_glint/config.py
@@ -0,0 +1,18 @@
+import pathlib
+import yaml
+
+DOT_ROS_FOLDER = pathlib.Path('~/.ros').expanduser()
+POSSIBLE_CONFIG_FILES = ['roscompile.yaml', 'glint.yaml']
+CONFIG = None
+
+
+def get_config():
+    global CONFIG
+    if CONFIG is None:
+        for possible_config_file in POSSIBLE_CONFIG_FILES:
+            path = DOT_ROS_FOLDER / possible_config_file
+            if path.exists():
+                CONFIG = yaml.safe_load(open(path))
+                return CONFIG
+        CONFIG = {}
+    return CONFIG
diff --git a/src/ros_glint/core.py b/src/ros_glint/core.py
@@ -1,11 +1,8 @@
 import collections
-import pathlib
 
 
 _glinter_functions = collections.OrderedDict()
 
-root = pathlib.Path(__file__).parent.parent.parent
-
 
 # Decorator function for gathering all the functions
 def glinter(f):

diff --git a/src/ros_glint/data/package.ignore b/src/ros_glint/data/package.ignore
@@ -0,0 +1,43 @@
+One maintainer tag required, multiple allowed, one person per tag
+Example:
+<maintainer email="jane.doe@example.com">Jane Doe</maintainer>
+One license tag required, multiple allowed, one license per tag
+Commonly used license strings:
+BSD, MIT, Boost Software License, GPLv2, GPLv3, LGPLv2.1, LGPLv3
+Url tags are optional, but mutiple are allowed, one per tag
+Url tags are optional, but multiple are allowed, one per tag
+Optional attribute type can be: website, bugtracker, or repository
+Example:
+Author tags are optional, mutiple are allowed, one per tag
+Author tags are optional, multiple are allowed, one per tag
+Authors do not have to be maintianers, but could be
+Authors do not have to be maintainers, but could be
+Example:
+<author email="jane.doe@example.com">Jane Doe</author>
+The *_depend tags are used to specify dependencies
+The *depend tags are used to specify dependencies
+Dependencies can be catkin packages or system dependencies
+Examples:
+Use build_depend for packages you need at compile time:
+<build_depend>message_generation</build_depend>
+Use buildtool_depend for build tool packages:
+<buildtool_depend>catkin</buildtool_depend>
+Use run_depend for packages you need at runtime:
+Use run_depend for packages you need in order to build against this package:
+<run_depend>message_runtime</run_depend>
+<run_depend>message_generation</run_depend>
+Use test_depend for packages you need only for testing:
+<test_depend>gtest</test_depend>
+The export tag contains other, unspecified, tags
+Other tools can request additional information be placed here
+Use depend as a shortcut for packages that are both build and exec dependencies
+<depend>roscpp</depend>
+Note that this is equivalent to the following:
+<build_depend>roscpp</build_depend>
+<exec_depend>roscpp</exec_depend>
+Use build_export_depend for packages you need in order to build against this package:
+<build_export_depend>message_generation</build_export_depend>
+Use exec_depend for packages you need at runtime:
+<exec_depend>message_runtime</exec_depend>
+Use doc_depend for packages you need only for building documentation:
+<doc_depend>doxygen</doc_depend>
diff --git a/src/ros_glint/data/package_patterns.ignore b/src/ros_glint/data/package_patterns.ignore
@@ -0,0 +1 @@
+<url type="website">http://wiki.ros.org/%(package)s</url>
diff --git a/src/ros_glint/glinters/package_xml.py b/src/ros_glint/glinters/package_xml.py
@@ -0,0 +1,254 @@
+from ..core import glinter
+from ..util import get_ignore_data
+from ..config import get_config
+from ros_introspect.package import DependencyType
+from ros_introspect.components.package_xml import count_trailing_spaces, get_chunks, get_sort_key, PEOPLE_TAGS
+
+
+@glinter
+def check_manifest_dependencies(package, config=None):
+    if config is None:
+        config = get_config()  # pragma: no cover
+    prefer_depend_tag = config.get('prefer_depend_tag', False)
+
+    dep_dict = {}
+    for dt in DependencyType:
+        dep_dict[dt] = package.get_dependencies(dt)
+
+    package.package_xml.add_dependencies(dep_dict, prefer_depend_tag)
+
+    # Special handling for interface dependencies
+    # because not all run deps are build deps
+    if not package.get_ros_interfaces():
+        return
+    if package.ros_version == 1:
+        build_dep = 'message_generation'
+        run_dep = 'message_runtime'
+        export = 'message_runtime'
+        export_tag = 'build_export_depend'
+    else:
+        build_dep = 'rosidl_default_generators'
+        run_dep = 'rosidl_default_runtime'
+        export = 'rosidl_interface_packages'
+        export_tag = 'member_of_group'
+
+    if package.package_xml.xml_format == 1:
+        pairs = [('build_depend', build_dep),
+                 ('run_depend', run_dep)]
+    else:
+        pairs = [('build_depend', build_dep),
+                 (export_tag, export),
+                 ('exec_depend', run_dep)]
+        package.package_xml.remove_dependencies('depend', [build_dep, run_dep])
+    for tag, msg_pkg in pairs:
+        existing = package.package_xml.get_packages_by_tag(tag)
+        if msg_pkg not in existing:
+            package.package_xml.insert_new_packages(tag, [msg_pkg])
+
+
+@glinter
+def enforce_manifest_ordering(package, alphabetize=True):
+    root = package.package_xml.root
+    chunks = get_chunks(root.childNodes)
+
+    new_children = []
+
+    for a, b in sorted(chunks, key=lambda d: get_sort_key(d[0], alphabetize, package.package_xml.xml_format)):
+        new_children += b
+
+    if root.childNodes != new_children:
+        package.package_xml.changed = True
+        root.childNodes = new_children
+
+
+@glinter
+def remove_empty_export_tag(package):
+
+    def has_element_child(node):
+        for child in node.childNodes:
+            if child.nodeType == child.ELEMENT_NODE:
+                return True
+        return False
+
+    manifest = package.package_xml
+    exports = manifest.root.getElementsByTagName('export')
+    if len(exports) == 0:
+        return False
+    for export in exports:
+        if not has_element_child(export):
+            manifest.remove_element(export)
+            return True
+
+
+@glinter
+def enforce_manifest_tabbing(package):
+    def enforce_tabbing_helper(manifest, node, tabs=1):
+        ideal_length = manifest.std_tab * tabs
+        prev_was_node = True
+        insert_before_list = []
+        if not node:
+            return
+        changed = False
+        for c in node.childNodes:
+            if c.nodeType == c.TEXT_NODE:
+                prev_was_node = False
+                if c == node.childNodes[-1]:
+                    continue
+
+                if '\n' not in c.data:
+                    c.data = '\n' + c.data
+                    changed = True
+                spaces = count_trailing_spaces(c.data)
+                if spaces != ideal_length:
+                    last_nl = c.data.rindex('\n')
+                    c.data = c.data[: last_nl + 1] + (' ' * ideal_length)
+                    changed = True
+            else:
+                if prev_was_node:
+                    changed = True
+                    insert_before_list.append(c)
+
+                prev_was_node = True
+                enforce_tabbing_helper(manifest, c, tabs + 1)
+
+        for c in insert_before_list:
+            node.insertBefore(manifest.create_new_tab_element(tabs), c)
+
+        manifest.changed = manifest.changed or changed
+
+        if len(node.childNodes) == 0:
+            return
+        last = node.childNodes[-1]
+        if last.nodeType != last.TEXT_NODE:
+            node.appendChild(manifest.create_new_tab_element(tabs - 1))
+            manifest.changed = True
+
+    enforce_tabbing_helper(package.package_xml, package.package_xml.root)
+
+
+@glinter
+def remove_empty_manifest_lines(package):
+    def remove_empty_lines_helper(node):
+        changed = False
+        for child in node.childNodes:
+            if child.nodeType == child.TEXT_NODE:
+                while '\n\n\n' in child.data:
+                    child.data = child.data.replace('\n\n\n', '\n\n')
+                    changed = True
+            else:
+                changed = remove_empty_lines_helper(child) or changed
+        return changed
+
+    if remove_empty_lines_helper(package.package_xml.root):
+        package.package_xml.changed = True
+
+
+def cleanup_text_elements(node):
+    new_children = []
+    changed = False
+
+    for child in node.childNodes:
+        if child.nodeType == child.TEXT_NODE and len(new_children) and new_children[-1].nodeType == child.TEXT_NODE:
+            changed = True
+            new_children[-1].data += child.data
+        elif child.nodeType == child.TEXT_NODE and child.data == '':
+            continue
+        else:
+            new_children.append(child)
+
+    node.childNodes = new_children
+    return changed
+
+
+def replace_text_node_contents(node, ignorables):
+    changed = False
+    removable = []
+    for i, c in enumerate(node.childNodes):
+        if c.nodeType == c.TEXT_NODE:
+            continue
+        elif c.nodeType == c.COMMENT_NODE:
+            short = c.data.strip()
+            if short in ignorables:
+                removable.append(i)
+                changed = True
+                continue
+        else:
+            changed = replace_text_node_contents(c, ignorables) or changed
+    for node_index in reversed(removable):  # backwards not to affect earlier indices
+        if node_index > 0:
+            before = node.childNodes[node_index - 1]
+            if before.nodeType == c.TEXT_NODE:
+                trailing = count_trailing_spaces(before.data)
+                before.data = before.data[:-trailing]
+
+        if node_index < len(node.childNodes) - 1:
+            after = node.childNodes[node_index + 1]
+            if after.nodeType == c.TEXT_NODE:
+                while len(after.data) and after.data[0] == ' ':
+                    after.data = after.data[1:]
+                if len(after.data) and after.data[0] == '\n':
+                    after.data = after.data[1:]
+
+        node.childNodes.remove(node.childNodes[node_index])
+    changed = cleanup_text_elements(node) or changed
+    return changed
+
+
+@glinter
+def remove_boilerplate_manifest_comments(package):
+    ignorables = get_ignore_data('package', {'package': package.name}, add_newline=False)
+    changed = replace_text_node_contents(package.package_xml.root, ignorables)
+    if changed:
+        package.package_xml.changed = changed
+        remove_empty_manifest_lines(package)
+
+
+def replace_package_set(package_xml, source_tags, new_tag):
+    """Replace all the elements with tags in source_tags with new elements with new_tag."""
+    intersection = None
+    for tag in source_tags:
+        pkgs = set(package_xml.get_packages_by_tag(tag))
+        if intersection is None:
+            intersection = pkgs
+        else:
+            intersection = intersection.intersection(pkgs)
+    for tag in source_tags:
+        package_xml.remove_dependencies(tag, intersection)
+    package_xml.insert_new_packages(new_tag, intersection)
+
+
+@glinter
+def greedy_depend_tag(package):
+    if package.package_xml.xml_format == 1:
+        return
+    replace_package_set(package.package_xml, ['build_depend', 'build_export_depend', 'exec_depend'], 'depend')
+
+
+@glinter
+def update_people(package, config=None):
+    if config is None:
+        config = get_config()  # pragma: no cover
+    for d in config.get('replace_rules', []):
+        target_name = d['to']['name']
+        target_email = d['to']['email']
+        search_name = d['from'].get('name')
+        search_email = d['from'].get('email')
+
+        for el in package.package_xml.get_elements_by_tags(PEOPLE_TAGS):
+            name = el.childNodes[0].nodeValue
+            email = el.getAttribute('email') if el.hasAttribute('email') else ''
+            if (search_name is None or name == search_name) and (search_email is None or email == search_email):
+                el.childNodes[0].nodeValue = target_name
+                if target_email:
+                    el.setAttribute('email', target_email)
+                package.package_xml.changed = True
+
+
+@glinter
+def update_license(package, config=None):
+    if config is None:
+        config = get_config()  # pragma: no cover
+    if 'default_license' not in config or 'TODO' not in package.package_xml.get_license():
+        return
+
+    package.package_xml.set_license(config['default_license'])
diff --git a/src/ros_glint/util.py b/src/ros_glint/util.py
@@ -1,4 +1,5 @@
 import os
+import importlib_resources as resources
 import stat
 
 
@@ -9,3 +10,25 @@ def set_executable(fn, state):
     else:
         flags = ~stat.S_IXUSR | ~stat.S_IXGRP | ~stat.S_IXOTH
     os.chmod(fn, existing_permissions | flags)
+
+
+def get_data_file(name):
+    return (resources.files('ros_glint.data') / name).read_text()
+
+
+def get_ignore_data(name, variables, add_newline=True):
+    def get_ignore_data_helper(basename, add_newline=True):
+        lines = []
+        for s in get_data_file(basename + '.ignore').split('\n'):
+            if s == '':
+                continue
+            if add_newline:
+                lines.append(s + '\n')
+            else:
+                lines.append(s)
+        return lines
+
+    ignore_lines = get_ignore_data_helper(name, add_newline)
+    for pattern in get_ignore_data_helper(name + '_patterns', add_newline):
+        ignore_lines.append(pattern % variables)
+    return ignore_lines
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		<url type="website">http://wiki.ros.org/%(package)s</url>