Moving to internal VLA type

This release changes the default C++ generator to email a built-in VariableLengthArray type. This type makes no use of the standard library's containers and allocators and is optimized for Cyphal types. It does support using STL allocators but also includes a special pattern that allows the VLA to be backed by a fixed array with minimal overhead. # PR activity * Fix missing array capacity constant generated * Fix typo * Fix typo[2] * Update src/nunavut/lang/cpp/templates/_fields.j2 Merge suggestion by @pavel-kirienko Co-authored-by: Pavel Kirienko <pavel.kirienko@gmail.com> * Proper support for variable_length_array This change fixes the support for a built-in, default variable length array implementation for C++ types that can be overridden by the user. To accomplish this the concept of “support resource type” was added to allow different behavior for serialization support resource than for datatype support resources. * fixing windows build * checkpoint almost there. * Carefully adding a couple more features * it aint pretty but it works * not sure why you no Windows? * fixing windows and adding except specifiers to new contrustors * fixing test * fixing gcc builds * fixing logic error in fp test * tests are broken but pushing to save my work * fixing test * fixing 32bit builds * removing vestigal comment * fixing linter error * fixing code smell Co-authored-by: Pavel Pletenev <cpp.create@gmail.com> Co-authored-by: Pavel Kirienko <pavel.kirienko@gmail.com>
edu-rossrobotics · Sep 26, 2022 · 25bc50b · 25bc50b
1 parent 498dc38
commit 25bc50b
Show file tree

Hide file tree

Showing 28 changed files with 1,178 additions and 306 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,5 +1,5 @@
 {
-    "python.pythonPath": ".tox/local/bin/python",
+    "python.defaultInterpreterPath": "${workspaceFolder}/.tox/local",
     "python.linting.flake8Enabled": true,
     "python.linting.pylintEnabled": false,
     "python.linting.mypyEnabled": true,
@@ -11,7 +11,6 @@
     ],
     "python.testing.cwd": "${workspaceFolder}",
     "python.testing.unittestEnabled": false,
-    "python.testing.nosetestsEnabled": false,
     "python.testing.pytestEnabled": true,
     "python.formatting.provider": "black",
     "python.formatting.blackArgs": [
@@ -41,8 +40,8 @@
     "C_Cpp.configurationWarnings": "Disabled",
     "files.associations": {
         "*.py.template": "python",
-        "*.cc": "c++",
-        "*.hpp": "c++",
+        "*.cc": "cpp",
+        "*.hpp": "cpp",
         "__bit_reference": "cpp",
         "__config": "cpp",
         "__debug": "cpp",
@@ -137,7 +136,11 @@
         "any": "cpp",
         "span": "cpp",
         "variant": "cpp",
-        "__bits": "cpp"
+        "__bits": "cpp",
+        "filesystem": "cpp",
+        "__memory": "cpp",
+        "compare": "cpp",
+        "concepts": "cpp"
     },
     "python.linting.mypyArgs": [
         "--config-file=${workspaceFolder}/tox.ini"

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -82,6 +82,7 @@ To run the language verification build you'll need to use a different docker con
 
     docker pull uavcan/c_cpp:ubuntu-20.04
     docker run --rm -it -v $PWD:/repo uavcan/c_cpp:ubuntu-20.04
+    cd /repo
     ./.github/verify.py -l c
     ./.github/verify.py -l cpp
 

diff --git a/conf.py b/conf.py
@@ -77,7 +77,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "monokai"

diff --git a/conftest.py b/conftest.py
@@ -14,6 +14,7 @@
 import tempfile
 import textwrap
 import typing
+import urllib
 from doctest import ELLIPSIS
 
 import pydsdl
@@ -187,7 +188,7 @@ def out_dir(self) -> pathlib.Path:
         The directory to place test output under for this test case.
         """
         if self._out_dir is None:
-            self._out_dir = self.create_new_temp_dir(self.test_name)
+            self._out_dir = self.create_new_temp_dir(urllib.parse.quote_plus(self.test_name))
         return self._out_dir
 
     @property

diff --git a/docs/index.rst b/docs/index.rst
@@ -4,6 +4,7 @@
    :hidden:
 
    api/library
+   languages
    templates
    CLI (nnvg) <cli>
    dev

diff --git a/docs/languages.rst b/docs/languages.rst
@@ -0,0 +1,7 @@
+################################################
+Software Language Generation Guide
+################################################
+
+.. note ::
+    This is a placeholder for documentation this project owes you, the user, for how to integrate nnvg with build
+    systems and how to tune and optimize source code generation for each supported language.
diff --git a/setup.cfg b/setup.cfg
@@ -7,7 +7,7 @@ description = Generate code from DSDL using Jinja2 templates.
 long_description = file: README.rst
 long_description_content_type = text/x-rst
 license = MIT
-license_file = LICENSE.rst
+license_files = LICENSE.rst
 keywords = uavcan, dsdl, can, can-bus, codegen, cyphal, opencyphal
 classifiers =
     Development Status :: 3 - Alpha

diff --git a/src/nunavut/_utilities.py b/src/nunavut/_utilities.py
@@ -93,6 +93,18 @@ def test_truth(cls, ynd_value: "YesNoDefault", default_value: bool) -> bool:
     DEFAULT = 2
 
 
+@enum.unique
+class ResourceType(enum.Enum):
+    """
+    Common Nunavut classifications for Python package resources.
+    """
+
+    ANY = 0
+    CONFIGURATION = 1
+    SERIALIZATION_SUPPORT = 2
+    TYPE_SUPPORT = 3
+
+
 def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pathlib.Path, None, None]:
     """
     >>> from nunavut._utilities import iter_package_resources
@@ -104,5 +116,22 @@ def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pat
 
     """
     for resource in importlib_resources.files(pkg_name).iterdir():
-        if any(suffix == resource.suffix for suffix in suffix_filters):  # type: ignore
-            yield cast(pathlib.Path, resource)
+        if resource.is_file() and isinstance(resource, pathlib.Path):
+            # Not sure why this works but it's seemed to so far. importlib_resources.as_file(resource)
+            # may be more correct but this can create temporary files which would disappear after the iterator
+            # had copied their paths. If you are reading this because this method isn't working for some packaging
+            # scheme then we may need to use importlib_resources.as_file(resource) to create a runtime cache of
+            # temporary objects that live for a given nunavut session. This, of course, wouldn't help across sessions
+            # which is a common use case when integrating Nunavut with build systems. So...here be dragons.
+            file_resource = cast(pathlib.Path, resource)
+            if any(suffix == file_resource.suffix for suffix in suffix_filters):
+                yield file_resource
+
+
+def empty_list_support_files() -> Generator[pathlib.Path, None, None]:
+    """
+    Helper for implementing the list_support_files method in language support packages. This provides an empty
+    iterator with the correct type annotations.
+    """
+    # works in Python 3.3 and newer. Thanks https://stackoverflow.com/a/13243870
+    yield from ()
diff --git a/src/nunavut/cli/__init__.py b/src/nunavut/cli/__init__.py
@@ -511,7 +511,7 @@ def extension_type(raw_arg: str) -> str:
 def _extra_includes_from_env(env_var_name: str) -> typing.List[str]:
     try:
         extra_includes_from_env = os.environ[env_var_name].split(os.pathsep)
-        logging.info("Additional include directories from {}: %s", env_var_name, str(extra_includes_from_env))
+        logging.info("Additional include directories from {}: {}".format(env_var_name, str(extra_includes_from_env)))
         return extra_includes_from_env
     except KeyError:
         return []

diff --git a/src/nunavut/jinja/__init__.py b/src/nunavut/jinja/__init__.py
@@ -19,7 +19,7 @@
 import nunavut.lang
 import nunavut.postprocessors
 import pydsdl
-from nunavut._utilities import YesNoDefault
+from nunavut._utilities import ResourceType, YesNoDefault
 from yaml import Dumper as YamlDumper
 from yaml import dump as yaml_dump
 
@@ -779,9 +779,9 @@ def __init__(self, namespace: nunavut.Namespace, **kwargs: typing.Any):
         target_language = self.language_context.get_target_language()
 
         self._sub_folders = None  # type: typing.Optional[pathlib.Path]
-        self._support_enabled = False  # If not enabled then we remove any support files found
+        self._serialization_support_enabled = False
         if target_language is not None:
-            self._support_enabled = not target_language.omit_serialization_support
+            self._serialization_support_enabled = not target_language.omit_serialization_support
 
             #  Create the sub-folder to copy-to based on the support namespace.
             self._sub_folders = pathlib.Path("")
@@ -794,11 +794,11 @@ def __init__(self, namespace: nunavut.Namespace, **kwargs: typing.Any):
     # +-----------------------------------------------------------------------+
     def get_templates(self) -> typing.Iterable[pathlib.Path]:
         files = []
-        target_language = self.language_context.get_target_language()
-
-        if target_language is not None:
-            for resource in target_language.support_files:
+        if self._serialization_support_enabled:
+            for resource in self._get_templates_by_support_type(ResourceType.SERIALIZATION_SUPPORT):
                 files.append(resource)
+        for resource in self._get_templates_by_support_type(ResourceType.TYPE_SUPPORT):
+            files.append(resource)
         return files
 
     def generate_all(self, is_dryrun: bool = False, allow_overwrite: bool = True) -> typing.Iterable[pathlib.Path]:
@@ -812,6 +812,14 @@ def generate_all(self, is_dryrun: bool = False, allow_overwrite: bool = True) ->
     # +-----------------------------------------------------------------------+
     # | Private
     # +-----------------------------------------------------------------------+
+    def _get_templates_by_support_type(self, resource_type: ResourceType) -> typing.Iterable[pathlib.Path]:
+        files = []
+        target_language = self.language_context.get_target_language()
+
+        if target_language is not None:
+            for resource in target_language.get_support_files(resource_type):
+                files.append(resource)
+        return files
 
     def _generate_all(
         self, target_language: nunavut.lang.Language, sub_folders: pathlib.Path, is_dryrun: bool, allow_overwrite: bool
@@ -833,27 +841,14 @@ def _generate_all(
         for resource in self.get_templates():
             target = (target_path / resource.name).with_suffix(target_language.extension)
             logger.info("Generating support file: %s", target)
-            if not self._support_enabled:
-                self._remove_header(target, is_dryrun, allow_overwrite)
-            elif resource.suffix == TEMPLATE_SUFFIX:
+            if resource.suffix == TEMPLATE_SUFFIX:
                 self._generate_header(resource, target, is_dryrun, allow_overwrite)
                 generated.append(target)
             else:
                 self._copy_header(resource, target, is_dryrun, allow_overwrite, line_pps, file_pps)
                 generated.append(target)
         return generated
 
-    def _remove_header(self, target: pathlib.Path, is_dryrun: bool, allow_overwrite: bool) -> None:
-        if not is_dryrun:
-            if not allow_overwrite and target.exists():
-                raise PermissionError("{} exists. Refusing to remove.".format(str(target)))
-            try:
-                target.unlink()
-            except FileNotFoundError:
-                # missing_ok was added in python 3.8 so this try/except statement will
-                # go away someday when python 3.7 support is dropped.
-                pass
-
     def _generate_header(
         self, template_path: pathlib.Path, output_path: pathlib.Path, is_dryrun: bool, allow_overwrite: bool
     ) -> pathlib.Path:

diff --git a/src/nunavut/lang/__init__.py b/src/nunavut/lang/__init__.py
@@ -18,8 +18,8 @@
 
 import pydsdl
 
+from .._utilities import ResourceType, YesNoDefault, iter_package_resources, empty_list_support_files
 from ..dependencies import Dependencies, DependencyBuilder
-from .._utilities import YesNoDefault, iter_package_resources
 from ._config import LanguageConfig, VersionReader
 
 logger = logging.getLogger(__name__)
@@ -467,10 +467,13 @@ def omit_serialization_support(self) -> bool:
         """
         return self._omit_serialization_support
 
-    @property
-    def support_files(self) -> typing.Generator[pathlib.Path, None, None]:
+    def get_support_files(
+        self, resource_type: ResourceType = ResourceType.ANY
+    ) -> typing.Generator[pathlib.Path, None, None]:
         """
-        Iterates over non-templated supporting files embedded within the Nunavut distribution.
+        Iterates over supporting files embedded within the Nunavut distribution.
+
+        :param resource_type: The type of support resources to enumerate.
 
         .. invisible-code-block: python
 
@@ -483,7 +486,7 @@ def support_files(self) -> typing.Generator[pathlib.Path, None, None]:
 
             my_lang = _GenericLanguage(mock_module, mock_config, True)
             my_lang._section = "nunavut.lang.not_a_language_really_not_a_language"
-            for support_file in my_lang.support_files:
+            for support_file in my_lang.get_support_files():
                 # if the module doesn't exist it shouldn't have any support files.
                 assert False
 
@@ -495,15 +498,10 @@ def support_files(self) -> typing.Generator[pathlib.Path, None, None]:
             # to allow the copy generator access to the packaged support files.
             list_support_files = getattr(
                 module, "list_support_files"
-            )  # type: typing.Callable[[], typing.Generator[pathlib.Path, None, None]]
-            return list_support_files()
+            )  # type: typing.Callable[[ResourceType], typing.Generator[pathlib.Path, None, None]]
+            return list_support_files(resource_type)
         else:
-            # No serialization support for this language
-            def list_support_files() -> typing.Generator[pathlib.Path, None, None]:
-                # This makes both MyPy and sonarqube happy.
-                return typing.cast(typing.Generator[pathlib.Path, None, None], iter(()))
-
-            return list_support_files()
+            return empty_list_support_files()
 
     def get_option(
         self, option_key: str, default_value: typing.Union[typing.Mapping[str, typing.Any], str, None] = None

diff --git a/src/nunavut/lang/_common.py b/src/nunavut/lang/_common.py
@@ -14,6 +14,7 @@
 import typing
 
 import pydsdl
+from nunavut._utilities import ResourceType
 
 from . import Language
 
@@ -30,13 +31,13 @@ def generate_include_filepart_list(self, output_extension: str, sort: bool) -> t
             self.make_path(dt, self._language, output_extension).as_posix() for dt in dep_types.composite_types
         ]
 
+        namespace_path = pathlib.Path("")
+        for namespace_part in self._language.support_namespace:
+            namespace_path = namespace_path / pathlib.Path(namespace_part)
         if not self._language.omit_serialization_support:
-            namespace_path = pathlib.Path("")
-            for namespace_part in self._language.support_namespace:
-                namespace_path = namespace_path / pathlib.Path(namespace_part)
             path_list += [
                 (namespace_path / pathlib.Path(p.name).with_suffix(output_extension)).as_posix()
-                for p in self._language.support_files
+                for p in self._language.get_support_files(ResourceType.SERIALIZATION_SUPPORT)
             ]
 
         prefer_system_includes = self._language.get_config_value_as_bool("prefer_system_includes", False)
@@ -46,7 +47,7 @@ def generate_include_filepart_list(self, output_extension: str, sort: bool) -> t
             path_list_with_punctuation = ['"{}"'.format(p) for p in path_list]
 
         if sort:
-            return sorted(path_list_with_punctuation) + self._language.get_includes(dep_types)
+            return sorted(path_list_with_punctuation + self._language.get_includes(dep_types))
         else:
             return path_list_with_punctuation + self._language.get_includes(dep_types)
 

diff --git a/src/nunavut/lang/c/support/__init__.py b/src/nunavut/lang/c/support/__init__.py
@@ -8,15 +8,17 @@
 """
 import pathlib
 import typing
-from nunavut._utilities import iter_package_resources
+
+from nunavut._utilities import ResourceType, empty_list_support_files, iter_package_resources
 
 __version__ = "1.0.0"
 """Version of the c support headers."""
 
 
-def list_support_files() -> typing.Generator[pathlib.Path, None, None]:
+def list_support_files(resource_type: ResourceType = ResourceType.ANY) -> typing.Generator[pathlib.Path, None, None]:
     """
     Get a list of C support headers embedded in this package.
+    :param resource_type: A type of support file to list.
 
     .. invisible-code-block: python
 
@@ -37,4 +39,8 @@ def list_support_files() -> typing.Generator[pathlib.Path, None, None]:
 
     :return: A list of C support header resources.
     """
-    return iter_package_resources(__name__, ".h", ".j2")
+    # The c support only has serialization support resources
+    if resource_type not in (ResourceType.ANY, ResourceType.SERIALIZATION_SUPPORT):
+        return empty_list_support_files()
+    else:
+        return iter_package_resources(__name__, ".h", ".j2")