diff --git a/docs/notes/2.25.x.md b/docs/notes/2.25.x.md index 912d8553ca3..cc617221c0f 100644 --- a/docs/notes/2.25.x.md +++ b/docs/notes/2.25.x.md @@ -70,6 +70,11 @@ Reverence to Python Build Standalone not refer to the [GitHub organization](http The default version of the [Pex](https://docs.pex-tool.org/) tool has been updated from 2.20.3 to [2.27.1](https://github.com/pex-tool/pex/releases/tag/v2.24.3). Among many improvements and bug fixes, this unlocks support for pip [24.3.1](https://pip.pypa.io/en/stable/news/#v24-3-1). +##### NEW: Python for OpenAPI + +A new experimental `pants.backend.experimental.openapi.codegen.python` backend +was added to support python codegen for OpenAPI documents. + #### Shell The previously deprecated `[shell-setup].tailor` option has now been removed. See [`[shell-setup].tailor_sources`](https://www.pantsbuild.org/2.25/reference/subsystems/shell-setup#tailor_sources) and [`[shell-setup].tailor_shunit2_tests`](https://www.pantsbuild.org/2.25/reference/subsystems/shell#tailor_shunit2_tests) to update. diff --git a/src/python/pants/backend/experimental/openapi/codegen/python/BUILD b/src/python/pants/backend/experimental/openapi/codegen/python/BUILD new file mode 100644 index 00000000000..3928f7e3ba8 --- /dev/null +++ b/src/python/pants/backend/experimental/openapi/codegen/python/BUILD @@ -0,0 +1,4 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() diff --git a/src/python/pants/backend/experimental/openapi/codegen/python/__init__.py b/src/python/pants/backend/experimental/openapi/codegen/python/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/experimental/openapi/codegen/python/register.py b/src/python/pants/backend/experimental/openapi/codegen/python/register.py new file mode 100644 index 00000000000..bdb2ba67e32 --- /dev/null +++ b/src/python/pants/backend/experimental/openapi/codegen/python/register.py @@ -0,0 +1,18 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from pants.backend.experimental.openapi.register import rules as openapi_rules +from pants.backend.experimental.openapi.register import target_types as openapi_target_types +from pants.backend.experimental.python.register import rules as python_rules +from pants.backend.experimental.python.register import target_types as python_target_types +from pants.backend.openapi.codegen.python.rules import rules as openapi_python_codegen_rules + + +def target_types(): + return [*python_target_types(), *openapi_target_types()] + + +def rules(): + return [*python_rules(), *openapi_rules(), *openapi_python_codegen_rules()] diff --git a/src/python/pants/backend/openapi/codegen/java/rules.py b/src/python/pants/backend/openapi/codegen/java/rules.py index 2976f632148..fd14295fc7e 100644 --- a/src/python/pants/backend/openapi/codegen/java/rules.py +++ b/src/python/pants/backend/openapi/codegen/java/rules.py @@ -21,10 +21,7 @@ OpenApiSourceField, ) from pants.backend.openapi.util_rules import generator_process, pom_parser -from pants.backend.openapi.util_rules.generator_process import ( - OpenAPIGeneratorProcess, - OpenAPIGeneratorType, -) +from pants.backend.openapi.util_rules.generator_process import OpenAPIGeneratorProcess from pants.backend.openapi.util_rules.pom_parser import AnalysePomRequest, PomReport from pants.engine.fs import ( EMPTY_SNAPSHOT, @@ -106,7 +103,7 @@ async def compile_openapi_into_java( merged_digests = await Get(Digest, MergeDigests([request.input_digest, output_digest])) process = OpenAPIGeneratorProcess( - generator_type=OpenAPIGeneratorType.JAVA, + generator_name="java", argv=[ *( ("--additional-properties", f"apiPackage={request.api_package}") diff --git a/src/python/pants/backend/openapi/codegen/python/BUILD b/src/python/pants/backend/openapi/codegen/python/BUILD new file mode 100644 index 00000000000..b20c17b5f68 --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/python/BUILD @@ -0,0 +1,8 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() + +python_tests(name="tests", dependencies=[":lockfiles"]) + +resources(name="lockfiles", sources=["*.test.lock"]) diff --git a/src/python/pants/backend/openapi/codegen/python/__init__.py b/src/python/pants/backend/openapi/codegen/python/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/openapi/codegen/python/extra_fields.py b/src/python/pants/backend/openapi/codegen/python/extra_fields.py new file mode 100644 index 00000000000..71efb488e5e --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/python/extra_fields.py @@ -0,0 +1,42 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +import itertools + +from pants.backend.openapi.target_types import OpenApiDocumentGeneratorTarget, OpenApiDocumentTarget +from pants.backend.python.target_types import PrefixedPythonResolveField +from pants.engine.target import BoolField, DictStringToStringField, StringField + + +class OpenApiPythonGeneratorNameField(StringField): + alias = "python_generator_name" + required = False + help = "Python generator name" + + +class OpenApiPythonAdditionalPropertiesField(DictStringToStringField): + alias = "python_additional_properties" + help = "Additional properties for python generator" + + +class OpenApiPythonSkipField(BoolField): + alias = "skip_python" + default = False + help = "If true, skips generation of Python sources from this target" + + +def rules(): + return [ + target.register_plugin_field(field) + for target, field in itertools.product( + ( + OpenApiDocumentTarget, + OpenApiDocumentGeneratorTarget, + ), + ( + OpenApiPythonSkipField, + OpenApiPythonGeneratorNameField, + OpenApiPythonAdditionalPropertiesField, + PrefixedPythonResolveField, + ), + ) + ] diff --git a/src/python/pants/backend/openapi/codegen/python/generate.py b/src/python/pants/backend/openapi/codegen/python/generate.py new file mode 100644 index 00000000000..8f511870ad4 --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/python/generate.py @@ -0,0 +1,345 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import itertools +import logging +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Tuple + +from packaging.utils import canonicalize_name as canonicalize_project_name + +from pants.backend.codegen.utils import MissingPythonCodegenRuntimeLibrary +from pants.backend.openapi.codegen.python.extra_fields import ( + OpenApiPythonAdditionalPropertiesField, + OpenApiPythonGeneratorNameField, + OpenApiPythonSkipField, +) +from pants.backend.openapi.sample.resources import PETSTORE_SAMPLE_SPEC +from pants.backend.openapi.subsystems.openapi_generator import OpenAPIGenerator +from pants.backend.openapi.target_types import ( + OpenApiDocumentDependenciesField, + OpenApiDocumentField, + OpenApiSourceField, +) +from pants.backend.openapi.util_rules.generator_process import OpenAPIGeneratorProcess +from pants.backend.python.dependency_inference.module_mapper import AllPythonTargets +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import ( + PrefixedPythonResolveField, + PythonRequirementResolveField, + PythonRequirementsField, + PythonSourceField, +) +from pants.engine.fs import ( + EMPTY_SNAPSHOT, + AddPrefix, + CreateDigest, + Digest, + DigestContents, + DigestSubset, + Directory, + FileContent, + MergeDigests, + PathGlobs, + RemovePrefix, + Snapshot, +) +from pants.engine.internals.native_engine import Address +from pants.engine.process import ProcessResult +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import ( + FieldSet, + GeneratedSources, + GenerateSourcesRequest, + HydratedSources, + HydrateSourcesRequest, + InferDependenciesRequest, + InferredDependencies, + TransitiveTargets, + TransitiveTargetsRequest, +) +from pants.engine.unions import UnionRule +from pants.source.source_root import SourceRoot, SourceRootRequest +from pants.util.frozendict import FrozenDict +from pants.util.logging import LogLevel +from pants.util.pip_requirement import PipRequirement +from pants.util.requirements import parse_requirements_file +from pants.util.strutil import softwrap + +logger = logging.getLogger(__name__) + + +class GeneratePythonFromOpenAPIRequest(GenerateSourcesRequest): + input = OpenApiDocumentField + output = PythonSourceField + + +@dataclass(frozen=True) +class OpenApiDocumentPythonFieldSet(FieldSet): + required_fields = (OpenApiDocumentField,) + + source: OpenApiDocumentField + dependencies: OpenApiDocumentDependenciesField + generator_name: OpenApiPythonGeneratorNameField + additional_properties: OpenApiPythonAdditionalPropertiesField + skip: OpenApiPythonSkipField + + +@dataclass(frozen=True) +class CompileOpenApiIntoPythonRequest: + input_file: str + input_digest: Digest + description: str + generator_name: str + additional_properties: FrozenDict[str, str] | None = None + + +@dataclass(frozen=True) +class CompiledPythonFromOpenApi: + output_digest: Digest + runtime_dependencies: tuple[PipRequirement, ...] + + +@rule +async def compile_openapi_into_python( + request: CompileOpenApiIntoPythonRequest, +) -> CompiledPythonFromOpenApi: + output_dir = "__gen" + output_digest = await Get(Digest, CreateDigest([Directory(output_dir)])) + + merged_digests = await Get(Digest, MergeDigests([request.input_digest, output_digest])) + + additional_properties: Iterable[str] = ( + itertools.chain( + *[ + ("--additional-properties", f"{k}={v}") + for k, v in request.additional_properties.items() + ] + ) + if request.additional_properties + else () + ) + + process = OpenAPIGeneratorProcess( + generator_name=request.generator_name, + argv=[ + *additional_properties, + "-i", + request.input_file, + "-o", + output_dir, + ], + input_digest=merged_digests, + output_directories=(output_dir,), + description=request.description, + level=LogLevel.DEBUG, + ) + + result = await Get(ProcessResult, OpenAPIGeneratorProcess, process) + normalized_digest = await Get(Digest, RemovePrefix(result.output_digest, output_dir)) + + requirements_digest, python_sources_digest = await MultiGet( + Get(Digest, DigestSubset(normalized_digest, PathGlobs(["requirements.txt"]))), + Get(Digest, DigestSubset(normalized_digest, PathGlobs(["**/*.py"]))), + ) + requirements_contents = await Get(DigestContents, Digest, requirements_digest) + runtime_dependencies: Tuple[PipRequirement, ...] = () + if len(requirements_contents) > 0: + file = requirements_contents[0] + runtime_dependencies = tuple( + parse_requirements_file( + file.content.decode("utf-8"), + rel_path=file.path, + ) + ) + + return CompiledPythonFromOpenApi( + output_digest=python_sources_digest, + runtime_dependencies=runtime_dependencies, + ) + + +@rule +async def generate_python_from_openapi( + request: GeneratePythonFromOpenAPIRequest, +) -> GeneratedSources: + field_set = OpenApiDocumentPythonFieldSet.create(request.protocol_target) + if field_set.skip.value: + return GeneratedSources(EMPTY_SNAPSHOT) + + (document_sources, transitive_targets) = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(field_set.source)), + Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), + ) + + document_dependencies = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(tgt[OpenApiSourceField])) + for tgt in transitive_targets.dependencies + if tgt.has_field(OpenApiSourceField) + ) + + input_digest = await Get( + Digest, + MergeDigests( + [ + document_sources.snapshot.digest, + *[dependency.snapshot.digest for dependency in document_dependencies], + ] + ), + ) + + gets = [] + for file in document_sources.snapshot.files: + generator_name = field_set.generator_name.value + if generator_name is None: + raise ValueError( + f"Field `{OpenApiPythonGeneratorNameField.alias}` is required for target {field_set.address}" + ) + + gets.append( + Get( + CompiledPythonFromOpenApi, + CompileOpenApiIntoPythonRequest( + file, + input_digest=input_digest, + description=f"Generating Python sources from OpenAPI definition {field_set.address}", + generator_name=generator_name, + additional_properties=field_set.additional_properties.value, + ), + ) + ) + + compiled_sources = await MultiGet(gets) + + logger.info("digests: %s", [sources.output_digest for sources in compiled_sources]) + output_digest, source_root = await MultiGet( + Get(Digest, MergeDigests([sources.output_digest for sources in compiled_sources])), + Get(SourceRoot, SourceRootRequest, SourceRootRequest.for_target(request.protocol_target)), + ) + + source_root_restored = ( + await Get(Snapshot, AddPrefix(output_digest, source_root.path)) + if source_root.path != "." + else await Get(Snapshot, Digest, output_digest) + ) + return GeneratedSources(source_root_restored) + + +@dataclass(frozen=True) +class OpenApiDocumentPythonRuntimeInferenceFieldSet(FieldSet): + required_fields = (OpenApiDocumentDependenciesField, PrefixedPythonResolveField) + + dependencies: OpenApiDocumentDependenciesField + python_resolve: PrefixedPythonResolveField + generator_name: OpenApiPythonGeneratorNameField + additional_properties: OpenApiPythonAdditionalPropertiesField + skip: OpenApiPythonSkipField + + +class InferOpenApiPythonRuntimeDependencyRequest(InferDependenciesRequest): + infer_from = OpenApiDocumentPythonRuntimeInferenceFieldSet + + +@dataclass(frozen=True) +class PythonRequirements: + resolves_to_requirements_to_addresses: FrozenDict[str, FrozenDict[str, Address]] + + +@rule +async def get_python_requirements( + python_targets: AllPythonTargets, + python_setup: PythonSetup, +) -> PythonRequirements: + result: defaultdict[str, dict[str, Address]] = defaultdict(dict) + for target in python_targets.third_party: + for python_requirement in target[PythonRequirementsField].value: + project_name = canonicalize_project_name(python_requirement.project_name) + resolve = target[PythonRequirementResolveField].normalized_value(python_setup) + result[resolve][project_name] = target.address + + return PythonRequirements( + resolves_to_requirements_to_addresses=FrozenDict( + ( + resolve, + FrozenDict( + (requirements, addresses) + for requirements, addresses in requirements_to_addresses.items() + ), + ) + for resolve, requirements_to_addresses in result.items() + ), + ) + + +@rule +async def infer_openapi_python_dependencies( + request: InferOpenApiPythonRuntimeDependencyRequest, + python_setup: PythonSetup, + openapi_generator: OpenAPIGenerator, + python_requirements: PythonRequirements, +) -> InferredDependencies: + if request.field_set.skip.value: + return InferredDependencies([]) + + resolve = request.field_set.python_resolve.normalized_value(python_setup) + + # Because the runtime dependencies are the same regardless of the source being compiled + # we use a sample OpenAPI spec to find out what are the runtime dependencies + # for the given resolve and prevent creating a cycle in our rule engine. + sample_spec_name = "__sample_spec.yaml" + sample_source_digest = await Get( + Digest, + CreateDigest( + [FileContent(path=sample_spec_name, content=PETSTORE_SAMPLE_SPEC.encode("utf-8"))] + ), + ) + compiled_sources = await Get( + CompiledPythonFromOpenApi, + CompileOpenApiIntoPythonRequest( + input_file=sample_spec_name, + input_digest=sample_source_digest, + description=f"Inferring Python runtime dependencies for OpenAPI v{openapi_generator.version}", + generator_name=request.field_set.generator_name.value, + additional_properties=request.field_set.additional_properties.value, + ), + ) + + logger.info("Looking for thirdparty dependencies: %s", compiled_sources.runtime_dependencies) + + requirements_to_addresses = python_requirements.resolves_to_requirements_to_addresses[resolve] + + addresses, missing_requirements = [], [] + for runtime_dependency in compiled_sources.runtime_dependencies: + project_name = runtime_dependency.project_name + address = requirements_to_addresses.get(project_name.lower()) + if address is not None: + addresses.append(address) + else: + missing_requirements.append(project_name) + + if missing_requirements: + for_resolve_str = f" for the resolve '{resolve}'" if python_setup.enable_resolves else "" + missing = ", ".join(f"`{project_name}`" for project_name in missing_requirements) + raise MissingPythonCodegenRuntimeLibrary( + softwrap( + f""" + No `python_requirement` target was found with the packages {missing} + in your project{for_resolve_str}, so the Python code generated from the target + {request.field_set.address} will not work properly. + """ + ) + ) + + return InferredDependencies(addresses) + + +def rules(): + return [ + *collect_rules(), + UnionRule(GenerateSourcesRequest, GeneratePythonFromOpenAPIRequest), + UnionRule(InferDependenciesRequest, InferOpenApiPythonRuntimeDependencyRequest), + ] diff --git a/src/python/pants/backend/openapi/codegen/python/generate_integration_test.py b/src/python/pants/backend/openapi/codegen/python/generate_integration_test.py new file mode 100644 index 00000000000..4f951b699b7 --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/python/generate_integration_test.py @@ -0,0 +1,411 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from importlib import resources +from textwrap import dedent +from typing import Iterable + +import pytest + +from pants.backend.openapi.codegen.python.generate import GeneratePythonFromOpenAPIRequest +from pants.backend.openapi.codegen.python.rules import rules as python_codegen_rules +from pants.backend.openapi.sample.resources import PETSTORE_SAMPLE_SPEC +from pants.backend.openapi.target_types import ( + OpenApiDocumentDependenciesField, + OpenApiDocumentField, + OpenApiDocumentGeneratorTarget, + OpenApiDocumentTarget, + OpenApiSourceGeneratorTarget, + OpenApiSourceTarget, +) +from pants.backend.openapi.target_types import rules as target_types_rules +from pants.backend.openapi.util_rules import openapi_bundle +from pants.backend.python.register import rules as python_backend_rules +from pants.backend.python.register import target_types as python_target_types +from pants.core.goals.test import rules as test_rules +from pants.core.util_rules.config_files import rules as config_files_rules +from pants.engine.addresses import Address, Addresses +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + GeneratedSources, + HydratedSources, + HydrateSourcesRequest, +) +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + target_types=[ + *python_target_types(), + OpenApiSourceTarget, + OpenApiSourceGeneratorTarget, + OpenApiDocumentTarget, + OpenApiDocumentGeneratorTarget, + ], + rules=[ + *test_rules(), + *config_files_rules(), + *python_backend_rules(), + *python_codegen_rules(), + *openapi_bundle.rules(), + *target_types_rules(), + QueryRule(HydratedSources, (HydrateSourcesRequest,)), + QueryRule(GeneratedSources, (GeneratePythonFromOpenAPIRequest,)), + QueryRule(Addresses, (DependenciesRequest,)), + ], + ) + rule_runner.set_options(args=[], env_inherit=PYTHON_BOOTSTRAP_ENV) + return rule_runner + + +def _get_generated_files( + rule_runner: RuleRunner, + address: Address, + *, + source_roots: Iterable[str] | None = None, + extra_args: Iterable[str] = (), +) -> tuple[str, ...]: + args = [] + if source_roots: + args.append(f"--source-root-patterns={repr(source_roots)}") + args.extend(extra_args) + rule_runner.set_options(args, env_inherit=PYTHON_BOOTSTRAP_ENV) + + tgt = rule_runner.get_target(address) + protocol_sources = rule_runner.request( + HydratedSources, [HydrateSourcesRequest(tgt[OpenApiDocumentField])] + ) + generated_sources = rule_runner.request( + GeneratedSources, + [GeneratePythonFromOpenAPIRequest(protocol_sources.snapshot, tgt)], + ) + return generated_sources.snapshot.files + + +def test_skip_generate_python(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "BUILD": "openapi_document(name='petstore', source='petstore_spec.yaml', skip_python=True)", + "petstore_spec.yaml": PETSTORE_SAMPLE_SPEC, + } + ) + + def assert_gen(address: Address, expected: Iterable[str]) -> None: + generated_files = _get_generated_files(rule_runner, address) + # We only assert expected files are a subset of all generated since the generator creates a lot of support classes + assert set(generated_files) == set(expected) + + tgt_address = Address("", target_name="petstore") + assert_gen(tgt_address, []) + + tgt = rule_runner.get_target(tgt_address) + runtime_dependencies = rule_runner.request( + Addresses, [DependenciesRequest(tgt[OpenApiDocumentDependenciesField])] + ) + assert not runtime_dependencies + + +@pytest.fixture +def requirements_text() -> str: + return dedent( + """\ + python_requirement( + name="urllib3", + requirements=["urllib3"], + ) + python_requirement( + name="python-dateutil", + requirements=["python-dateutil"], + ) + python_requirement( + name="setuptools", + requirements=["setuptools"], + ) + """ + ) + + +def test_generate_python_sources(rule_runner: RuleRunner, requirements_text: str) -> None: + rule_runner.write_files( + { + "3rdparty/python/default.lock": resources.files(__package__) + .joinpath("openapi.test.lock") + .read_text(), + "3rdparty/python/BUILD": requirements_text, + "src/openapi/BUILD": dedent( + """\ + openapi_document( + name="petstore", + source="petstore_spec.yaml", + python_generator_name="python", + ) + """ + ), + "src/openapi/petstore_spec.yaml": PETSTORE_SAMPLE_SPEC, + } + ) + + def assert_gen(address: Address, expected: Iterable[str]) -> None: + generated_files = _get_generated_files(rule_runner, address, source_roots=["src/openapi"]) + # We only assert expected files are a subset of all generated since the generator creates a lot of support classes + assert set(generated_files) == set(expected) + + tgt_address = Address("src/openapi", target_name="petstore") + assert_gen( + tgt_address, + [ + # The list might change because it depends on openapi template. + # TODO Vendor template? + "src/openapi/openapi_client/__init__.py", + "src/openapi/openapi_client/api/__init__.py", + "src/openapi/openapi_client/api/pets_api.py", + "src/openapi/openapi_client/api_client.py", + "src/openapi/openapi_client/apis/__init__.py", + "src/openapi/openapi_client/configuration.py", + "src/openapi/openapi_client/exceptions.py", + "src/openapi/openapi_client/model/__init__.py", + "src/openapi/openapi_client/model/error.py", + "src/openapi/openapi_client/model/pet.py", + "src/openapi/openapi_client/model/pets.py", + "src/openapi/openapi_client/model_utils.py", + "src/openapi/openapi_client/models/__init__.py", + "src/openapi/openapi_client/rest.py", + "src/openapi/setup.py", + "src/openapi/test/__init__.py", + "src/openapi/test/test_error.py", + "src/openapi/test/test_pet.py", + "src/openapi/test/test_pets.py", + "src/openapi/test/test_pets_api.py", + ], + ) + + tgt = rule_runner.get_target(tgt_address) + runtime_dependencies = rule_runner.request( + Addresses, [DependenciesRequest(tgt[OpenApiDocumentDependenciesField])] + ) + assert runtime_dependencies + + +@pytest.fixture +def fastapi_requirements_text() -> str: + return dedent( + """\ + python_requirement(name="jinja2", requirements=["jinja2"]) + python_requirement(name="markupsafe", requirements=["markupsafe"]) + python_requirement(name="pyyaml", requirements=["pyyaml"]) + python_requirement(name="rx", requirements=["rx"]) + python_requirement(name="aiofiles", requirements=["aiofiles"]) + python_requirement(name="aniso8601", requirements=["aniso8601"]) + python_requirement(name="async-exit-stack", requirements=["async-exit-stack"]) + python_requirement(name="async-generator", requirements=["async-generator"]) + python_requirement(name="certifi", requirements=["certifi"]) + python_requirement(name="chardet", requirements=["chardet"]) + python_requirement(name="click", requirements=["click"]) + python_requirement(name="dnspython", requirements=["dnspython"]) + python_requirement(name="email-validator", requirements=["email-validator"]) + python_requirement(name="fastapi", requirements=["fastapi"]) + python_requirement(name="graphene", requirements=["graphene"]) + python_requirement(name="graphql-core", requirements=["graphql-core"]) + python_requirement(name="graphql-relay", requirements=["graphql-relay"]) + python_requirement(name="h11", requirements=["h11"]) + python_requirement(name="httptools", requirements=["httptools"]) + python_requirement(name="idna", requirements=["idna"]) + python_requirement(name="itsdangerous", requirements=["itsdangerous"]) + python_requirement(name="orjson", requirements=["orjson"]) + python_requirement(name="promise", requirements=["promise"]) + python_requirement(name="pydantic", requirements=["pydantic"]) + python_requirement(name="python-dotenv", requirements=["python-dotenv"]) + python_requirement(name="python-multipart", requirements=["python-multipart"]) + python_requirement(name="requests", requirements=["requests"]) + python_requirement(name="six", requirements=["six"]) + python_requirement(name="starlette", requirements=["starlette"]) + python_requirement(name="typing-extensions", requirements=["typing-extensions"]) + python_requirement(name="ujson", requirements=["ujson"]) + python_requirement(name="urllib3", requirements=["urllib3"]) + python_requirement(name="uvicorn", requirements=["uvicorn"]) + python_requirement(name="uvloop", requirements=["uvloop"]) + python_requirement(name="watchgod", requirements=["watchgod"]) + python_requirement(name="websockets", requirements=["websockets"]) + """ + ) + + +def test_generate_python_sources_with_a_different_generator( + rule_runner: RuleRunner, fastapi_requirements_text: str +) -> None: + rule_runner.write_files( + { + "3rdparty/python/default.lock": resources.files(__package__) + .joinpath("openapi.test.lock") + .read_text(), + "3rdparty/python/BUILD": fastapi_requirements_text, + "src/openapi/BUILD": dedent( + """\ + openapi_document( + name="petstore", + source="petstore_spec.yaml", + python_generator_name="python-fastapi", + ) + """ + ), + "src/openapi/petstore_spec.yaml": PETSTORE_SAMPLE_SPEC, + } + ) + + def assert_gen(address: Address, expected: Iterable[str]) -> None: + generated_files = _get_generated_files(rule_runner, address, source_roots=["src/openapi"]) + # We only assert expected files are a subset of all generated since the generator creates a lot of support classes + assert set(generated_files) == set(expected) + + tgt_address = Address("src/openapi", target_name="petstore") + assert_gen( + tgt_address, + [ + # The list might change because it depends on openapi template. + # TODO Vendor template? + "src/openapi/src/openapi_server/apis/__init__.py", + "src/openapi/src/openapi_server/apis/pets_api.py", + "src/openapi/src/openapi_server/main.py", + "src/openapi/src/openapi_server/models/__init__.py", + "src/openapi/src/openapi_server/models/error.py", + "src/openapi/src/openapi_server/models/extra_models.py", + "src/openapi/src/openapi_server/models/pet.py", + "src/openapi/src/openapi_server/security_api.py", + "src/openapi/tests/conftest.py", + "src/openapi/tests/test_pets_api.py", + ], + ) + + tgt = rule_runner.get_target(tgt_address) + runtime_dependencies = rule_runner.request( + Addresses, [DependenciesRequest(tgt[OpenApiDocumentDependenciesField])] + ) + assert runtime_dependencies + + +def test_openapi_generator_name_validation(rule_runner: RuleRunner, requirements_text: str): + rule_runner.write_files( + { + "3rdparty/python/default.lock": resources.files(__package__) + .joinpath("openapi.test.lock") + .read_text(), + "3rdparty/python/BUILD": requirements_text, + "src/openapi/BUILD": dedent( + """\ + openapi_document( + name="petstore", + source="petstore_spec.yaml", + python_generator_name="python-xxx", + ) + """ + ), + "src/openapi/petstore_spec.yaml": PETSTORE_SAMPLE_SPEC, + } + ) + + address = Address("src/openapi", target_name="petstore") + with pytest.raises( + ExecutionError, + match="ValueError: OpenAPI generator `python-xxx` is not found, available generators: ", + ): + _get_generated_files(rule_runner, address, source_roots=["src/openapi"]) + + +def test_generate_python_sources_using_custom_package_name( + rule_runner: RuleRunner, + requirements_text: str, +) -> None: + rule_runner.write_files( + { + "3rdparty/python/default.lock": resources.files(__package__) + .joinpath("openapi.test.lock") + .read_text(), + "3rdparty/python/BUILD": requirements_text, + "src/openapi/BUILD": dedent( + """\ + openapi_document( + name="petstore", + source="petstore_spec.yaml", + python_generator_name="python", + python_additional_properties={ + "packageName": "petstore_client", + }, + ) + """ + ), + "src/openapi/petstore_spec.yaml": PETSTORE_SAMPLE_SPEC, + } + ) + + def assert_gen(address: Address, expected: Iterable[str]) -> None: + generated_files = _get_generated_files(rule_runner, address, source_roots=["src/openapi"]) + # We only assert expected files are a subset of all generated since the generator creates a lot of support classes + assert set(generated_files) == set(expected) + + assert_gen( + Address("src/openapi", target_name="petstore"), + [ + # The list might change because it depends on openapi template. + # TODO Vendor template? + "src/openapi/petstore_client/__init__.py", + "src/openapi/petstore_client/api/__init__.py", + "src/openapi/petstore_client/api/pets_api.py", + "src/openapi/petstore_client/api_client.py", + "src/openapi/petstore_client/apis/__init__.py", + "src/openapi/petstore_client/configuration.py", + "src/openapi/petstore_client/exceptions.py", + "src/openapi/petstore_client/model/__init__.py", + "src/openapi/petstore_client/model/error.py", + "src/openapi/petstore_client/model/pet.py", + "src/openapi/petstore_client/model/pets.py", + "src/openapi/petstore_client/model_utils.py", + "src/openapi/petstore_client/models/__init__.py", + "src/openapi/petstore_client/rest.py", + "src/openapi/setup.py", + "src/openapi/test/__init__.py", + "src/openapi/test/test_error.py", + "src/openapi/test/test_pet.py", + "src/openapi/test/test_pets.py", + "src/openapi/test/test_pets_api.py", + ], + ) + + +def test_python_dependency_inference(rule_runner: RuleRunner, requirements_text: str) -> None: + rule_runner.write_files( + { + "3rdparty/python/default.lock": resources.files(__package__) + .joinpath("openapi.test.lock") + .read_text(), + "3rdparty/python/BUILD": requirements_text, + "src/openapi/BUILD": dedent( + """\ + openapi_document( + name="petstore", + source="petstore_spec.yaml", + python_generator_name="python", + python_additional_properties={ + "packageName": "petstore_client", + }, + ) + """ + ), + "src/openapi/petstore_spec.yaml": PETSTORE_SAMPLE_SPEC, + "src/python/BUILD": "python_sources()", + "src/python/example.py": dedent( + """\ + from petstore_client.api_client import ApiClient + """ + ), + } + ) + + tgt = rule_runner.get_target(Address("src/python", relative_file_path="example.py")) + dependencies = rule_runner.request(Addresses, [DependenciesRequest(tgt[Dependencies])]) + assert Address("src/openapi", target_name="petstore") in dependencies diff --git a/src/python/pants/backend/openapi/codegen/python/openapi.test.lock b/src/python/pants/backend/openapi/codegen/python/openapi.test.lock new file mode 100644 index 00000000000..cf062cb3b54 --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/python/openapi.test.lock @@ -0,0 +1,195 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// pants generate-lockfiles --resolve=custom +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 3, +// "valid_for_interpreter_constraints": [ +// "CPython==3.11.*", +// "CPython==3.12.*" +// ], +// "generated_with_requirements": [ +// "python-dateutil", +// "setuptools", +// "urllib3" +// ], +// "manylinux": "manylinux2014", +// "requirement_constraints": [], +// "only_binary": [], +// "no_binary": [] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "excluded": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", + "url": "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "url": "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz" + } + ], + "project_name": "python-dateutil", + "requires_dists": [ + "six>=1.5" + ], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7", + "version": "2.9.0.post0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4", + "url": "https://files.pythonhosted.org/packages/6e/ec/06715d912351edc453e37f93f3fc80dcffd5ca0e70386c87529aca296f05/setuptools-72.2.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9", + "url": "https://files.pythonhosted.org/packages/ce/ef/013ded5b0d259f3fa636bf35de186f0061c09fbe124020ce6b8db68c83af/setuptools-72.2.0.tar.gz" + } + ], + "project_name": "setuptools", + "requires_dists": [ + "build[virtualenv]>=1.0.3; extra == \"test\"", + "filelock>=3.4.0; extra == \"test\"", + "furo; extra == \"doc\"", + "importlib-metadata; extra == \"test\"", + "importlib-metadata>=6; python_version < \"3.10\" and extra == \"core\"", + "importlib-resources>=5.10.2; python_version < \"3.9\" and extra == \"core\"", + "ini2toml[lite]>=0.14; extra == \"test\"", + "jaraco.develop>=7.21; (python_version >= \"3.9\" and sys_platform != \"cygwin\") and extra == \"test\"", + "jaraco.envs>=2.2; extra == \"test\"", + "jaraco.packaging>=9.3; extra == \"doc\"", + "jaraco.path>=3.2.0; extra == \"test\"", + "jaraco.test; extra == \"test\"", + "jaraco.text>=3.7; extra == \"core\"", + "jaraco.tidelift>=1.4; extra == \"doc\"", + "more-itertools>=8.8; extra == \"core\"", + "mypy==1.11.*; extra == \"test\"", + "ordered-set>=3.1.1; extra == \"core\"", + "packaging>=23.2; extra == \"test\"", + "packaging>=24; extra == \"core\"", + "pip>=19.1; extra == \"test\"", + "platformdirs>=2.6.2; extra == \"core\"", + "pygments-github-lexers==0.0.5; extra == \"doc\"", + "pyproject-hooks!=1.1; extra == \"doc\"", + "pyproject-hooks!=1.1; extra == \"test\"", + "pytest!=8.1.*,>=6; extra == \"test\"", + "pytest-checkdocs>=2.4; extra == \"test\"", + "pytest-cov; extra == \"test\"", + "pytest-enabler>=2.2; extra == \"test\"", + "pytest-home>=0.5; extra == \"test\"", + "pytest-mypy; extra == \"test\"", + "pytest-perf; sys_platform != \"cygwin\" and extra == \"test\"", + "pytest-ruff<0.4; platform_system == \"Windows\" and extra == \"test\"", + "pytest-ruff>=0.2.1; sys_platform != \"cygwin\" and extra == \"test\"", + "pytest-ruff>=0.3.2; sys_platform != \"cygwin\" and extra == \"test\"", + "pytest-subprocess; extra == \"test\"", + "pytest-timeout; extra == \"test\"", + "pytest-xdist>=3; extra == \"test\"", + "rst.linker>=1.9; extra == \"doc\"", + "sphinx-favicon; extra == \"doc\"", + "sphinx-inline-tabs; extra == \"doc\"", + "sphinx-lint; extra == \"doc\"", + "sphinx-notfound-page<2,>=1; extra == \"doc\"", + "sphinx-reredirects; extra == \"doc\"", + "sphinx>=3.5; extra == \"doc\"", + "sphinxcontrib-towncrier; extra == \"doc\"", + "tomli-w>=1.0.0; extra == \"test\"", + "tomli; extra == \"test\"", + "tomli>=2.0.1; python_version < \"3.11\" and extra == \"core\"", + "towncrier<24.7; extra == \"doc\"", + "virtualenv>=13.0.0; extra == \"test\"", + "wheel; extra == \"test\"", + "wheel>=0.43.0; extra == \"core\"" + ], + "requires_python": ">=3.8", + "version": "72.2.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", + "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "url": "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz" + } + ], + "project_name": "six", + "requires_dists": [], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7", + "version": "1.16.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", + "url": "https://files.pythonhosted.org/packages/ca/1c/89ffc63a9605b583d5df2be791a27bc1a42b7c32bab68d3c8f2f73a98cd4/urllib3-2.2.2-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168", + "url": "https://files.pythonhosted.org/packages/43/6d/fa469ae21497ddc8bc93e5877702dca7cb8f911e337aca7452b5724f1bb6/urllib3-2.2.2.tar.gz" + } + ], + "project_name": "urllib3", + "requires_dists": [ + "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"", + "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"", + "h2<5,>=4; extra == \"h2\"", + "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", + "zstandard>=0.18.0; extra == \"zstd\"" + ], + "requires_python": ">=3.8", + "version": "2.2.2" + } + ], + "platform_tag": null + } + ], + "only_builds": [], + "only_wheels": [], + "overridden": [], + "path_mappings": {}, + "pex_version": "2.11.0", + "pip_version": "23.2", + "prefer_older_binary": false, + "requirements": [ + "python-dateutil", + "setuptools", + "urllib3" + ], + "requires_python": [ + "==3.11.*", + "==3.12.*" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/src/python/pants/backend/openapi/codegen/python/package_mapper.py b/src/python/pants/backend/openapi/codegen/python/package_mapper.py new file mode 100644 index 00000000000..7684173eff9 --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/python/package_mapper.py @@ -0,0 +1,75 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +import logging +from collections import defaultdict +from pathlib import PurePath +from typing import DefaultDict + +from pants.backend.openapi.codegen.python.generate import GeneratePythonFromOpenAPIRequest +from pants.backend.openapi.target_types import AllOpenApiDocumentTargets, OpenApiDocumentField +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + FirstPartyPythonMappingImplMarker, + ModuleProvider, + ModuleProviderType, + ResolveName, + module_from_stripped_path, +) +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import PythonResolveField +from pants.core.util_rules.stripped_source_files import StrippedFileName, StrippedFileNameRequest +from pants.engine.internals.selectors import Get, MultiGet +from pants.engine.rules import collect_rules, rule +from pants.engine.target import GeneratedSources, HydratedSources, HydrateSourcesRequest +from pants.engine.unions import UnionRule + +logger = logging.getLogger(__name__) + + +class PythonOpenApiMappingMarker(FirstPartyPythonMappingImplMarker): + pass + + +@rule +async def map_openapi_documents_to_python_modules( + all_openapi_document_targets: AllOpenApiDocumentTargets, + python_setup: PythonSetup, + _: PythonOpenApiMappingMarker, +) -> FirstPartyPythonMappingImpl: + hydrated_sources = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(target[OpenApiDocumentField])) + for target in all_openapi_document_targets + ) + generated_sources = await MultiGet( + Get(GeneratedSources, GeneratePythonFromOpenAPIRequest(sources.snapshot, target)) + for (target, sources) in zip(all_openapi_document_targets, hydrated_sources) + ) + stripped_file_per_target_sources = await MultiGet( + MultiGet( + Get(StrippedFileName, StrippedFileNameRequest(file)) for file in sources.snapshot.files + ) + for sources in generated_sources + ) + + resolves_to_modules_to_providers: DefaultDict[ + ResolveName, DefaultDict[str, list[ModuleProvider]] + ] = defaultdict(lambda: defaultdict(list)) + + for target, files in zip(all_openapi_document_targets, stripped_file_per_target_sources): + resolve_name = target[PythonResolveField].normalized_value(python_setup) + provider = ModuleProvider(target.address, ModuleProviderType.IMPL) + for stripped_file in files: + stripped_f = PurePath(stripped_file.value) + module = module_from_stripped_path(stripped_f) + resolves_to_modules_to_providers[resolve_name][module].append(provider) + + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) + + +def rules(): + return [ + *collect_rules(), + UnionRule(FirstPartyPythonMappingImplMarker, PythonOpenApiMappingMarker), + ] diff --git a/src/python/pants/backend/openapi/codegen/python/rules.py b/src/python/pants/backend/openapi/codegen/python/rules.py new file mode 100644 index 00000000000..b15c87fea0b --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/python/rules.py @@ -0,0 +1,14 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from pants.backend.openapi.codegen.python import extra_fields, generate, package_mapper +from pants.backend.openapi.util_rules import generator_process, pom_parser + + +def rules(): + return [ + *generate.rules(), + *extra_fields.rules(), + *generator_process.rules(), + *pom_parser.rules(), + *package_mapper.rules(), + ] diff --git a/src/python/pants/backend/openapi/util_rules/generator_process.py b/src/python/pants/backend/openapi/util_rules/generator_process.py index b5e4480df7c..c71542f2a96 100644 --- a/src/python/pants/backend/openapi/util_rules/generator_process.py +++ b/src/python/pants/backend/openapi/util_rules/generator_process.py @@ -4,14 +4,15 @@ from __future__ import annotations import dataclasses +import re from dataclasses import dataclass -from enum import Enum, unique -from typing import Iterable, Mapping +from typing import Iterable, Iterator, Mapping from pants.backend.openapi.subsystems import openapi_generator from pants.backend.openapi.subsystems.openapi_generator import OpenAPIGenerator from pants.engine.fs import Digest -from pants.engine.process import Process, ProcessCacheScope +from pants.engine.internals.native_engine import EMPTY_DIGEST +from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.engine.rules import Get, collect_rules, rule from pants.jvm import jdk_rules, non_jvm_dependencies from pants.jvm.jdk_rules import InternalJdk, JvmProcess @@ -23,15 +24,10 @@ from pants.util.logging import LogLevel -@unique -class OpenAPIGeneratorType(Enum): - JAVA = "java" - - @dataclass(frozen=True) class OpenAPIGeneratorProcess: argv: tuple[str, ...] - generator_type: OpenAPIGeneratorType + generator_name: str input_digest: Digest description: str = dataclasses.field(compare=False) level: LogLevel @@ -46,7 +42,7 @@ class OpenAPIGeneratorProcess: def __init__( self, *, - generator_type: OpenAPIGeneratorType, + generator_name: str, argv: Iterable[str], input_digest: Digest, description: str, @@ -59,7 +55,7 @@ def __init__( extra_jvm_options: Iterable[str] | None = None, cache_scope: ProcessCacheScope | None = None, ): - object.__setattr__(self, "generator_type", generator_type) + object.__setattr__(self, "generator_name", generator_name) object.__setattr__(self, "argv", tuple(argv)) object.__setattr__(self, "input_digest", input_digest) object.__setattr__(self, "description", description) @@ -78,10 +74,62 @@ def __init__( _GENERATOR_CLASS_NAME = "org.openapitools.codegen.OpenAPIGenerator" +@dataclass(frozen=True) +class OpenAPIGeneratorNames: + names: tuple[str, ...] + + +def _parse_names(stdout: str) -> Iterator[str]: + regex = re.compile(r"^ *- (?P[^ ]+)") + for line in stdout.splitlines(): + if (match := regex.match(line)) is not None: + yield match.group("name") + + +@rule +async def get_openapi_generator_names( + subsystem: OpenAPIGenerator, jdk: InternalJdk +) -> OpenAPIGeneratorNames: + tool_classpath = await Get( + ToolClasspath, ToolClasspathRequest(lockfile=GenerateJvmLockfileFromTool.create(subsystem)) + ) + + toolcp_relpath = "__toolcp" + immutable_input_digests = { + toolcp_relpath: tool_classpath.digest, + } + + classpath_entries = [ + *tool_classpath.classpath_entries(toolcp_relpath), + ] + + jvm_process = JvmProcess( + jdk=jdk, + argv=[_GENERATOR_CLASS_NAME, "list"], + classpath_entries=classpath_entries, + input_digest=EMPTY_DIGEST, + extra_immutable_input_digests=immutable_input_digests, + extra_jvm_options=subsystem.jvm_options, + description="Get openapi generator names.", + cache_scope=ProcessCacheScope.SUCCESSFUL, + ) + result = await Get(ProcessResult, JvmProcess, jvm_process) + return OpenAPIGeneratorNames(names=tuple(_parse_names(result.stdout.decode("utf-8")))) + + @rule async def openapi_generator_process( - request: OpenAPIGeneratorProcess, jdk: InternalJdk, subsystem: OpenAPIGenerator + request: OpenAPIGeneratorProcess, + jdk: InternalJdk, + subsystem: OpenAPIGenerator, + generator_names: OpenAPIGeneratorNames, ) -> Process: + if request.generator_name not in generator_names.names: + names = ", ".join(f"`{name}`" for name in generator_names.names) + raise ValueError( + f"OpenAPI generator `{request.generator_name}` is not found, available generators: {names}" + ) + tool_classpath = await Get( ToolClasspath, ToolClasspathRequest(lockfile=GenerateJvmLockfileFromTool.create(subsystem)) ) @@ -105,7 +153,7 @@ async def openapi_generator_process( _GENERATOR_CLASS_NAME, "generate", "-g", - request.generator_type.value, + request.generator_name, *request.argv, ], classpath_entries=classpath_entries, diff --git a/src/python/pants/backend/openapi/util_rules/generator_process_test.py b/src/python/pants/backend/openapi/util_rules/generator_process_test.py index fb6e3065d99..26c166a0eed 100644 --- a/src/python/pants/backend/openapi/util_rules/generator_process_test.py +++ b/src/python/pants/backend/openapi/util_rules/generator_process_test.py @@ -6,10 +6,7 @@ import pytest from pants.backend.openapi.util_rules import generator_process -from pants.backend.openapi.util_rules.generator_process import ( - OpenAPIGeneratorProcess, - OpenAPIGeneratorType, -) +from pants.backend.openapi.util_rules.generator_process import OpenAPIGeneratorProcess from pants.core.util_rules import config_files, external_tool, source_files, system_binaries from pants.engine.fs import EMPTY_DIGEST from pants.engine.process import Process @@ -37,12 +34,12 @@ def rule_runner() -> RuleRunner: @maybe_skip_jdk_test def test_generator_process(rule_runner: RuleRunner) -> None: generator_process = OpenAPIGeneratorProcess( - generator_type=OpenAPIGeneratorType.JAVA, + generator_name="java", argv=["foo"], description="Test generator process", input_digest=EMPTY_DIGEST, ) process = rule_runner.request(Process, [generator_process]) - assert OpenAPIGeneratorType.JAVA.value in process.argv + assert "java" in process.argv assert "org.openapitools.codegen.OpenAPIGenerator" in process.argv diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 0076953b685..e8fcdec26a7 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -164,6 +164,10 @@ def normalized_value(self, python_setup: PythonSetup) -> str: return resolve +class PrefixedPythonResolveField(PythonResolveField): + alias = "python_resolve" + + class PythonRunGoalUseSandboxField(TriBoolField): alias = "run_goal_use_sandbox" help = help_text( @@ -1341,7 +1345,7 @@ class PythonRequirementTypeStubModulesField(StringSequenceField): def normalize_module_mapping( - mapping: Mapping[str, Iterable[str]] | None + mapping: Mapping[str, Iterable[str]] | None, ) -> FrozenDict[str, tuple[str, ...]]: return FrozenDict({canonicalize_project_name(k): tuple(v) for k, v in (mapping or {}).items()})