From 991444a21f78f1a14230ab8e16f5df9687a64be7 Mon Sep 17 00:00:00 2001 From: David Dymko Date: Mon, 1 Mar 2021 13:46:07 -0500 Subject: [PATCH 01/22] merged in upstream --- ChangeLog | 3 + README.md | 38 +- cloudinit/apport.py | 1 + cloudinit/settings.py | 1 + cloudinit/sources/DataSourceVultr.py | 118 +++++ cloudinit/sources/helpers/vultr.py | 337 ++++++++++++++ cloudinit/version.py | 4 + doc/rtd/topics/availability.rst | 4 + doc/rtd/topics/datasources.rst | 2 +- doc/rtd/topics/datasources/vultr.rst | 35 ++ doc/rtd/topics/network-config.rst | 11 + integration-requirements.txt | 4 + .../unittests/test_datasource/test_common.py | 19 + tests/unittests/test_datasource/test_vultr.py | 439 ++++++++++++++++++ tools/ds-identify | 17 +- 15 files changed, 1030 insertions(+), 3 deletions(-) create mode 100644 cloudinit/sources/DataSourceVultr.py create mode 100644 cloudinit/sources/helpers/vultr.py create mode 100644 doc/rtd/topics/datasources/vultr.rst create mode 100644 tests/unittests/test_datasource/test_vultr.py diff --git a/ChangeLog b/ChangeLog index 44b504107f4..9fd4c619df6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +<<<<<<< HEAD 21.1 - Azure: Support for VMs without ephemeral resource disks. (#800) [Johnson Shi] (LP: #1901011) @@ -105,6 +106,8 @@ - cla: add xnox (#692) [Dimitri John Ledkov] - Collect logs from integration test runs (#675) +======= +>>>>>>> vultr-support 20.4.1 - Revert "ssh_util: handle non-default AuthorizedKeysFile config (#586)" diff --git a/README.md b/README.md index 435405dab7a..6e01751f499 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,39 @@ +# **Note:** Vultr support is work-in-progress + +This current fork and branch [vultr-support](https://github.com/vultr/cloud-init) is currently in a `beta` state and is a work in progress. The code and functionality may be changed or altered at any point during this development phase. + +Please **do not** use this in a production environment. + +The following repo can be built from source or you can use one of the +following packaged builds. + +### RHEL +Latest +* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_rhel_latest.rpm +* https://ewr1.vultrobjects.com/cloud_init_beta/rhel_latest_MD5 + +Nightly +* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_rhel_nightly.rpm +* https://ewr1.vultrobjects.com/cloud_init_beta/rhel_nightly_MD5 + +### Debian 10 (Possibly other versions) +Latest +* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_debian10_latest.deb +* https://ewr1.vultrobjects.com/cloud_init_beta/debian10_latest_MD5 + +Nightly +* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_debian10_nightly.deb +* https://ewr1.vultrobjects.com/cloud_init_beta/debian10_nightly_MD5 + +### Deb (Universal) +Latest +* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_universal_latest.deb +* https://ewr1.vultrobjects.com/cloud_init_beta/universal_latest_MD5 + +Nightly +* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_universal_nightly.deb +* https://ewr1.vultrobjects.com/cloud_init_beta/universal_nightly_MD5 + # cloud-init [![Build Status](https://travis-ci.com/canonical/cloud-init.svg?branch=master)](https://travis-ci.com/canonical/cloud-init) [![Read the Docs](https://readthedocs.org/projects/cloudinit/badge/?version=latest&style=flat)](https://cloudinit.readthedocs.org) @@ -39,7 +75,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| +| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| ## To start developing cloud-init diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 25f254e36ee..aadc638fb87 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -41,6 +41,7 @@ 'SmartOS', 'UpCloud', 'VMware', + 'Vultr', 'ZStack', 'Other' ] diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 91e1bfe7ed8..23e4c0ad819 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -30,6 +30,7 @@ 'GCE', 'OpenStack', 'AliYun', + 'Vultr', 'Ec2', 'CloudSigma', 'CloudStack', diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py new file mode 100644 index 00000000000..d700f77bcff --- /dev/null +++ b/cloudinit/sources/DataSourceVultr.py @@ -0,0 +1,118 @@ +# Author: Eric Benner +# +# This file is part of cloud-init. See LICENSE file for license information. + +# Vultr Metadata API: +# https://www.vultr.com/metadata/ + +import json +import re + +from cloudinit import log as log +from cloudinit import sources +from cloudinit import util + +import cloudinit.sources.helpers.vultr as vultr + +LOGGER = log.getLogger(__name__) +BUILTIN_DS_CONFIG = { + 'url': 'http://169.254.169.254', + 'retries': 30, + 'timeout': 2, + 'wait': 2 +} +CONFIG = BUILTIN_DS_CONFIG.copy() + + +class DataSourceVultr(sources.DataSource): + + dsname = 'Vultr' + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceVultr, self).__init__(sys_cfg, distro, paths) + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "Vultr"], {}), + BUILTIN_DS_CONFIG]) + CONFIG['url'] = self.ds_cfg.get( + 'url', BUILTIN_DS_CONFIG['url']) + CONFIG['retries'] = self.ds_cfg.get( + 'retries', BUILTIN_DS_CONFIG['retries']) + CONFIG['timeout'] = self.ds_cfg.get( + 'timeout', BUILTIN_DS_CONFIG['timeout']) + CONFIG['wait'] = self.ds_cfg.get( + 'wait', BUILTIN_DS_CONFIG['wait']) + + # Initiate data and check if Vultr + def _get_data(self): + LOGGER.info("Detecting if machine is a Vultr instance") + if not vultr.is_vultr(): + LOGGER.info("Machine is not a Vultr instance") + return False + + LOGGER.info("Machine is a Vultr instance") + + config = vultr.generate_config(CONFIG) + + # Dump vendor config so diagnosing failures is manageable + LOGGER.info("Vultr Vendor Config:") + LOGGER.info(json.dumps(config)) + + md = self.get_metadata() + + self.metadata_full = md + self.metadata['instanceid'] = self.metadata_full['instanceid'] + self.metadata['local-hostname'] = re.sub( + r'\W+', '', self.metadata_full['hostname']) + + # Default hostname is "vultr" + if self.metadata['local-hostname'] == "": + self.metadata['local-hostname'] = "vultr" + + self.metadata['public-keys'] = md["public-keys"].splitlines() + self.userdata_raw = md["user-data"] + if self.userdata_raw == "": + self.userdata_raw = None + self.vendordata_raw = "#cloud-config\n%s" % json.dumps(config) + + # Dump some data so diagnosing failures is manageable + LOGGER.info("SUBID: %s", self.metadata['instanceid']) + LOGGER.info("Hostname: %s", self.metadata['local-hostname']) + if self.userdata_raw is not None: + LOGGER.info("User-Data:") + LOGGER.info(self.userdata_raw) + + return True + + # Get the metadata by flag + def get_metadata(self): + return vultr.get_metadata(CONFIG) + + # Currently unsupported + @property + def launch_index(self): + return None + + # Write the base configs every time. These are subject to change + @property + def network_config(self): + config = vultr.generate_network_config(CONFIG) + config_raw = json.dumps(config) + + # Dump network config so diagnosing failures is manageable + LOGGER.info("Generated Network:") + LOGGER.info(config_raw) + + return config + + +# Used to match classes to dependencies +datasources = [ + (DataSourceVultr, (sources.DEP_FILESYSTEM, )), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py new file mode 100644 index 00000000000..d2201b96e1b --- /dev/null +++ b/cloudinit/sources/helpers/vultr.py @@ -0,0 +1,337 @@ +# Author: Eric Benner +# +# This file is part of cloud-init. See LICENSE file for license information. + +import json +import os +import copy +import re + +from cloudinit import log as log +from cloudinit import url_helper +from cloudinit import dmi +from cloudinit import util +from cloudinit import net +from cloudinit import subp +from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError + +# Get logger +LOGGER = log.getLogger(__name__) + +# Cache +MAC_TO_NICS = None +METADATA = None +EHP = None + + +def bring_up_interface(connectivity_url=None): + global EHP + + # If for whatever reason this is up, bail + if EHP is not None: + return + + # Make sure its not up already + if net.has_url_connectivity(connectivity_url): + return + + # Bring up interface in local + try: + EHP = EphemeralDHCPv4(net.find_fallback_nic()) + EHP.obtain_lease() + except (NoDHCPLeaseError) as exc: + LOGGER.error("DHCP failed, cannot continue. Exception: %s", + exc) + raise + + +# Close EphermalDHCP so its not left open +def close_ephermeral(): + global EHP + + # No action if its not open + if EHP is None: + return + + EHP.clean_network() + + # Cleanup + EHP = None + + +# Cache the metadata for optimization +def get_metadata(params): + global METADATA + + if not METADATA: + # Bring up interface in local + bring_up_interface(params['url']) + + # Fetch the metadata + v1 = fetch_metadata(params) + + # Close EphermeralDHCP when we are done + close_ephermeral() + + v1_json = json.loads(v1) + METADATA = v1_json + + # This comes through as a string but is JSON, make a dict + METADATA['vendor-config'] = json.loads(METADATA['vendor-config']) + + return METADATA + + +# Read the system information from SMBIOS +def get_sysinfo(): + return { + 'manufacturer': dmi.read_dmi_data("system-manufacturer"), + 'subid': dmi.read_dmi_data("system-serial-number"), + 'product': dmi.read_dmi_data("system-product-name"), + 'family': dmi.read_dmi_data("system-family") + } + + +# Get kernel parameters +def get_kernel_parameters(): + if not os.path.exists("/proc/cmdline"): + return "" + + file = open("/proc/cmdline") + content = file.read() + file.close() + + if "root=" not in content: + return "" + + return re.sub(r'.+root=', '', content)[1].strip() + + +# Confirm is Vultr +def is_vultr(): + # VC2, VDC, and HFC use DMI + sysinfo = get_sysinfo() + + if sysinfo['manufacturer'] == "Vultr": + return True + + # Baremetal requires a kernel parameter + if "vultr" in get_kernel_parameters(): + return True + + # An extra fallback if the others fail + # This needs to be a directory + if os.path.exists("/etc/vultr") and os.path.isdir("/etc/vultr"): + return True + + return False + + +# Write vendor startup script +def write_vendor_script(fname, content): + os.makedirs("/var/lib/scripts/vendor/", exist_ok=True) + file = open("/var/lib/scripts/vendor/%s" % fname, "w") + for line in content: + file.write(line) + file.close() + command = ["chmod", "+x", "/var/lib/scripts/vendor/%s" % fname] + + try: + subp.subp(command) + except Exception as err: + LOGGER.error( + "Command: %s failed to execute. Error: %s", + " ".join(command), err) + raise + + +# Read Metadata endpoint +def read_metadata(params): + response = url_helper.readurl(params['url'], + timeout=params['timeout'], + retries=params['retries'], + headers={'Metadata-Token': 'vultr'}, + sec_between=params['wait']) + + if not response.ok(): + raise RuntimeError("Failed to connect to %s: Code: %s" % + params['url'], response.code) + + return response.contents.decode() + + +# Get Metadata by flag +def fetch_metadata(params): + req = dict(params) + req['url'] = "%s/v1.json" % params['url'] + + return read_metadata(req) + + +# Convert macs to nics +def get_interface_name(mac): + global MAC_TO_NICS + + # Define it if empty + if not MAC_TO_NICS: + MAC_TO_NICS = net.get_interfaces_by_mac() + + if mac not in MAC_TO_NICS: + return None + + return MAC_TO_NICS.get(mac) + + +# Generate network configs +def generate_network_config(config): + md = get_metadata(config) + + network = { + "version": 1, + "config": [ + { + "type": "nameserver", + "address": [ + "108.61.10.10" + ] + } + ] + } + + # Prepare interface 0, public + if len(md['interfaces']) > 0: + network['config'].append(generate_public_network_interface(md)) + + # Prepare interface 1, private + if len(md['interfaces']) > 1: + network['config'].append(generate_private_network_interface(md)) + + return network + + +# Input Metadata and generate public network config part +def generate_public_network_interface(md): + interface_name = get_interface_name(md['interfaces'][0]['mac']) + if not interface_name: + raise RuntimeError( + "Interface: %s could not be found on the system" % + md['interfaces'][0]['mac']) + + netcfg = { + "name": interface_name, + "type": "physical", + "mac_address": md['interfaces'][0]['mac'], + "accept-ra": 1, + "subnets": [ + { + "type": "dhcp", + "control": "auto" + }, + { + "type": "dhcp6", + "control": "auto" + }, + ] + } + + # Check for additional IP's + additional_count = len(md['interfaces'][0]['ipv4']['additional']) + if "ipv4" in md['interfaces'][0] and additional_count > 0: + for additional in md['interfaces'][0]['ipv4']['additional']: + add = { + "type": "static", + "control": "auto", + "address": additional['address'], + "netmask": additional['netmask'] + } + netcfg['subnets'].append(add) + + # Check for additional IPv6's + additional_count = len(md['interfaces'][0]['ipv6']['additional']) + if "ipv6" in md['interfaces'][0] and additional_count > 0: + for additional in md['interfaces'][0]['ipv6']['additional']: + add = { + "type": "static6", + "control": "auto", + "address": additional['address'], + "netmask": additional['netmask'] + } + netcfg['subnets'].append(add) + + # Add config to template + return netcfg + + +# Input Metadata and generate private network config part +def generate_private_network_interface(md): + interface_name = get_interface_name(md['interfaces'][1]['mac']) + if not interface_name: + raise RuntimeError( + "Interface: %s could not be found on the system" % + md['interfaces'][1]['mac']) + + netcfg = { + "name": interface_name, + "type": "physical", + "mac_address": md['interfaces'][1]['mac'], + "accept-ra": 1, + "subnets": [ + { + "type": "static", + "control": "auto", + "address": md['interfaces'][1]['ipv4']['address'], + "netmask": md['interfaces'][1]['ipv4']['netmask'] + } + ] + } + + return netcfg + + +# Generate the vendor config +# This configuration is to replicate how +# images are deployed on Vultr before Cloud-Init +def generate_config(config): + md = get_metadata(config) + + # Grab the startup script + script = md['startup-script'] + + # Create vendor config + config_template = copy.deepcopy(md['vendor-config']) + + # Add generated network parts + config_template['network'] = generate_network_config(config) + + # Linux specific packages + if util.is_Linux(): + config_template['packages'].append("ethtool") + + # Define vendor script + vendor_script = [] + vendor_script.append("!/bin/bash") + + # Go through the interfaces + for netcfg in config_template['network']['config']: + # If the interface has a mac and is physical + if "mac_address" in netcfg and netcfg['type'] == "physical": + # Enable multi-queue on linux + # This is executed as a vendor script + if util.is_Linux(): + # Set its multi-queue to num of cores as per RHEL Docs + name = netcfg['name'] + command = "ethtool -L %s combined $(nproc --all)" % name + vendor_script.append(command) + + # Write vendor script + write_vendor_script("vultr_deploy.sh", vendor_script) + + # Write the startup script + if script and script != "echo No configured startup script": + lines = script.splitlines() + write_vendor_script("vultr_user_startup.sh", lines) + + return config_template + + +# vi: ts=4 expandtab diff --git a/cloudinit/version.py b/cloudinit/version.py index 94afd60dbc2..9104dc66de5 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,11 @@ # # This file is part of cloud-init. See LICENSE file for license information. +<<<<<<< HEAD __VERSION__ = "21.1" +======= +__VERSION__ = "20.4.1" +>>>>>>> vultr-support _PACKAGED_VERSION = '@@PACKAGED_VERSION@@' FEATURES = [ diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index f58b2b388e8..2dabcfdb05e 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -56,6 +56,10 @@ environments in the public cloud: - AltCloud - SmartOS - UpCloud +<<<<<<< HEAD +======= +- Vultr +>>>>>>> vultr-support Additionally, cloud-init is supported on these private clouds: diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index 228173d25d8..497b1467b9e 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -49,7 +49,7 @@ The following is a list of documents for each supported datasource: datasources/smartos.rst datasources/upcloud.rst datasources/zstack.rst - + datasources/vultr.rst Creation ======== diff --git a/doc/rtd/topics/datasources/vultr.rst b/doc/rtd/topics/datasources/vultr.rst new file mode 100644 index 00000000000..e73406a815a --- /dev/null +++ b/doc/rtd/topics/datasources/vultr.rst @@ -0,0 +1,35 @@ +.. _datasource_vultr: + +Vultr +===== + +The `Vultr`_ datasource retrieves basic configuration values from the locally +accessible `metadata service`_. All data is served over HTTP from the address +169.254.169.254. The endpoints are documented in, +`https://www.vultr.com/metadata/ +`_ + +Configuration +------------- + +Vultr's datasource can be configured as follows: + + datasource: + Vultr: + url: 'http://169.254.169.254' + retries: 3 + timeout: 2 + wait: 2 + +- *url*: The URL used to aquire the metadata configuration from +- *retries*: Determines the number of times to attempt to connect to the + metadata service +- *timeout*: Determines the timeout in seconds to wait for a response from the + metadata service +- *wait*: Determines the timeout in seconds to wait before retrying after + accessible failure + +.. _Vultr: https://www.vultr.com/ +.. _metadata service: https://www.vultr.com/metadata/ + +.. vi: textwidth=78 diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index 07cad765443..b9bd8ebc0c6 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -148,6 +148,13 @@ The following Datasources optionally provide network configuration: - `UpCloud JSON metadata`_ +<<<<<<< HEAD +======= +- :ref:`datasource_vultr` + + - `Vultr JSON metadata`_ + +>>>>>>> vultr-support For more information on network configuration formats .. toctree:: @@ -262,5 +269,9 @@ Example output converting V2 to sysconfig: .. _OpenStack Metadata Service Network: https://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/metadata-service-network-info.html .. _SmartOS JSON Metadata: https://eng.joyent.com/mdata/datadict.html .. _UpCloud JSON metadata: https://developers.upcloud.com/1.3/8-servers/#metadata-service +<<<<<<< HEAD +======= +.. _Vultr JSON metadata: https://www.vultr.com/metadata/ +>>>>>>> vultr-support .. vi: textwidth=78 diff --git a/integration-requirements.txt b/integration-requirements.txt index 6b5964260ed..52d6830a061 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,5 +1,9 @@ # PyPI requirements for cloud-init integration testing # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # +<<<<<<< HEAD pycloudlib @ git+https://github.com/canonical/pycloudlib.git@da8445325875674394ffd85aaefaa3d2d0e0020d +======= +pycloudlib @ git+https://github.com/canonical/pycloudlib.git@3a6c668fed769f00d83d1e6bea7d68953787cc38 +>>>>>>> vultr-support pytest diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 5912f7eeb1d..3c418da8699 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -28,6 +28,10 @@ DataSourceScaleway as Scaleway, DataSourceSmartOS as SmartOS, DataSourceUpCloud as UpCloud, +<<<<<<< HEAD +======= + DataSourceVultr as Vultr, +>>>>>>> vultr-support ) from cloudinit.sources import DataSourceNone as DSNone @@ -45,11 +49,19 @@ Oracle.DataSourceOracle, OVF.DataSourceOVF, SmartOS.DataSourceSmartOS, + Vultr.DataSourceVultr, Ec2.DataSourceEc2Local, OpenStack.DataSourceOpenStackLocal, RbxCloud.DataSourceRbxCloud, +<<<<<<< HEAD + Scaleway.DataSourceScaleway +======= Scaleway.DataSourceScaleway, UpCloud.DataSourceUpCloudLocal, +<<<<<<< HEAD +======= +>>>>>>> upstream/master +>>>>>>> vultr-support ] DEFAULT_NETWORK = [ @@ -64,8 +76,15 @@ MAAS.DataSourceMAAS, NoCloud.DataSourceNoCloudNet, OpenStack.DataSourceOpenStack, +<<<<<<< HEAD + OVF.DataSourceOVFNet +======= OVF.DataSourceOVFNet, UpCloud.DataSourceUpCloud, +<<<<<<< HEAD +======= +>>>>>>> upstream/master +>>>>>>> vultr-support ] diff --git a/tests/unittests/test_datasource/test_vultr.py b/tests/unittests/test_datasource/test_vultr.py new file mode 100644 index 00000000000..c947befe16d --- /dev/null +++ b/tests/unittests/test_datasource/test_vultr.py @@ -0,0 +1,439 @@ +# Author: Eric Benner +# +# This file is part of cloud-init. See LICENSE file for license information. + +# Vultr Metadata API: +# https://www.vultr.com/metadata/ + +import json + +from cloudinit import helpers +from cloudinit import settings +from cloudinit.sources import DataSourceVultr +from cloudinit.sources.helpers import vultr + +from cloudinit.tests.helpers import mock, CiTestCase + +# Vultr metadata test data +VULTR_V1_1 = { + 'bgp': { + 'ipv4': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + }, + 'ipv6': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + } + }, + 'hostname': 'CLOUDINIT_1', + 'instanceid': '42506325', + 'interfaces': [ + { + 'ipv4': { + 'additional': [ + ], + 'address': '108.61.89.242', + 'gateway': '108.61.89.1', + 'netmask': '255.255.255.0' + }, + 'ipv6': { + 'additional': [ + ], + 'address': '2001:19f0:5:56c2:5400:03ff:fe15:c465', + 'network': '2001:19f0:5:56c2::', + 'prefix': '64' + }, + 'mac': '56:00:03:15:c4:65', + 'network-type': 'public' + } + ], + 'public-keys': 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key\n', + 'region': { + 'regioncode': 'EWR' + }, + 'user-defined': [ + ], + 'startup-script': 'echo No configured startup script', + 'user-data': [ + ], + 'vendor-config': { + 'package_upgrade': 'true', + 'disable_root': 0, + 'packages': [], + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$S2Smuj.../VqxmIR9Urw0jPZ88i4yvB/' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } + } + } +} + +VULTR_V1_2 = { + 'bgp': { + 'ipv4': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + }, + 'ipv6': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + } + }, + 'hostname': 'CLOUDINIT_2', + 'instance-v2-id': '29bea708-2e6e-480a-90ad-0e6b5d5ad62f', + 'instanceid': '42872224', + 'interfaces': [ + { + 'ipv4': { + 'additional': [ + ], + 'address':'45.76.7.171', + 'gateway':'45.76.6.1', + 'netmask':'255.255.254.0' + }, + 'ipv6':{ + 'additional': [ + ], + 'address':'2001:19f0:5:28a7:5400:03ff:fe1b:4eca', + 'network':'2001:19f0:5:28a7::', + 'prefix':'64' + }, + 'mac':'56:00:03:1b:4e:ca', + 'network-type':'public' + }, + { + 'ipv4': { + 'additional': [ + ], + 'address':'10.1.112.3', + 'gateway':'', + 'netmask':'255.255.240.0' + }, + 'ipv6':{ + 'additional': [ + ], + 'network':'', + 'prefix':'' + }, + 'mac':'5a:00:03:1b:4e:ca', + 'network-type':'private', + 'network-v2-id':'fbbe2b5b-b986-4396-87f5-7246660ccb64', + 'networkid':'net5e7155329d730' + } + ], + 'public-keys': 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key\n', + 'region': { + 'regioncode': 'EWR' + }, + 'user-defined': [ + ], + 'startup-script': 'echo No configured startup script', + 'user-data': [ + ], + 'vendor-config': { + 'package_upgrade': 'true', + 'disable_root': 0, + 'packages': [], + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$SxXx...k2mJNIzZB5vMCDBlYT1' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } + } + } +} + +SSH_KEYS_1 = [ + "ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key" +] + +# Expected generated objects + +# Expected config object from generator +EXPECTED_VULTR_CONFIG_1 = { + 'package_upgrade': 'true', + 'disable_root': 0, + 'packages': [ + 'ethtool' + ], + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$S2Smuj.../VqxmIR9Urw0jPZ88i4yvB/' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } + }, + 'network': { + 'version': 1, + 'config': [ + { + 'type': 'nameserver', + 'address': ['108.61.10.10'] + }, + { + 'name': 'eth0', + 'type': 'physical', + 'mac_address': '56:00:03:15:c4:65', + 'accept-ra': 1, + 'subnets': [ + {'type': 'dhcp', 'control': 'auto'}, + {'type': 'dhcp6', 'control': 'auto'} + ] + } + ] + } +} + +EXPECTED_VULTR_CONFIG_2 = { + 'package_upgrade': 'true', + 'disable_root': 0, + 'packages': [ + 'ethtool' + ], + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$SxXx...k2mJNIzZB5vMCDBlYT1' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } + }, + 'network': { + 'version': 1, + 'config': [ + { + 'type': 'nameserver', + 'address': ['108.61.10.10'] + }, + { + 'name': 'eth0', + 'type': 'physical', + 'mac_address': '56:00:03:1b:4e:ca', + 'accept-ra': 1, + 'subnets': [ + {'type': 'dhcp', 'control': 'auto'}, + {'type': 'dhcp6', 'control': 'auto'} + ] + }, + { + 'name': 'eth1', + 'type': 'physical', + 'mac_address': '5a:00:03:1b:4e:ca', + 'accept-ra': 1, + 'subnets': [ + { + "type": "static", + "control": "auto", + "address": "10.1.112.3", + "netmask": "255.255.240.0" + } + ], + } + ] + } +} + +# Expected network config object from generator +EXPECTED_VULTR_NETWORK_1 = { + 'version': 1, + 'config': [ + { + 'type': 'nameserver', + 'address': ['108.61.10.10'] + }, + { + 'name': 'eth0', + 'type': 'physical', + 'mac_address': '56:00:03:15:c4:65', + 'accept-ra': 1, + 'subnets': [ + {'type': 'dhcp', 'control': 'auto'}, + {'type': 'dhcp6', 'control': 'auto'} + ], + } + ] +} + +EXPECTED_VULTR_NETWORK_2 = { + 'version': 1, + 'config': [ + { + 'type': 'nameserver', + 'address': ['108.61.10.10'] + }, + { + 'name': 'eth0', + 'type': 'physical', + 'mac_address': '56:00:03:1b:4e:ca', + 'accept-ra': 1, + 'subnets': [ + {'type': 'dhcp', 'control': 'auto'}, + {'type': 'dhcp6', 'control': 'auto'} + ], + }, + { + 'name': 'eth1', + 'type': 'physical', + 'mac_address': '5a:00:03:1b:4e:ca', + 'accept-ra': 1, + 'subnets': [ + { + "type": "static", + "control": "auto", + "address": "10.1.112.3", + "netmask": "255.255.240.0" + } + ], + } + ] +} + + +INTERFACE_MAP = { + '56:00:03:15:c4:65': 'eth0', + '56:00:03:1b:4e:ca': 'eth0', + '5a:00:03:1b:4e:ca': 'eth1' +} + + +class TestDataSourceVultr(CiTestCase): + def setUp(self): + super(TestDataSourceVultr, self).setUp() + self.tmp = self.tmp_dir() + + # Test the datasource itself + @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.sources.helpers.vultr.is_vultr') + @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') + def test_datasource(self, + mock_getmeta, + mock_isvultr, + mock_netmap, + mock_write_vendor_script): + mock_getmeta.return_value = VULTR_V1_2 + mock_isvultr.return_value = True + mock_netmap.return_value = INTERFACE_MAP + mock_write_vendor_script.return_value = True + + source = DataSourceVultr.DataSourceVultr( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + + # Test for failure + self.assertEqual(True, source._get_data()) + + # Test instance id + self.assertEqual("42872224", source.metadata['instanceid']) + + # Test hostname + self.assertEqual("CLOUDINIT_2", source.metadata['local-hostname']) + + # Test ssh keys + self.assertEqual(SSH_KEYS_1, source.metadata['public-keys']) + + # Test vendor data generation + orig_val = self.maxDiff + self.maxDiff = None + self.assertEqual( + EXPECTED_VULTR_CONFIG_2, + json.loads(source.vendordata_raw.replace("#cloud-config", ""))) + self.maxDiff = orig_val + + # Test network config generation + self.assertEqual(EXPECTED_VULTR_NETWORK_2, source.network_config) + + # Test overall config generation + @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') + def test_get_data_1(self, + mock_getmeta, + mock_netmap, + mock_write_vendor_script): + mock_getmeta.return_value = VULTR_V1_1 + + mock_netmap.return_value = INTERFACE_MAP + mock_write_vendor_script.return_value = True + + # Test data + self.assertEqual(EXPECTED_VULTR_CONFIG_1, vultr.generate_config({})) + + # Test overall config generation + @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') + def test_get_data_2(self, + mock_getmeta, + mock_netmap, + mock_write_vendor_script): + mock_getmeta.return_value = VULTR_V1_2 + mock_netmap.return_value = INTERFACE_MAP + mock_write_vendor_script.return_value = True + + # Test data with private networking + self.assertEqual(EXPECTED_VULTR_CONFIG_2, vultr.generate_config({})) + + # Test network config generation + @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') + def test_network_config(self, + mock_getmeta, + mock_netmap, + mock_write_vendor_script): + mock_getmeta.return_value = VULTR_V1_1 + + mock_netmap.return_value = INTERFACE_MAP + mock_write_vendor_script.return_value = True + + self.assertEqual(EXPECTED_VULTR_NETWORK_1, + vultr.generate_network_config({})) + + # Test Private Networking config generation + @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') + def test_private_network_config(self, + mock_getmeta, + mock_netmap, + mock_write_vendor_script): + mock_getmeta.return_value = VULTR_V1_2 + + mock_netmap.return_value = INTERFACE_MAP + mock_write_vendor_script.return_value = True + + self.assertEqual(EXPECTED_VULTR_NETWORK_2, + vultr.generate_network_config({})) + +# vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index 2f2486f76a3..373ce475304 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -125,7 +125,8 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud" +OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud Vultr" + DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -1350,6 +1351,20 @@ dscheck_IBMCloud() { return ${DS_NOT_FOUND} } +dscheck_Vultr() { + dmi_sys_vendor_is Vultr && return $DS_FOUND + + case " $DI_KERNEL_CMDLINE " in + *\ vultr\ *) return $DS_FOUND ;; + esac + + if [ -f "${PATH_ROOT}/etc/vultr" ]; then + return $DS_FOUND + fi + + return $DS_NOT_FOUND +} + collect_info() { read_uname_info read_virt From 584fd4093b75ee4798ca3406a72ba6000945c335 Mon Sep 17 00:00:00 2001 From: David Dymko Date: Mon, 1 Mar 2021 13:47:55 -0500 Subject: [PATCH 02/22] merge conflicts --- ChangeLog | 3 --- cloudinit/version.py | 4 ---- doc/rtd/topics/availability.rst | 3 --- doc/rtd/topics/network-config.rst | 6 ------ integration-requirements.txt | 4 ---- .../unittests/test_datasource/test_common.py | 21 ++----------------- 6 files changed, 2 insertions(+), 39 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9fd4c619df6..44b504107f4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,3 @@ -<<<<<<< HEAD 21.1 - Azure: Support for VMs without ephemeral resource disks. (#800) [Johnson Shi] (LP: #1901011) @@ -106,8 +105,6 @@ - cla: add xnox (#692) [Dimitri John Ledkov] - Collect logs from integration test runs (#675) -======= ->>>>>>> vultr-support 20.4.1 - Revert "ssh_util: handle non-default AuthorizedKeysFile config (#586)" diff --git a/cloudinit/version.py b/cloudinit/version.py index 9104dc66de5..94afd60dbc2 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,11 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -<<<<<<< HEAD __VERSION__ = "21.1" -======= -__VERSION__ = "20.4.1" ->>>>>>> vultr-support _PACKAGED_VERSION = '@@PACKAGED_VERSION@@' FEATURES = [ diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index 2dabcfdb05e..f3e13edce92 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -56,10 +56,7 @@ environments in the public cloud: - AltCloud - SmartOS - UpCloud -<<<<<<< HEAD -======= - Vultr ->>>>>>> vultr-support Additionally, cloud-init is supported on these private clouds: diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index b9bd8ebc0c6..5f7a74f83d4 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -148,13 +148,10 @@ The following Datasources optionally provide network configuration: - `UpCloud JSON metadata`_ -<<<<<<< HEAD -======= - :ref:`datasource_vultr` - `Vultr JSON metadata`_ ->>>>>>> vultr-support For more information on network configuration formats .. toctree:: @@ -269,9 +266,6 @@ Example output converting V2 to sysconfig: .. _OpenStack Metadata Service Network: https://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/metadata-service-network-info.html .. _SmartOS JSON Metadata: https://eng.joyent.com/mdata/datadict.html .. _UpCloud JSON metadata: https://developers.upcloud.com/1.3/8-servers/#metadata-service -<<<<<<< HEAD -======= .. _Vultr JSON metadata: https://www.vultr.com/metadata/ ->>>>>>> vultr-support .. vi: textwidth=78 diff --git a/integration-requirements.txt b/integration-requirements.txt index 52d6830a061..6b5964260ed 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,9 +1,5 @@ # PyPI requirements for cloud-init integration testing # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # -<<<<<<< HEAD pycloudlib @ git+https://github.com/canonical/pycloudlib.git@da8445325875674394ffd85aaefaa3d2d0e0020d -======= -pycloudlib @ git+https://github.com/canonical/pycloudlib.git@3a6c668fed769f00d83d1e6bea7d68953787cc38 ->>>>>>> vultr-support pytest diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 3c418da8699..abd7d21ab41 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -28,10 +28,7 @@ DataSourceScaleway as Scaleway, DataSourceSmartOS as SmartOS, DataSourceUpCloud as UpCloud, -<<<<<<< HEAD -======= DataSourceVultr as Vultr, ->>>>>>> vultr-support ) from cloudinit.sources import DataSourceNone as DSNone @@ -53,15 +50,8 @@ Ec2.DataSourceEc2Local, OpenStack.DataSourceOpenStackLocal, RbxCloud.DataSourceRbxCloud, -<<<<<<< HEAD - Scaleway.DataSourceScaleway -======= Scaleway.DataSourceScaleway, - UpCloud.DataSourceUpCloudLocal, -<<<<<<< HEAD -======= ->>>>>>> upstream/master ->>>>>>> vultr-support + UpCloud.DataSourceUpCloudLocal ] DEFAULT_NETWORK = [ @@ -76,15 +66,8 @@ MAAS.DataSourceMAAS, NoCloud.DataSourceNoCloudNet, OpenStack.DataSourceOpenStack, -<<<<<<< HEAD - OVF.DataSourceOVFNet -======= OVF.DataSourceOVFNet, - UpCloud.DataSourceUpCloud, -<<<<<<< HEAD -======= ->>>>>>> upstream/master ->>>>>>> vultr-support + UpCloud.DataSourceUpCloud ] From a7876570438570db169542339101599d337a55e4 Mon Sep 17 00:00:00 2001 From: David Dymko Date: Mon, 1 Mar 2021 13:50:54 -0500 Subject: [PATCH 03/22] fixing merge issues --- tests/unittests/test_datasource/test_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index abd7d21ab41..5e9c547ac63 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -51,7 +51,7 @@ OpenStack.DataSourceOpenStackLocal, RbxCloud.DataSourceRbxCloud, Scaleway.DataSourceScaleway, - UpCloud.DataSourceUpCloudLocal + UpCloud.DataSourceUpCloudLocal, ] DEFAULT_NETWORK = [ @@ -67,7 +67,7 @@ NoCloud.DataSourceNoCloudNet, OpenStack.DataSourceOpenStack, OVF.DataSourceOVFNet, - UpCloud.DataSourceUpCloud + UpCloud.DataSourceUpCloud, ] From 7431c4cd8da743527de4f5707a3843fd1cf23307 Mon Sep 17 00:00:00 2001 From: David Dymko Date: Mon, 1 Mar 2021 13:53:46 -0500 Subject: [PATCH 04/22] cleaning up readme --- README.md | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/README.md b/README.md index 6e01751f499..aa6d84ae452 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,3 @@ -# **Note:** Vultr support is work-in-progress - -This current fork and branch [vultr-support](https://github.com/vultr/cloud-init) is currently in a `beta` state and is a work in progress. The code and functionality may be changed or altered at any point during this development phase. - -Please **do not** use this in a production environment. - -The following repo can be built from source or you can use one of the -following packaged builds. - -### RHEL -Latest -* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_rhel_latest.rpm -* https://ewr1.vultrobjects.com/cloud_init_beta/rhel_latest_MD5 - -Nightly -* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_rhel_nightly.rpm -* https://ewr1.vultrobjects.com/cloud_init_beta/rhel_nightly_MD5 - -### Debian 10 (Possibly other versions) -Latest -* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_debian10_latest.deb -* https://ewr1.vultrobjects.com/cloud_init_beta/debian10_latest_MD5 - -Nightly -* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_debian10_nightly.deb -* https://ewr1.vultrobjects.com/cloud_init_beta/debian10_nightly_MD5 - -### Deb (Universal) -Latest -* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_universal_latest.deb -* https://ewr1.vultrobjects.com/cloud_init_beta/universal_latest_MD5 - -Nightly -* https://ewr1.vultrobjects.com/cloud_init_beta/cloud-init_universal_nightly.deb -* https://ewr1.vultrobjects.com/cloud_init_beta/universal_nightly_MD5 - # cloud-init [![Build Status](https://travis-ci.com/canonical/cloud-init.svg?branch=master)](https://travis-ci.com/canonical/cloud-init) [![Read the Docs](https://readthedocs.org/projects/cloudinit/badge/?version=latest&style=flat)](https://cloudinit.readthedocs.org) From e7ca19795f9f60850352bfff3ea4973dccd5372f Mon Sep 17 00:00:00 2001 From: David Dymko Date: Mon, 1 Mar 2021 14:08:53 -0500 Subject: [PATCH 05/22] added ddymko to cla signers --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index aca0ee5ef51..7ba528a4ff3 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -11,6 +11,7 @@ BirknerAlex candlerb cawamata dankenigsberg +ddymko dermotbradley dhensby eandersson From fc769fc37bb3c4f2b367433c0c8d64bfff2af201 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Thu, 4 Mar 2021 15:26:56 -0500 Subject: [PATCH 06/22] Requested Changes --- cloudinit/sources/DataSourceVultr.py | 62 ++++--- cloudinit/sources/helpers/vultr.py | 168 +++++++----------- tests/unittests/test_datasource/test_vultr.py | 73 ++++---- tools/ds-identify | 5 +- 4 files changed, 147 insertions(+), 161 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index d700f77bcff..59b616b6f89 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -6,7 +6,6 @@ # https://www.vultr.com/metadata/ import json -import re from cloudinit import log as log from cloudinit import sources @@ -14,14 +13,13 @@ import cloudinit.sources.helpers.vultr as vultr -LOGGER = log.getLogger(__name__) +LOG = log.getLogger(__name__) BUILTIN_DS_CONFIG = { 'url': 'http://169.254.169.254', 'retries': 30, 'timeout': 2, 'wait': 2 } -CONFIG = BUILTIN_DS_CONFIG.copy() class DataSourceVultr(sources.DataSource): @@ -33,36 +31,35 @@ def __init__(self, sys_cfg, distro, paths): self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, ["datasource", "Vultr"], {}), BUILTIN_DS_CONFIG]) - CONFIG['url'] = self.ds_cfg.get( + BUILTIN_DS_CONFIG['url'] = self.ds_cfg.get( 'url', BUILTIN_DS_CONFIG['url']) - CONFIG['retries'] = self.ds_cfg.get( + BUILTIN_DS_CONFIG['retries'] = self.ds_cfg.get( 'retries', BUILTIN_DS_CONFIG['retries']) - CONFIG['timeout'] = self.ds_cfg.get( + BUILTIN_DS_CONFIG['timeout'] = self.ds_cfg.get( 'timeout', BUILTIN_DS_CONFIG['timeout']) - CONFIG['wait'] = self.ds_cfg.get( + BUILTIN_DS_CONFIG['wait'] = self.ds_cfg.get( 'wait', BUILTIN_DS_CONFIG['wait']) # Initiate data and check if Vultr def _get_data(self): - LOGGER.info("Detecting if machine is a Vultr instance") + LOG.debug("Detecting if machine is a Vultr instance") if not vultr.is_vultr(): - LOGGER.info("Machine is not a Vultr instance") + LOG.debug("Machine is not a Vultr instance") return False - LOGGER.info("Machine is a Vultr instance") + LOG.debug("Machine is a Vultr instance") - config = vultr.generate_config(CONFIG) + config = vultr.generate_config(BUILTIN_DS_CONFIG) # Dump vendor config so diagnosing failures is manageable - LOGGER.info("Vultr Vendor Config:") - LOGGER.info(json.dumps(config)) + LOG.debug("Vultr Vendor Config:") + LOG.debug(json.dumps(config)) md = self.get_metadata() self.metadata_full = md self.metadata['instanceid'] = self.metadata_full['instanceid'] - self.metadata['local-hostname'] = re.sub( - r'\W+', '', self.metadata_full['hostname']) + self.metadata['local-hostname'] = self.metadata_full['hostname'] # Default hostname is "vultr" if self.metadata['local-hostname'] == "": @@ -75,17 +72,22 @@ def _get_data(self): self.vendordata_raw = "#cloud-config\n%s" % json.dumps(config) # Dump some data so diagnosing failures is manageable - LOGGER.info("SUBID: %s", self.metadata['instanceid']) - LOGGER.info("Hostname: %s", self.metadata['local-hostname']) + LOG.debug("SUBID: %s", self.metadata['instanceid']) + LOG.debug("Hostname: %s", self.metadata['local-hostname']) if self.userdata_raw is not None: - LOGGER.info("User-Data:") - LOGGER.info(self.userdata_raw) + LOG.debug("User-Data:") + LOG.debug(self.userdata_raw) return True # Get the metadata by flag def get_metadata(self): - return vultr.get_metadata(CONFIG) + return vultr.get_cached_metadata(BUILTIN_DS_CONFIG) + + # Compare subid as instance id + def check_instance_id(self, sys_cfg): + subid = vultr.get_sysinfo()['subid'] + return sources.instance_id_matches_system_uuid(subid) # Currently unsupported @property @@ -95,12 +97,12 @@ def launch_index(self): # Write the base configs every time. These are subject to change @property def network_config(self): - config = vultr.generate_network_config(CONFIG) + config = vultr.generate_network_config(BUILTIN_DS_CONFIG) config_raw = json.dumps(config) # Dump network config so diagnosing failures is manageable - LOGGER.info("Generated Network:") - LOGGER.info(config_raw) + LOG.debug("Generated Network:") + LOG.debug(config_raw) return config @@ -115,4 +117,18 @@ def network_config(self): def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + +if __name__ == "__main__": + import sys + + if not vultr.is_vultr(): + print("Machine is not a Vultr instance") + sys.exit(1) + + config = vultr.generate_config(BUILTIN_DS_CONFIG) + sysinfo = vultr.get_sysinfo() + + print(json.dumps(sysinfo, indent=1)) + print(json.dumps(config, indent=1)) + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index d2201b96e1b..e9fb6bf64c8 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -5,81 +5,45 @@ import json import os import copy -import re +import base64 from cloudinit import log as log from cloudinit import url_helper from cloudinit import dmi from cloudinit import util from cloudinit import net -from cloudinit import subp from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from functools import lru_cache -# Get logger -LOGGER = log.getLogger(__name__) +# Get LOG +LOG = log.getLogger(__name__) -# Cache -MAC_TO_NICS = None -METADATA = None -EHP = None +@lru_cache() +def get_metadata(params): + params = json.loads(params) -def bring_up_interface(connectivity_url=None): - global EHP - - # If for whatever reason this is up, bail - if EHP is not None: - return - - # Make sure its not up already - if net.has_url_connectivity(connectivity_url): - return - - # Bring up interface in local + # Bring up interface try: - EHP = EphemeralDHCPv4(net.find_fallback_nic()) - EHP.obtain_lease() + with EphemeralDHCPv4(connectivity_url=params['url']): + # Fetch the metadata + v1 = fetch_metadata(params) except (NoDHCPLeaseError) as exc: - LOGGER.error("DHCP failed, cannot continue. Exception: %s", - exc) + LOG.error("DHCP failed, cannot continue. Exception: %s", + exc) raise + v1_json = json.loads(v1) + metadata = v1_json -# Close EphermalDHCP so its not left open -def close_ephermeral(): - global EHP - - # No action if its not open - if EHP is None: - return - - EHP.clean_network() - - # Cleanup - EHP = None + # This comes through as a string but is JSON, make a dict + metadata['vendor-config'] = json.loads(metadata['vendor-config']) + return json.dumps(metadata) -# Cache the metadata for optimization -def get_metadata(params): - global METADATA - - if not METADATA: - # Bring up interface in local - bring_up_interface(params['url']) - - # Fetch the metadata - v1 = fetch_metadata(params) - - # Close EphermeralDHCP when we are done - close_ephermeral() - v1_json = json.loads(v1) - METADATA = v1_json - - # This comes through as a string but is JSON, make a dict - METADATA['vendor-config'] = json.loads(METADATA['vendor-config']) - - return METADATA +def get_cached_metadata(args): + return json.loads(get_metadata(json.dumps(args))) # Read the system information from SMBIOS @@ -92,21 +56,6 @@ def get_sysinfo(): } -# Get kernel parameters -def get_kernel_parameters(): - if not os.path.exists("/proc/cmdline"): - return "" - - file = open("/proc/cmdline") - content = file.read() - file.close() - - if "root=" not in content: - return "" - - return re.sub(r'.+root=', '', content)[1].strip() - - # Confirm is Vultr def is_vultr(): # VC2, VDC, and HFC use DMI @@ -116,7 +65,7 @@ def is_vultr(): return True # Baremetal requires a kernel parameter - if "vultr" in get_kernel_parameters(): + if "vultr" in util.get_cmdline(): return True # An extra fallback if the others fail @@ -127,22 +76,10 @@ def is_vultr(): return False -# Write vendor startup script -def write_vendor_script(fname, content): - os.makedirs("/var/lib/scripts/vendor/", exist_ok=True) - file = open("/var/lib/scripts/vendor/%s" % fname, "w") - for line in content: - file.write(line) - file.close() - command = ["chmod", "+x", "/var/lib/scripts/vendor/%s" % fname] - - try: - subp.subp(command) - except Exception as err: - LOGGER.error( - "Command: %s failed to execute. Error: %s", - " ".join(command), err) - raise +def convert_to_base64(string): + string_bytes = string.encode('ascii') + b64_bytes = base64.b64encode(string_bytes) + return b64_bytes.decode('ascii') # Read Metadata endpoint @@ -168,23 +105,25 @@ def fetch_metadata(params): return read_metadata(req) +# Wrapped for caching +@lru_cache() +def get_interface_map(): + return net.get_interfaces_by_mac() + + # Convert macs to nics def get_interface_name(mac): - global MAC_TO_NICS - - # Define it if empty - if not MAC_TO_NICS: - MAC_TO_NICS = net.get_interfaces_by_mac() + macs_to_nic = get_interface_map() - if mac not in MAC_TO_NICS: + if mac not in macs_to_nic: return None - return MAC_TO_NICS.get(mac) + return macs_to_nic.get(mac) # Generate network configs def generate_network_config(config): - md = get_metadata(config) + md = get_cached_metadata(config) network = { "version": 1, @@ -292,7 +231,8 @@ def generate_private_network_interface(md): # This configuration is to replicate how # images are deployed on Vultr before Cloud-Init def generate_config(config): - md = get_metadata(config) + LOG.debug("DS: %s", json.dumps(config)) + md = get_cached_metadata(config) # Grab the startup script script = md['startup-script'] @@ -308,8 +248,7 @@ def generate_config(config): config_template['packages'].append("ethtool") # Define vendor script - vendor_script = [] - vendor_script.append("!/bin/bash") + vendor_script = "#!/bin/bash" # Go through the interfaces for netcfg in config_template['network']['config']: @@ -321,15 +260,34 @@ def generate_config(config): # Set its multi-queue to num of cores as per RHEL Docs name = netcfg['name'] command = "ethtool -L %s combined $(nproc --all)" % name - vendor_script.append(command) - - # Write vendor script - write_vendor_script("vultr_deploy.sh", vendor_script) + vendor_script = '%s\n%s' % (vendor_script, command) + + # Add write_files if it is not present in the template + if 'write_files' not in config_template.keys(): + config_template['write_files'] = [] + + # Add vendor script to config + config_template['write_files'].append( + { + 'encoding': 'b64', + 'content': convert_to_base64(vendor_script), + 'owner': 'root:root', + 'path': '/var/lib/scripts/vendor/vultr-interface-setup.sh', + 'permissions': '0750' + } + ) # Write the startup script if script and script != "echo No configured startup script": - lines = script.splitlines() - write_vendor_script("vultr_user_startup.sh", lines) + config_template['write_files'].append( + { + 'encoding': 'b64', + 'content': convert_to_base64(script), + 'owner': 'root:root', + 'path': '/var/lib/scripts/vendor/vultr-user-startup.sh', + 'permissions': '0750' + } + ) return config_template diff --git a/tests/unittests/test_datasource/test_vultr.py b/tests/unittests/test_datasource/test_vultr.py index c947befe16d..501fa45ff20 100644 --- a/tests/unittests/test_datasource/test_vultr.py +++ b/tests/unittests/test_datasource/test_vultr.py @@ -14,6 +14,13 @@ from cloudinit.tests.helpers import mock, CiTestCase +# Large data +SCRIPT1 = 'IyEvYmluL2Jhc2gKZXRodG9vbCAtTCBldGgwIGNvbWJp' \ + 'bmVkICQobnByb2MgLS1hbGwp' +SCRIPT2 = 'IyEvYmluL2Jhc2gKZXRodG9vbCAtTCBldGgwIGNvbWJp' \ + 'bmVkICQobnByb2MgLS1hbGwpCmV0aHRvb2wgLUwgZXRo' \ + 'MSBjb21iaW5lZCAkKG5wcm9jIC0tYWxsKQ==' + # Vultr metadata test data VULTR_V1_1 = { 'bgp': { @@ -208,7 +215,16 @@ ] } ] - } + }, + 'write_files': [ + { + 'content': SCRIPT1, + 'encoding': 'b64', + 'owner': 'root:root', + 'path': '/var/lib/scripts/vendor/vultr-interface-setup.sh', + 'permissions': '0750' + } + ] } EXPECTED_VULTR_CONFIG_2 = { @@ -261,7 +277,16 @@ ], } ] - } + }, + 'write_files': [ + { + 'content': SCRIPT2, + 'encoding': 'b64', + 'owner': 'root:root', + 'path': '/var/lib/scripts/vendor/vultr-interface-setup.sh', + 'permissions': '0750' + } + ] } # Expected network config object from generator @@ -333,19 +358,16 @@ def setUp(self): self.tmp = self.tmp_dir() # Test the datasource itself - @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') @mock.patch('cloudinit.net.get_interfaces_by_mac') @mock.patch('cloudinit.sources.helpers.vultr.is_vultr') @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') def test_datasource(self, mock_getmeta, mock_isvultr, - mock_netmap, - mock_write_vendor_script): - mock_getmeta.return_value = VULTR_V1_2 + mock_netmap): + mock_getmeta.return_value = json.dumps(VULTR_V1_2) mock_isvultr.return_value = True mock_netmap.return_value = INTERFACE_MAP - mock_write_vendor_script.return_value = True source = DataSourceVultr.DataSourceVultr( settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) @@ -374,64 +396,55 @@ def test_datasource(self, self.assertEqual(EXPECTED_VULTR_NETWORK_2, source.network_config) # Test overall config generation - @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') @mock.patch('cloudinit.net.get_interfaces_by_mac') @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') def test_get_data_1(self, mock_getmeta, - mock_netmap, - mock_write_vendor_script): - mock_getmeta.return_value = VULTR_V1_1 - + mock_netmap): + mock_getmeta.return_value = json.dumps(VULTR_V1_1) mock_netmap.return_value = INTERFACE_MAP - mock_write_vendor_script.return_value = True # Test data + orig_val = self.maxDiff + self.maxDiff = None self.assertEqual(EXPECTED_VULTR_CONFIG_1, vultr.generate_config({})) + self.maxDiff = orig_val # Test overall config generation - @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') @mock.patch('cloudinit.net.get_interfaces_by_mac') @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') def test_get_data_2(self, mock_getmeta, - mock_netmap, - mock_write_vendor_script): - mock_getmeta.return_value = VULTR_V1_2 + mock_netmap): + mock_getmeta.return_value = json.dumps(VULTR_V1_2) mock_netmap.return_value = INTERFACE_MAP - mock_write_vendor_script.return_value = True # Test data with private networking + orig_val = self.maxDiff + self.maxDiff = None self.assertEqual(EXPECTED_VULTR_CONFIG_2, vultr.generate_config({})) + self.maxDiff = orig_val # Test network config generation - @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') @mock.patch('cloudinit.net.get_interfaces_by_mac') @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') def test_network_config(self, mock_getmeta, - mock_netmap, - mock_write_vendor_script): - mock_getmeta.return_value = VULTR_V1_1 - + mock_netmap): + mock_getmeta.return_value = json.dumps(VULTR_V1_1) mock_netmap.return_value = INTERFACE_MAP - mock_write_vendor_script.return_value = True self.assertEqual(EXPECTED_VULTR_NETWORK_1, vultr.generate_network_config({})) # Test Private Networking config generation - @mock.patch('cloudinit.sources.helpers.vultr.write_vendor_script') @mock.patch('cloudinit.net.get_interfaces_by_mac') @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') def test_private_network_config(self, mock_getmeta, - mock_netmap, - mock_write_vendor_script): - mock_getmeta.return_value = VULTR_V1_2 - + mock_netmap): + mock_getmeta.return_value = json.dumps(VULTR_V1_2) mock_netmap.return_value = INTERFACE_MAP - mock_write_vendor_script.return_value = True self.assertEqual(EXPECTED_VULTR_NETWORK_2, vultr.generate_network_config({})) diff --git a/tools/ds-identify b/tools/ds-identify index 373ce475304..e9017d83c2c 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -124,9 +124,8 @@ DI_DSNAME="" # this has to match the builtin list in cloud-init, it is what will # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ -CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud Vultr" - +CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ +OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud upCloud" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" From c70c644e9196f8f19fdae7a58439d4d6f4d679cf Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Thu, 4 Mar 2021 15:32:58 -0500 Subject: [PATCH 07/22] Add to signers --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 7ba528a4ff3..65f6ac45faa 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -15,6 +15,7 @@ ddymko dermotbradley dhensby eandersson +eb3095 emmanuelthome izzyleung johnsonshi From 2d04effd0b51e087b49b176a626c67381e3d3754 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Fri, 5 Mar 2021 11:39:05 -0500 Subject: [PATCH 08/22] Make new requested changes --- cloudinit/sources/DataSourceVultr.py | 7 +++++++ cloudinit/sources/helpers/vultr.py | 11 ++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index 59b616b6f89..ce58d0f21cd 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -86,6 +86,13 @@ def get_metadata(self): # Compare subid as instance id def check_instance_id(self, sys_cfg): + if not vultr.is_vultr(): + return None + + # Baremetal has no way to implement this in local + if vultr.is_baremetal(): + return None + subid = vultr.get_sysinfo()['subid'] return sources.instance_id_matches_system_uuid(subid) diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index e9fb6bf64c8..79d57924f1f 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -50,12 +50,17 @@ def get_cached_metadata(args): def get_sysinfo(): return { 'manufacturer': dmi.read_dmi_data("system-manufacturer"), - 'subid': dmi.read_dmi_data("system-serial-number"), - 'product': dmi.read_dmi_data("system-product-name"), - 'family': dmi.read_dmi_data("system-family") + 'subid': dmi.read_dmi_data("system-serial-number") } +# Assumes is Vultr is already checked +def is_baremetal(): + if get_sysinfo()['manufacturer'] != "Vultr": + return True + return False + + # Confirm is Vultr def is_vultr(): # VC2, VDC, and HFC use DMI From ca7149ffbdf6f29b9ff358da964dd7d5cd828d4d Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Fri, 5 Mar 2021 11:41:02 -0500 Subject: [PATCH 09/22] Make new requested changes --- cloudinit/sources/helpers/vultr.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index 79d57924f1f..8868932a177 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -73,11 +73,6 @@ def is_vultr(): if "vultr" in util.get_cmdline(): return True - # An extra fallback if the others fail - # This needs to be a directory - if os.path.exists("/etc/vultr") and os.path.isdir("/etc/vultr"): - return True - return False From 41476a0421e55db76da09ae74de1d8ecb857d5b0 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Fri, 5 Mar 2021 11:54:07 -0500 Subject: [PATCH 10/22] Make new requested changes --- cloudinit/sources/helpers/vultr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index 8868932a177..555f19f844b 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -3,7 +3,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import json -import os import copy import base64 From 8f0c4794fdca345e2489b97b4210c7411cd44ef7 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Mon, 8 Mar 2021 15:37:22 -0500 Subject: [PATCH 11/22] Fix upcload case --- tools/ds-identify | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ds-identify b/tools/ds-identify index e9017d83c2c..73e27c7129d 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -125,7 +125,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud upCloud" +OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" From f7c9dca0e77e27413b98057767cc2fbfadf486ad Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Mon, 8 Mar 2021 15:54:26 -0500 Subject: [PATCH 12/22] Make remaining changes --- cloudinit/sources/DataSourceVultr.py | 19 +++++++------------ cloudinit/sources/helpers/vultr.py | 3 +-- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index ce58d0f21cd..3400ea49283 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -49,7 +49,7 @@ def _get_data(self): LOG.debug("Machine is a Vultr instance") - config = vultr.generate_config(BUILTIN_DS_CONFIG) + config = vultr.generate_config(self.ds_config) # Dump vendor config so diagnosing failures is manageable LOG.debug("Vultr Vendor Config:") @@ -82,16 +82,16 @@ def _get_data(self): # Get the metadata by flag def get_metadata(self): - return vultr.get_cached_metadata(BUILTIN_DS_CONFIG) + return vultr.get_cached_metadata(self.ds_config) # Compare subid as instance id def check_instance_id(self, sys_cfg): if not vultr.is_vultr(): - return None + return False # Baremetal has no way to implement this in local if vultr.is_baremetal(): - return None + return False subid = vultr.get_sysinfo()['subid'] return sources.instance_id_matches_system_uuid(subid) @@ -104,12 +104,7 @@ def launch_index(self): # Write the base configs every time. These are subject to change @property def network_config(self): - config = vultr.generate_network_config(BUILTIN_DS_CONFIG) - config_raw = json.dumps(config) - - # Dump network config so diagnosing failures is manageable - LOG.debug("Generated Network:") - LOG.debug(config_raw) + config = vultr.generate_network_config(self.ds_config) return config @@ -135,7 +130,7 @@ def get_datasource_list(depends): config = vultr.generate_config(BUILTIN_DS_CONFIG) sysinfo = vultr.get_sysinfo() - print(json.dumps(sysinfo, indent=1)) - print(json.dumps(config, indent=1)) + print(util.json_dumps(sysinfo, indent=1)) + print(util.json_dumps(config, indent=1)) # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index 555f19f844b..abb38d37a50 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -28,8 +28,7 @@ def get_metadata(params): # Fetch the metadata v1 = fetch_metadata(params) except (NoDHCPLeaseError) as exc: - LOG.error("DHCP failed, cannot continue. Exception: %s", - exc) + LOG.error("Bailing, DHCP Exception: %s", exc) raise v1_json = json.loads(v1) From 62c4e99bc2f5d551eb2363e0b1398969af3b90f2 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Mon, 8 Mar 2021 16:06:22 -0500 Subject: [PATCH 13/22] Fix typo --- cloudinit/sources/DataSourceVultr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index 3400ea49283..5b6f4dc9dd4 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -49,7 +49,7 @@ def _get_data(self): LOG.debug("Machine is a Vultr instance") - config = vultr.generate_config(self.ds_config) + config = vultr.generate_config(self.ds_cfg) # Dump vendor config so diagnosing failures is manageable LOG.debug("Vultr Vendor Config:") @@ -82,7 +82,7 @@ def _get_data(self): # Get the metadata by flag def get_metadata(self): - return vultr.get_cached_metadata(self.ds_config) + return vultr.get_cached_metadata(self.ds_cfg) # Compare subid as instance id def check_instance_id(self, sys_cfg): @@ -104,7 +104,7 @@ def launch_index(self): # Write the base configs every time. These are subject to change @property def network_config(self): - config = vultr.generate_network_config(self.ds_config) + config = vultr.generate_network_config(self.ds_cfg) return config From 5193ad63475800e760c008392facfbcc17e4e6fc Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Mon, 8 Mar 2021 16:16:34 -0500 Subject: [PATCH 14/22] Fix typo --- cloudinit/sources/DataSourceVultr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index 5b6f4dc9dd4..300cb555a06 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -130,7 +130,7 @@ def get_datasource_list(depends): config = vultr.generate_config(BUILTIN_DS_CONFIG) sysinfo = vultr.get_sysinfo() - print(util.json_dumps(sysinfo, indent=1)) - print(util.json_dumps(config, indent=1)) + print(util.json_dumps(sysinfo)) + print(util.json_dumps(config)) # vi: ts=4 expandtab From 3805fe5411dfce233c62d2cde28c8cf4f73a99ba Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Tue, 9 Mar 2021 12:53:48 -0500 Subject: [PATCH 15/22] Wave one of requested changes --- cloudinit/sources/DataSourceVultr.py | 11 +++- cloudinit/sources/helpers/vultr.py | 90 +++++++++++++--------------- 2 files changed, 49 insertions(+), 52 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index 300cb555a06..aeee363bb0f 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -82,7 +82,10 @@ def _get_data(self): # Get the metadata by flag def get_metadata(self): - return vultr.get_cached_metadata(self.ds_cfg) + return vultr.get_cached_metadata(self.ds_cfg['url'], + self.ds_cfg['timeout'], + self.ds_cfg['retries'], + self.ds_cfg['wait']) # Compare subid as instance id def check_instance_id(self, sys_cfg): @@ -101,7 +104,6 @@ def check_instance_id(self, sys_cfg): def launch_index(self): return None - # Write the base configs every time. These are subject to change @property def network_config(self): config = vultr.generate_network_config(self.ds_cfg) @@ -127,7 +129,10 @@ def get_datasource_list(depends): print("Machine is not a Vultr instance") sys.exit(1) - config = vultr.generate_config(BUILTIN_DS_CONFIG) + config = vultr.generate_config(BUILTIN_DS_CONFIG['url'], + BUILTIN_DS_CONFIG['timeout'], + BUILTIN_DS_CONFIG['retries'], + BUILTIN_DS_CONFIG['wait']) sysinfo = vultr.get_sysinfo() print(util.json_dumps(sysinfo)) diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index abb38d37a50..a374c33e663 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -19,14 +19,12 @@ @lru_cache() -def get_metadata(params): - params = json.loads(params) - +def get_metadata(url, timeout, retries, sec_between): # Bring up interface try: with EphemeralDHCPv4(connectivity_url=params['url']): # Fetch the metadata - v1 = fetch_metadata(params) + v1 = read_metadata(url, timeout, retries, sec_between) except (NoDHCPLeaseError) as exc: LOG.error("Bailing, DHCP Exception: %s", exc) raise @@ -40,8 +38,8 @@ def get_metadata(params): return json.dumps(metadata) -def get_cached_metadata(args): - return json.loads(get_metadata(json.dumps(args))) +def get_cached_metadata(url, timeout, retries, sec_between): + return json.loads(get_metadata((url, timeout, retries, sec_between)) # Read the system information from SMBIOS @@ -74,35 +72,22 @@ def is_vultr(): return False -def convert_to_base64(string): - string_bytes = string.encode('ascii') - b64_bytes = base64.b64encode(string_bytes) - return b64_bytes.decode('ascii') - - # Read Metadata endpoint -def read_metadata(params): - response = url_helper.readurl(params['url'], - timeout=params['timeout'], - retries=params['retries'], +def read_metadata(url, timeout, retries, sec_between): + url = "%s/v1.json" % url + response = url_helper.readurl(url, + timeout=timeout, + retries=retries, headers={'Metadata-Token': 'vultr'}, - sec_between=params['wait']) + sec_between=sec_between) if not response.ok(): raise RuntimeError("Failed to connect to %s: Code: %s" % - params['url'], response.code) + url, response.code) return response.contents.decode() -# Get Metadata by flag -def fetch_metadata(params): - req = dict(params) - req['url'] = "%s/v1.json" % params['url'] - - return read_metadata(req) - - # Wrapped for caching @lru_cache() def get_interface_map(): @@ -121,7 +106,10 @@ def get_interface_name(mac): # Generate network configs def generate_network_config(config): - md = get_cached_metadata(config) + md = get_cached_metadata(config['url'], + config['timeout'], + config['retries'], + config['wait']) network = { "version": 1, @@ -135,29 +123,31 @@ def generate_network_config(config): ] } + intf = md['interfaces'] + # Prepare interface 0, public if len(md['interfaces']) > 0: - network['config'].append(generate_public_network_interface(md)) + network['config'].append(generate_public_network_interface(intf)) # Prepare interface 1, private if len(md['interfaces']) > 1: - network['config'].append(generate_private_network_interface(md)) + network['config'].append(generate_private_network_interface(intf)) return network # Input Metadata and generate public network config part -def generate_public_network_interface(md): - interface_name = get_interface_name(md['interfaces'][0]['mac']) +def generate_public_network_interface(interfaces): + interface_name = get_interface_name(interfaces[0]['mac']) if not interface_name: raise RuntimeError( "Interface: %s could not be found on the system" % - md['interfaces'][0]['mac']) + interfaces[0]['mac']) netcfg = { "name": interface_name, "type": "physical", - "mac_address": md['interfaces'][0]['mac'], + "mac_address": interfaces[0]['mac'], "accept-ra": 1, "subnets": [ { @@ -172,9 +162,9 @@ def generate_public_network_interface(md): } # Check for additional IP's - additional_count = len(md['interfaces'][0]['ipv4']['additional']) - if "ipv4" in md['interfaces'][0] and additional_count > 0: - for additional in md['interfaces'][0]['ipv4']['additional']: + additional_count = len(interfaces[0]['ipv4']['additional']) + if "ipv4" in interfaces[0] and additional_count > 0: + for additional in interfaces[0]['ipv4']['additional']: add = { "type": "static", "control": "auto", @@ -184,9 +174,9 @@ def generate_public_network_interface(md): netcfg['subnets'].append(add) # Check for additional IPv6's - additional_count = len(md['interfaces'][0]['ipv6']['additional']) - if "ipv6" in md['interfaces'][0] and additional_count > 0: - for additional in md['interfaces'][0]['ipv6']['additional']: + additional_count = leninterfaces[0]['ipv6']['additional']) + if "ipv6" in interfaces[0] and additional_count > 0: + for additional in interfaces[0]['ipv6']['additional']: add = { "type": "static6", "control": "auto", @@ -200,24 +190,24 @@ def generate_public_network_interface(md): # Input Metadata and generate private network config part -def generate_private_network_interface(md): - interface_name = get_interface_name(md['interfaces'][1]['mac']) +def generate_private_network_interface(interfaces): + interface_name = get_interface_name(interfaces[1]['mac']) if not interface_name: raise RuntimeError( "Interface: %s could not be found on the system" % - md['interfaces'][1]['mac']) + interfaces[1]['mac']) netcfg = { "name": interface_name, "type": "physical", - "mac_address": md['interfaces'][1]['mac'], + "mac_address": interfaces[1]['mac'], "accept-ra": 1, "subnets": [ { "type": "static", "control": "auto", - "address": md['interfaces'][1]['ipv4']['address'], - "netmask": md['interfaces'][1]['ipv4']['netmask'] + "address": interfaces[1]['ipv4']['address'], + "netmask": interfaces[1]['ipv4']['netmask'] } ] } @@ -229,8 +219,10 @@ def generate_private_network_interface(md): # This configuration is to replicate how # images are deployed on Vultr before Cloud-Init def generate_config(config): - LOG.debug("DS: %s", json.dumps(config)) - md = get_cached_metadata(config) + md = get_cached_metadata(config['url'], + config['timeout'], + config['retries'], + config['wait']) # Grab the startup script script = md['startup-script'] @@ -268,7 +260,7 @@ def generate_config(config): config_template['write_files'].append( { 'encoding': 'b64', - 'content': convert_to_base64(vendor_script), + 'content': util.b64d(vendor_script), 'owner': 'root:root', 'path': '/var/lib/scripts/vendor/vultr-interface-setup.sh', 'permissions': '0750' @@ -280,7 +272,7 @@ def generate_config(config): config_template['write_files'].append( { 'encoding': 'b64', - 'content': convert_to_base64(script), + 'content': util.b64d(script), 'owner': 'root:root', 'path': '/var/lib/scripts/vendor/vultr-user-startup.sh', 'permissions': '0750' From 1596185a303fe7567adeea02166a2a0cf073b758 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Wed, 10 Mar 2021 01:05:46 -0500 Subject: [PATCH 16/22] Wave two of requested changes --- cloudinit/sources/DataSourceVultr.py | 29 ++++--- cloudinit/sources/helpers/vultr.py | 75 ++++++------------ tests/unittests/test_datasource/test_vultr.py | 76 +++++++------------ 3 files changed, 70 insertions(+), 110 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index aeee363bb0f..4d4d0e473b8 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -49,14 +49,20 @@ def _get_data(self): LOG.debug("Machine is a Vultr instance") - config = vultr.generate_config(self.ds_cfg) + # Fetch metadata + md = self.get_metadata() + + config = vultr.generate_config(md) + + # This requires info generated in the vendor config + script = md['startup-script'] + user_scripts = [] + user_scripts = vultr.generate_user_scripts(script, config) # Dump vendor config so diagnosing failures is manageable LOG.debug("Vultr Vendor Config:") LOG.debug(json.dumps(config)) - md = self.get_metadata() - self.metadata_full = md self.metadata['instanceid'] = self.metadata_full['instanceid'] self.metadata['local-hostname'] = self.metadata_full['hostname'] @@ -69,7 +75,11 @@ def _get_data(self): self.userdata_raw = md["user-data"] if self.userdata_raw == "": self.userdata_raw = None - self.vendordata_raw = "#cloud-config\n%s" % json.dumps(config) + + self.vendordata_raw = [] + for uscript in user_scripts: + self.vendordata_raw.append(uscript) + self.vendordata_raw.append("#cloud-config\n%s" % json.dumps(config)) # Dump some data so diagnosing failures is manageable LOG.debug("SUBID: %s", self.metadata['instanceid']) @@ -82,10 +92,10 @@ def _get_data(self): # Get the metadata by flag def get_metadata(self): - return vultr.get_cached_metadata(self.ds_cfg['url'], - self.ds_cfg['timeout'], - self.ds_cfg['retries'], - self.ds_cfg['wait']) + return vultr.get_metadata(self.ds_cfg['url'], + self.ds_cfg['timeout'], + self.ds_cfg['retries'], + self.ds_cfg['wait']) # Compare subid as instance id def check_instance_id(self, sys_cfg): @@ -106,7 +116,8 @@ def launch_index(self): @property def network_config(self): - config = vultr.generate_network_config(self.ds_cfg) + md = self.get_metadata() + config = vultr.generate_network_config(md) return config diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index a374c33e663..49d4ee69828 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -4,7 +4,6 @@ import json import copy -import base64 from cloudinit import log as log from cloudinit import url_helper @@ -22,7 +21,7 @@ def get_metadata(url, timeout, retries, sec_between): # Bring up interface try: - with EphemeralDHCPv4(connectivity_url=params['url']): + with EphemeralDHCPv4(connectivity_url=url): # Fetch the metadata v1 = read_metadata(url, timeout, retries, sec_between) except (NoDHCPLeaseError) as exc: @@ -35,11 +34,7 @@ def get_metadata(url, timeout, retries, sec_between): # This comes through as a string but is JSON, make a dict metadata['vendor-config'] = json.loads(metadata['vendor-config']) - return json.dumps(metadata) - - -def get_cached_metadata(url, timeout, retries, sec_between): - return json.loads(get_metadata((url, timeout, retries, sec_between)) + return metadata # Read the system information from SMBIOS @@ -105,12 +100,7 @@ def get_interface_name(mac): # Generate network configs -def generate_network_config(config): - md = get_cached_metadata(config['url'], - config['timeout'], - config['retries'], - config['wait']) - +def generate_network_config(md): network = { "version": 1, "config": [ @@ -126,11 +116,11 @@ def generate_network_config(config): intf = md['interfaces'] # Prepare interface 0, public - if len(md['interfaces']) > 0: + if len(intf) > 0: network['config'].append(generate_public_network_interface(intf)) # Prepare interface 1, private - if len(md['interfaces']) > 1: + if len(intf) > 1: network['config'].append(generate_private_network_interface(intf)) return network @@ -174,7 +164,7 @@ def generate_public_network_interface(interfaces): netcfg['subnets'].append(add) # Check for additional IPv6's - additional_count = leninterfaces[0]['ipv6']['additional']) + additional_count = len(interfaces[0]['ipv6']['additional']) if "ipv6" in interfaces[0] and additional_count > 0: for additional in interfaces[0]['ipv6']['additional']: add = { @@ -218,30 +208,27 @@ def generate_private_network_interface(interfaces): # Generate the vendor config # This configuration is to replicate how # images are deployed on Vultr before Cloud-Init -def generate_config(config): - md = get_cached_metadata(config['url'], - config['timeout'], - config['retries'], - config['wait']) - - # Grab the startup script - script = md['startup-script'] - +def generate_config(md): # Create vendor config config_template = copy.deepcopy(md['vendor-config']) # Add generated network parts - config_template['network'] = generate_network_config(config) + config_template['network'] = generate_network_config(md) # Linux specific packages if util.is_Linux(): config_template['packages'].append("ethtool") + return config_template + + +# This is for the vendor and startup scripts +def generate_user_scripts(script, vendor_config): # Define vendor script vendor_script = "#!/bin/bash" # Go through the interfaces - for netcfg in config_template['network']['config']: + for netcfg in vendor_config['network']['config']: # If the interface has a mac and is physical if "mac_address" in netcfg and netcfg['type'] == "physical": # Enable multi-queue on linux @@ -252,34 +239,16 @@ def generate_config(config): command = "ethtool -L %s combined $(nproc --all)" % name vendor_script = '%s\n%s' % (vendor_script, command) - # Add write_files if it is not present in the template - if 'write_files' not in config_template.keys(): - config_template['write_files'] = [] - - # Add vendor script to config - config_template['write_files'].append( - { - 'encoding': 'b64', - 'content': util.b64d(vendor_script), - 'owner': 'root:root', - 'path': '/var/lib/scripts/vendor/vultr-interface-setup.sh', - 'permissions': '0750' - } - ) - - # Write the startup script + vendor_script = '%s\n' % vendor_script + + # Vendor script and start the array + user_scripts = [vendor_script] + + # Startup script if script and script != "echo No configured startup script": - config_template['write_files'].append( - { - 'encoding': 'b64', - 'content': util.b64d(script), - 'owner': 'root:root', - 'path': '/var/lib/scripts/vendor/vultr-user-startup.sh', - 'permissions': '0750' - } - ) + user_scripts.append("%s\n" % script) - return config_template + return user_scripts # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_vultr.py b/tests/unittests/test_datasource/test_vultr.py index 501fa45ff20..1c4ad81dae6 100644 --- a/tests/unittests/test_datasource/test_vultr.py +++ b/tests/unittests/test_datasource/test_vultr.py @@ -9,6 +9,7 @@ from cloudinit import helpers from cloudinit import settings +from cloudinit import util from cloudinit.sources import DataSourceVultr from cloudinit.sources.helpers import vultr @@ -16,10 +17,10 @@ # Large data SCRIPT1 = 'IyEvYmluL2Jhc2gKZXRodG9vbCAtTCBldGgwIGNvbWJp' \ - 'bmVkICQobnByb2MgLS1hbGwp' + 'bmVkICQobnByb2MgLS1hbGwpCg==' SCRIPT2 = 'IyEvYmluL2Jhc2gKZXRodG9vbCAtTCBldGgwIGNvbWJp' \ 'bmVkICQobnByb2MgLS1hbGwpCmV0aHRvb2wgLUwgZXRo' \ - 'MSBjb21iaW5lZCAkKG5wcm9jIC0tYWxsKQ==' + 'MSBjb21iaW5lZCAkKG5wcm9jIC0tYWxsKQo=' # Vultr metadata test data VULTR_V1_1 = { @@ -215,16 +216,7 @@ ] } ] - }, - 'write_files': [ - { - 'content': SCRIPT1, - 'encoding': 'b64', - 'owner': 'root:root', - 'path': '/var/lib/scripts/vendor/vultr-interface-setup.sh', - 'permissions': '0750' - } - ] + } } EXPECTED_VULTR_CONFIG_2 = { @@ -277,16 +269,7 @@ ], } ] - }, - 'write_files': [ - { - 'content': SCRIPT2, - 'encoding': 'b64', - 'owner': 'root:root', - 'path': '/var/lib/scripts/vendor/vultr-interface-setup.sh', - 'permissions': '0750' - } - ] + } } # Expected network config object from generator @@ -365,7 +348,7 @@ def test_datasource(self, mock_getmeta, mock_isvultr, mock_netmap): - mock_getmeta.return_value = json.dumps(VULTR_V1_2) + mock_getmeta.return_value = VULTR_V1_2 mock_isvultr.return_value = True mock_netmap.return_value = INTERFACE_MAP @@ -387,9 +370,20 @@ def test_datasource(self, # Test vendor data generation orig_val = self.maxDiff self.maxDiff = None + + expected_script = util.b64d(SCRIPT2) + vendordata = source.vendordata_raw + + # Test vendor script + self.assertEqual( + expected_script, + vendordata[0]) + + # Test vendor config self.assertEqual( EXPECTED_VULTR_CONFIG_2, - json.loads(source.vendordata_raw.replace("#cloud-config", ""))) + json.loads(vendordata[1].replace("#cloud-config", ""))) + self.maxDiff = orig_val # Test network config generation @@ -397,56 +391,42 @@ def test_datasource(self, # Test overall config generation @mock.patch('cloudinit.net.get_interfaces_by_mac') - @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') - def test_get_data_1(self, - mock_getmeta, - mock_netmap): - mock_getmeta.return_value = json.dumps(VULTR_V1_1) + def test_get_data_1(self, mock_netmap): mock_netmap.return_value = INTERFACE_MAP # Test data orig_val = self.maxDiff self.maxDiff = None - self.assertEqual(EXPECTED_VULTR_CONFIG_1, vultr.generate_config({})) + self.assertEqual(EXPECTED_VULTR_CONFIG_1, + vultr.generate_config(VULTR_V1_1)) self.maxDiff = orig_val # Test overall config generation @mock.patch('cloudinit.net.get_interfaces_by_mac') - @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') - def test_get_data_2(self, - mock_getmeta, - mock_netmap): - mock_getmeta.return_value = json.dumps(VULTR_V1_2) + def test_get_data_2(self, mock_netmap): mock_netmap.return_value = INTERFACE_MAP # Test data with private networking orig_val = self.maxDiff self.maxDiff = None - self.assertEqual(EXPECTED_VULTR_CONFIG_2, vultr.generate_config({})) + self.assertEqual(EXPECTED_VULTR_CONFIG_2, + vultr.generate_config(VULTR_V1_2)) self.maxDiff = orig_val # Test network config generation @mock.patch('cloudinit.net.get_interfaces_by_mac') - @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') - def test_network_config(self, - mock_getmeta, - mock_netmap): - mock_getmeta.return_value = json.dumps(VULTR_V1_1) + def test_network_config(self, mock_netmap): mock_netmap.return_value = INTERFACE_MAP self.assertEqual(EXPECTED_VULTR_NETWORK_1, - vultr.generate_network_config({})) + vultr.generate_network_config(VULTR_V1_1)) # Test Private Networking config generation @mock.patch('cloudinit.net.get_interfaces_by_mac') - @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') - def test_private_network_config(self, - mock_getmeta, - mock_netmap): - mock_getmeta.return_value = json.dumps(VULTR_V1_2) + def test_private_network_config(self, mock_netmap): mock_netmap.return_value = INTERFACE_MAP self.assertEqual(EXPECTED_VULTR_NETWORK_2, - vultr.generate_network_config({})) + vultr.generate_network_config(VULTR_V1_2)) # vi: ts=4 expandtab From decfddc267c3269a6f3706348d4d352631e1e31f Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Wed, 10 Mar 2021 03:18:49 -0500 Subject: [PATCH 17/22] Fix debug calls --- cloudinit/sources/DataSourceVultr.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index 4d4d0e473b8..f818fbafd38 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -140,10 +140,8 @@ def get_datasource_list(depends): print("Machine is not a Vultr instance") sys.exit(1) - config = vultr.generate_config(BUILTIN_DS_CONFIG['url'], - BUILTIN_DS_CONFIG['timeout'], - BUILTIN_DS_CONFIG['retries'], - BUILTIN_DS_CONFIG['wait']) + md = self.get_metadata() + config = vultr.generate_config(md) sysinfo = vultr.get_sysinfo() print(util.json_dumps(sysinfo)) From 889a9324967405aba041de553f2975f3c06ef911 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Wed, 10 Mar 2021 03:34:59 -0500 Subject: [PATCH 18/22] Fix debug calls --- cloudinit/sources/DataSourceVultr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index f818fbafd38..b5905c34aeb 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -140,7 +140,10 @@ def get_datasource_list(depends): print("Machine is not a Vultr instance") sys.exit(1) - md = self.get_metadata() + md = vultr.get_metadata(BUILTIN_DS_CONFIG['url'], + BUILTIN_DS_CONFIG['timeout'], + BUILTIN_DS_CONFIG['retries'], + BUILTIN_DS_CONFIG['wait']) config = vultr.generate_config(md) sysinfo = vultr.get_sysinfo() From 0b6093c8f46ceac2bdc362393914afb92d8facea Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Wed, 31 Mar 2021 15:04:28 -0400 Subject: [PATCH 19/22] Adjust ethtool usage and v1 changes --- cloudinit/sources/DataSourceVultr.py | 45 ++++---- cloudinit/sources/helpers/vultr.py | 66 ++++++------ tests/unittests/test_datasource/test_vultr.py | 101 +++++++++--------- 3 files changed, 109 insertions(+), 103 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index b5905c34aeb..2e2058e2a59 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -52,26 +52,40 @@ def _get_data(self): # Fetch metadata md = self.get_metadata() - config = vultr.generate_config(md) + self.metadata_full = md + self.metadata['instanceid'] = md['instanceid'] + self.metadata['local-hostname'] = md['hostname'] + self.metadata['public-keys'] = md["public-keys"] + self.userdata_raw = md["user-data"] - # This requires info generated in the vendor config - script = md['startup-script'] - user_scripts = [] - user_scripts = vultr.generate_user_scripts(script, config) + # Generate config and process data + config = self.get_datasource_data(md) - # Dump vendor config so diagnosing failures is manageable + # Dump some data so diagnosing failures is manageable LOG.debug("Vultr Vendor Config:") LOG.debug(json.dumps(config)) + LOG.debug("SUBID: %s", self.metadata['instanceid']) + LOG.debug("Hostname: %s", self.metadata['local-hostname']) + if self.userdata_raw is not None: + LOG.debug("User-Data:") + LOG.debug(self.userdata_raw) - self.metadata_full = md - self.metadata['instanceid'] = self.metadata_full['instanceid'] - self.metadata['local-hostname'] = self.metadata_full['hostname'] + return True + + # Process metadata + def get_datasource_data(self, md): + # Grab config + config = vultr.generate_config(md) + + # This requires info generated in the vendor config + user_scripts = [] + user_scripts = vultr.generate_user_scripts(md, config) # Default hostname is "vultr" if self.metadata['local-hostname'] == "": self.metadata['local-hostname'] = "vultr" - self.metadata['public-keys'] = md["public-keys"].splitlines() + self.metadata['public-keys'] = md["public-keys"] self.userdata_raw = md["user-data"] if self.userdata_raw == "": self.userdata_raw = None @@ -81,14 +95,7 @@ def _get_data(self): self.vendordata_raw.append(uscript) self.vendordata_raw.append("#cloud-config\n%s" % json.dumps(config)) - # Dump some data so diagnosing failures is manageable - LOG.debug("SUBID: %s", self.metadata['instanceid']) - LOG.debug("Hostname: %s", self.metadata['local-hostname']) - if self.userdata_raw is not None: - LOG.debug("User-Data:") - LOG.debug(self.userdata_raw) - - return True + return config # Get the metadata by flag def get_metadata(self): @@ -116,7 +123,7 @@ def launch_index(self): @property def network_config(self): - md = self.get_metadata() + md = self.metadata_full['interfaces'] config = vultr.generate_network_config(md) return config diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index 49d4ee69828..03717269b2a 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -32,7 +32,8 @@ def get_metadata(url, timeout, retries, sec_between): metadata = v1_json # This comes through as a string but is JSON, make a dict - metadata['vendor-config'] = json.loads(metadata['vendor-config']) + raw = metadata['vendor-data']['config'] + metadata['vendor-data']['config'] = json.loads(raw) return metadata @@ -61,7 +62,7 @@ def is_vultr(): return True # Baremetal requires a kernel parameter - if "vultr" in util.get_cmdline(): + if "vultr" in util.get_cmdline().split(): return True return False @@ -113,15 +114,13 @@ def generate_network_config(md): ] } - intf = md['interfaces'] - # Prepare interface 0, public - if len(intf) > 0: - network['config'].append(generate_public_network_interface(intf)) + if len(md) > 0: + network['config'].append(generate_public_network_interface(md)) # Prepare interface 1, private - if len(intf) > 1: - network['config'].append(generate_private_network_interface(intf)) + if len(md) > 1: + network['config'].append(generate_private_network_interface(md)) return network @@ -210,43 +209,48 @@ def generate_private_network_interface(interfaces): # images are deployed on Vultr before Cloud-Init def generate_config(md): # Create vendor config - config_template = copy.deepcopy(md['vendor-config']) + config_template = copy.deepcopy(md['vendor-data']['config']) # Add generated network parts - config_template['network'] = generate_network_config(md) - - # Linux specific packages - if util.is_Linux(): - config_template['packages'].append("ethtool") + config_template['network'] = generate_network_config(md['interfaces']) return config_template # This is for the vendor and startup scripts -def generate_user_scripts(script, vendor_config): - # Define vendor script - vendor_script = "#!/bin/bash" - - # Go through the interfaces - for netcfg in vendor_config['network']['config']: - # If the interface has a mac and is physical - if "mac_address" in netcfg and netcfg['type'] == "physical": - # Enable multi-queue on linux - # This is executed as a vendor script - if util.is_Linux(): +def generate_user_scripts(md, vendor_config): + user_scripts = [] + + # Raid 1 script + if md['vendor-data']['raid1-script']: + user_scripts.append(md['vendor-data']['raid1-script']) + + # Enable multi-queue on linux + if util.is_Linux() and md['vendor-data']['ethtool-script']: + ethtool_script = md['vendor-data']['ethtool-script'] + + # Tool location + tool = "/opt/vultr/ethtool" + + # Go through the interfaces + for netcfg in vendor_config['network']['config']: + # If the interface has a mac and is physical + if "mac_address" in netcfg and netcfg['type'] == "physical": # Set its multi-queue to num of cores as per RHEL Docs name = netcfg['name'] - command = "ethtool -L %s combined $(nproc --all)" % name - vendor_script = '%s\n%s' % (vendor_script, command) + command = "%s -L %s combined $(nproc --all)" % (tool, name) + ethtool_script = '%s\n%s' % (ethtool_script, command) - vendor_script = '%s\n' % vendor_script + user_scripts.append(ethtool_script) - # Vendor script and start the array - user_scripts = [vendor_script] + # This is for vendor scripts + if md['vendor-data']['vendor-script']: + user_scripts.append(md['vendor-data']['vendor-script']) # Startup script + script = md['startup-script'] if script and script != "echo No configured startup script": - user_scripts.append("%s\n" % script) + user_scripts.append(script) return user_scripts diff --git a/tests/unittests/test_datasource/test_vultr.py b/tests/unittests/test_datasource/test_vultr.py index 1c4ad81dae6..cec0d85cced 100644 --- a/tests/unittests/test_datasource/test_vultr.py +++ b/tests/unittests/test_datasource/test_vultr.py @@ -9,19 +9,11 @@ from cloudinit import helpers from cloudinit import settings -from cloudinit import util from cloudinit.sources import DataSourceVultr from cloudinit.sources.helpers import vultr from cloudinit.tests.helpers import mock, CiTestCase -# Large data -SCRIPT1 = 'IyEvYmluL2Jhc2gKZXRodG9vbCAtTCBldGgwIGNvbWJp' \ - 'bmVkICQobnByb2MgLS1hbGwpCg==' -SCRIPT2 = 'IyEvYmluL2Jhc2gKZXRodG9vbCAtTCBldGgwIGNvbWJp' \ - 'bmVkICQobnByb2MgLS1hbGwpCmV0aHRvb2wgLUwgZXRo' \ - 'MSBjb21iaW5lZCAkKG5wcm9jIC0tYWxsKQo=' - # Vultr metadata test data VULTR_V1_1 = { 'bgp': { @@ -60,29 +52,35 @@ 'network-type': 'public' } ], - 'public-keys': 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key\n', + 'public-keys': [ + 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key' + ], 'region': { 'regioncode': 'EWR' }, 'user-defined': [ ], 'startup-script': 'echo No configured startup script', + 'raid1-script': '', 'user-data': [ ], - 'vendor-config': { - 'package_upgrade': 'true', - 'disable_root': 0, - 'packages': [], - 'ssh_pwauth': 1, - 'chpasswd': { - 'expire': False, - 'list': [ - 'root:$6$S2Smuj.../VqxmIR9Urw0jPZ88i4yvB/' - ] - }, - 'system_info': { - 'default_user': { - 'name': 'root' + 'vendor-data': { + 'vendor-script': '', + 'ethtool-script': '', + 'config': { + 'package_upgrade': 'true', + 'disable_root': 0, + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$S2Smuj.../VqxmIR9Urw0jPZ88i4yvB/' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } } } } @@ -145,7 +143,9 @@ 'networkid':'net5e7155329d730' } ], - 'public-keys': 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key\n', + 'public-keys': [ + 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key' + ], 'region': { 'regioncode': 'EWR' }, @@ -154,20 +154,25 @@ 'startup-script': 'echo No configured startup script', 'user-data': [ ], - 'vendor-config': { - 'package_upgrade': 'true', - 'disable_root': 0, - 'packages': [], - 'ssh_pwauth': 1, - 'chpasswd': { - 'expire': False, - 'list': [ - 'root:$6$SxXx...k2mJNIzZB5vMCDBlYT1' - ] - }, - 'system_info': { - 'default_user': { - 'name': 'root' + + 'vendor-data': { + 'vendor-script': '', + 'ethtool-script': '', + 'raid1-script': '', + 'config': { + 'package_upgrade': 'true', + 'disable_root': 0, + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$SxXx...k2mJNIzZB5vMCDBlYT1' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } } } } @@ -183,9 +188,6 @@ EXPECTED_VULTR_CONFIG_1 = { 'package_upgrade': 'true', 'disable_root': 0, - 'packages': [ - 'ethtool' - ], 'ssh_pwauth': 1, 'chpasswd': { 'expire': False, @@ -222,9 +224,6 @@ EXPECTED_VULTR_CONFIG_2 = { 'package_upgrade': 'true', 'disable_root': 0, - 'packages': [ - 'ethtool' - ], 'ssh_pwauth': 1, 'chpasswd': { 'expire': False, @@ -371,18 +370,12 @@ def test_datasource(self, orig_val = self.maxDiff self.maxDiff = None - expected_script = util.b64d(SCRIPT2) vendordata = source.vendordata_raw - # Test vendor script - self.assertEqual( - expected_script, - vendordata[0]) - # Test vendor config self.assertEqual( EXPECTED_VULTR_CONFIG_2, - json.loads(vendordata[1].replace("#cloud-config", ""))) + json.loads(vendordata[0].replace("#cloud-config", ""))) self.maxDiff = orig_val @@ -417,16 +410,18 @@ def test_get_data_2(self, mock_netmap): @mock.patch('cloudinit.net.get_interfaces_by_mac') def test_network_config(self, mock_netmap): mock_netmap.return_value = INTERFACE_MAP + interf = VULTR_V1_1['interfaces'] self.assertEqual(EXPECTED_VULTR_NETWORK_1, - vultr.generate_network_config(VULTR_V1_1)) + vultr.generate_network_config(interf)) # Test Private Networking config generation @mock.patch('cloudinit.net.get_interfaces_by_mac') def test_private_network_config(self, mock_netmap): mock_netmap.return_value = INTERFACE_MAP + interf = VULTR_V1_2['interfaces'] self.assertEqual(EXPECTED_VULTR_NETWORK_2, - vultr.generate_network_config(VULTR_V1_2)) + vultr.generate_network_config(interf)) # vi: ts=4 expandtab From 6f1698ef5daa62a712c62b8874bccb7e4d7fd118 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Fri, 2 Apr 2021 12:03:10 -0400 Subject: [PATCH 20/22] Cleanup Cleanup Cleanup Cleanup Cleanup Cleanup Cleanup --- cloudinit/sources/DataSourceVultr.py | 37 +++--- cloudinit/sources/helpers/vultr.py | 22 +--- tests/unittests/test_datasource/test_vultr.py | 108 ++---------------- 3 files changed, 30 insertions(+), 137 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index 2e2058e2a59..7115ee6ad80 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -5,8 +5,6 @@ # Vultr Metadata API: # https://www.vultr.com/metadata/ -import json - from cloudinit import log as log from cloudinit import sources from cloudinit import util @@ -59,11 +57,11 @@ def _get_data(self): self.userdata_raw = md["user-data"] # Generate config and process data - config = self.get_datasource_data(md) + self.get_datasource_data(md) # Dump some data so diagnosing failures is manageable LOG.debug("Vultr Vendor Config:") - LOG.debug(json.dumps(config)) + LOG.debug(md['vendor-data']['config']) LOG.debug("SUBID: %s", self.metadata['instanceid']) LOG.debug("Hostname: %s", self.metadata['local-hostname']) if self.userdata_raw is not None: @@ -75,27 +73,27 @@ def _get_data(self): # Process metadata def get_datasource_data(self, md): # Grab config - config = vultr.generate_config(md) + config = md['vendor-data']['config'] + + # Generate network config + self.netcfg = vultr.generate_network_config(md['interfaces']) # This requires info generated in the vendor config - user_scripts = [] - user_scripts = vultr.generate_user_scripts(md, config) + user_scripts = vultr.generate_user_scripts(md, self.netcfg['config']) - # Default hostname is "vultr" + # Default hostname is "guest" for whitelabel if self.metadata['local-hostname'] == "": - self.metadata['local-hostname'] = "vultr" + self.metadata['local-hostname'] = "guest" - self.metadata['public-keys'] = md["public-keys"] self.userdata_raw = md["user-data"] if self.userdata_raw == "": self.userdata_raw = None + # Assemble vendor-data + # This adds provided scripts and the config self.vendordata_raw = [] - for uscript in user_scripts: - self.vendordata_raw.append(uscript) - self.vendordata_raw.append("#cloud-config\n%s" % json.dumps(config)) - - return config + self.vendordata_raw.extend(user_scripts) + self.vendordata_raw.append("#cloud-config\n%s" % config) # Get the metadata by flag def get_metadata(self): @@ -123,10 +121,7 @@ def launch_index(self): @property def network_config(self): - md = self.metadata_full['interfaces'] - config = vultr.generate_network_config(md) - - return config + return self.netcfg # Used to match classes to dependencies @@ -151,10 +146,10 @@ def get_datasource_list(depends): BUILTIN_DS_CONFIG['timeout'], BUILTIN_DS_CONFIG['retries'], BUILTIN_DS_CONFIG['wait']) - config = vultr.generate_config(md) + config = md['vendor-data']['config'] sysinfo = vultr.get_sysinfo() print(util.json_dumps(sysinfo)) - print(util.json_dumps(config)) + print(config) # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index 03717269b2a..ab0251d20a2 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -3,7 +3,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import json -import copy from cloudinit import log as log from cloudinit import url_helper @@ -31,10 +30,6 @@ def get_metadata(url, timeout, retries, sec_between): v1_json = json.loads(v1) metadata = v1_json - # This comes through as a string but is JSON, make a dict - raw = metadata['vendor-data']['config'] - metadata['vendor-data']['config'] = json.loads(raw) - return metadata @@ -204,21 +199,8 @@ def generate_private_network_interface(interfaces): return netcfg -# Generate the vendor config -# This configuration is to replicate how -# images are deployed on Vultr before Cloud-Init -def generate_config(md): - # Create vendor config - config_template = copy.deepcopy(md['vendor-data']['config']) - - # Add generated network parts - config_template['network'] = generate_network_config(md['interfaces']) - - return config_template - - # This is for the vendor and startup scripts -def generate_user_scripts(md, vendor_config): +def generate_user_scripts(md, network_config): user_scripts = [] # Raid 1 script @@ -233,7 +215,7 @@ def generate_user_scripts(md, vendor_config): tool = "/opt/vultr/ethtool" # Go through the interfaces - for netcfg in vendor_config['network']['config']: + for netcfg in network_config: # If the interface has a mac and is physical if "mac_address" in netcfg and netcfg['type'] == "physical": # Set its multi-queue to num of cores as per RHEL Docs diff --git a/tests/unittests/test_datasource/test_vultr.py b/tests/unittests/test_datasource/test_vultr.py index cec0d85cced..bbea2aa3818 100644 --- a/tests/unittests/test_datasource/test_vultr.py +++ b/tests/unittests/test_datasource/test_vultr.py @@ -184,44 +184,8 @@ # Expected generated objects -# Expected config object from generator -EXPECTED_VULTR_CONFIG_1 = { - 'package_upgrade': 'true', - 'disable_root': 0, - 'ssh_pwauth': 1, - 'chpasswd': { - 'expire': False, - 'list': [ - 'root:$6$S2Smuj.../VqxmIR9Urw0jPZ88i4yvB/' - ] - }, - 'system_info': { - 'default_user': { - 'name': 'root' - } - }, - 'network': { - 'version': 1, - 'config': [ - { - 'type': 'nameserver', - 'address': ['108.61.10.10'] - }, - { - 'name': 'eth0', - 'type': 'physical', - 'mac_address': '56:00:03:15:c4:65', - 'accept-ra': 1, - 'subnets': [ - {'type': 'dhcp', 'control': 'auto'}, - {'type': 'dhcp6', 'control': 'auto'} - ] - } - ] - } -} - -EXPECTED_VULTR_CONFIG_2 = { +# Expected config +EXPECTED_VULTR_CONFIG = { 'package_upgrade': 'true', 'disable_root': 0, 'ssh_pwauth': 1, @@ -235,39 +199,6 @@ 'default_user': { 'name': 'root' } - }, - 'network': { - 'version': 1, - 'config': [ - { - 'type': 'nameserver', - 'address': ['108.61.10.10'] - }, - { - 'name': 'eth0', - 'type': 'physical', - 'mac_address': '56:00:03:1b:4e:ca', - 'accept-ra': 1, - 'subnets': [ - {'type': 'dhcp', 'control': 'auto'}, - {'type': 'dhcp6', 'control': 'auto'} - ] - }, - { - 'name': 'eth1', - 'type': 'physical', - 'mac_address': '5a:00:03:1b:4e:ca', - 'accept-ra': 1, - 'subnets': [ - { - "type": "static", - "control": "auto", - "address": "10.1.112.3", - "netmask": "255.255.240.0" - } - ], - } - ] } } @@ -337,6 +268,15 @@ class TestDataSourceVultr(CiTestCase): def setUp(self): super(TestDataSourceVultr, self).setUp() + + # Stored as a dict to make it easier to maintain + raw1 = json.dumps(VULTR_V1_1['vendor-data']['config']) + raw2 = json.dumps(VULTR_V1_2['vendor-data']['config']) + + # Make expected format + VULTR_V1_1['vendor-data']['config'] = raw1 + VULTR_V1_2['vendor-data']['config'] = raw2 + self.tmp = self.tmp_dir() # Test the datasource itself @@ -374,7 +314,7 @@ def test_datasource(self, # Test vendor config self.assertEqual( - EXPECTED_VULTR_CONFIG_2, + EXPECTED_VULTR_CONFIG, json.loads(vendordata[0].replace("#cloud-config", ""))) self.maxDiff = orig_val @@ -382,30 +322,6 @@ def test_datasource(self, # Test network config generation self.assertEqual(EXPECTED_VULTR_NETWORK_2, source.network_config) - # Test overall config generation - @mock.patch('cloudinit.net.get_interfaces_by_mac') - def test_get_data_1(self, mock_netmap): - mock_netmap.return_value = INTERFACE_MAP - - # Test data - orig_val = self.maxDiff - self.maxDiff = None - self.assertEqual(EXPECTED_VULTR_CONFIG_1, - vultr.generate_config(VULTR_V1_1)) - self.maxDiff = orig_val - - # Test overall config generation - @mock.patch('cloudinit.net.get_interfaces_by_mac') - def test_get_data_2(self, mock_netmap): - mock_netmap.return_value = INTERFACE_MAP - - # Test data with private networking - orig_val = self.maxDiff - self.maxDiff = None - self.assertEqual(EXPECTED_VULTR_CONFIG_2, - vultr.generate_config(VULTR_V1_2)) - self.maxDiff = orig_val - # Test network config generation @mock.patch('cloudinit.net.get_interfaces_by_mac') def test_network_config(self, mock_netmap): From 70f4e9e54e369dcefe64c01182122cb35e4f8343 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Mon, 12 Apr 2021 13:44:33 -0400 Subject: [PATCH 21/22] Remove excess lines --- cloudinit/sources/DataSourceVultr.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py index 7115ee6ad80..c08ff848fe0 100644 --- a/cloudinit/sources/DataSourceVultr.py +++ b/cloudinit/sources/DataSourceVultr.py @@ -29,14 +29,6 @@ def __init__(self, sys_cfg, distro, paths): self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, ["datasource", "Vultr"], {}), BUILTIN_DS_CONFIG]) - BUILTIN_DS_CONFIG['url'] = self.ds_cfg.get( - 'url', BUILTIN_DS_CONFIG['url']) - BUILTIN_DS_CONFIG['retries'] = self.ds_cfg.get( - 'retries', BUILTIN_DS_CONFIG['retries']) - BUILTIN_DS_CONFIG['timeout'] = self.ds_cfg.get( - 'timeout', BUILTIN_DS_CONFIG['timeout']) - BUILTIN_DS_CONFIG['wait'] = self.ds_cfg.get( - 'wait', BUILTIN_DS_CONFIG['wait']) # Initiate data and check if Vultr def _get_data(self): From 07e9f5a889a632e88f453f033d83d75d6f86cb16 Mon Sep 17 00:00:00 2001 From: Eric Benner Date: Tue, 13 Apr 2021 11:04:41 -0400 Subject: [PATCH 22/22] Cleanup variable usage for network generation --- cloudinit/sources/helpers/vultr.py | 44 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index ab0251d20a2..c22cd0b12f5 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -96,7 +96,7 @@ def get_interface_name(mac): # Generate network configs -def generate_network_config(md): +def generate_network_config(interfaces): network = { "version": 1, "config": [ @@ -110,28 +110,30 @@ def generate_network_config(md): } # Prepare interface 0, public - if len(md) > 0: - network['config'].append(generate_public_network_interface(md)) + if len(interfaces) > 0: + public = generate_public_network_interface(interfaces[0]) + network['config'].append(public) # Prepare interface 1, private - if len(md) > 1: - network['config'].append(generate_private_network_interface(md)) + if len(interfaces) > 1: + private = generate_private_network_interface(interfaces[1]) + network['config'].append(private) return network # Input Metadata and generate public network config part -def generate_public_network_interface(interfaces): - interface_name = get_interface_name(interfaces[0]['mac']) +def generate_public_network_interface(interface): + interface_name = get_interface_name(interface['mac']) if not interface_name: raise RuntimeError( "Interface: %s could not be found on the system" % - interfaces[0]['mac']) + interface['mac']) netcfg = { "name": interface_name, "type": "physical", - "mac_address": interfaces[0]['mac'], + "mac_address": interface['mac'], "accept-ra": 1, "subnets": [ { @@ -146,9 +148,9 @@ def generate_public_network_interface(interfaces): } # Check for additional IP's - additional_count = len(interfaces[0]['ipv4']['additional']) - if "ipv4" in interfaces[0] and additional_count > 0: - for additional in interfaces[0]['ipv4']['additional']: + additional_count = len(interface['ipv4']['additional']) + if "ipv4" in interface and additional_count > 0: + for additional in interface['ipv4']['additional']: add = { "type": "static", "control": "auto", @@ -158,9 +160,9 @@ def generate_public_network_interface(interfaces): netcfg['subnets'].append(add) # Check for additional IPv6's - additional_count = len(interfaces[0]['ipv6']['additional']) - if "ipv6" in interfaces[0] and additional_count > 0: - for additional in interfaces[0]['ipv6']['additional']: + additional_count = len(interface['ipv6']['additional']) + if "ipv6" in interface and additional_count > 0: + for additional in interface['ipv6']['additional']: add = { "type": "static6", "control": "auto", @@ -174,24 +176,24 @@ def generate_public_network_interface(interfaces): # Input Metadata and generate private network config part -def generate_private_network_interface(interfaces): - interface_name = get_interface_name(interfaces[1]['mac']) +def generate_private_network_interface(interface): + interface_name = get_interface_name(interface['mac']) if not interface_name: raise RuntimeError( "Interface: %s could not be found on the system" % - interfaces[1]['mac']) + interface['mac']) netcfg = { "name": interface_name, "type": "physical", - "mac_address": interfaces[1]['mac'], + "mac_address": interface['mac'], "accept-ra": 1, "subnets": [ { "type": "static", "control": "auto", - "address": interfaces[1]['ipv4']['address'], - "netmask": interfaces[1]['ipv4']['netmask'] + "address": interface['ipv4']['address'], + "netmask": interface['ipv4']['netmask'] } ] }