Merge branch '2018.3' into silence-linux_distribution-deprecation-war…

…ning
saltstack · Feb 13, 2019 · c4feea1 · c4feea1
2 parents e1dcbb5 + edb1093
commit c4feea1
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 101 deletions.
diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst
@@ -803,6 +803,23 @@ A value of 10 minutes is a reasonable default.
 
     grains_refresh_every: 0
 
+.. conf_minion:: metadata_server_grains
+
+``metadata_server_grains``
+--------------------------
+
+.. versionadded:: 2017.7.0
+
+Default: ``False``
+
+Set this option to enable gathering of cloud metadata from
+``http://169.254.169.254/latest`` for use in grains (see :py:mod:`here
+<salt.grains.metadata>` for more information).
+
+.. code-block:: yaml
+
+    metadata_server_grains: True
+
 .. conf_minion:: fibre_channel_grains
 
 ``fibre_channel_grains``

diff --git a/salt/grains/core.py b/salt/grains/core.py
@@ -1395,7 +1395,9 @@ def id_():
     'KDE neon': 'Debian',
     'Void': 'Void',
     'IDMS': 'Debian',
-    'AIX': 'AIX'
+    'Funtoo': 'Gentoo',
+    'AIX': 'AIX',
+    'TurnKey': 'Debian',
 }
 
 # Matches any possible format:

diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py
@@ -280,7 +280,7 @@ def _run(cmd,
     '''
     if 'pillar' in kwargs and not pillar_override:
         pillar_override = kwargs['pillar']
-    if _is_valid_shell(shell) is False:
+    if output_loglevel != 'quiet' and _is_valid_shell(shell) is False:
         log.warning(
             'Attempt to run a shell command with what may be an invalid shell! '
             'Check to ensure that the shell <%s> is valid for this user.',

diff --git a/salt/pillar/ec2_pillar.py b/salt/pillar/ec2_pillar.py
@@ -2,27 +2,37 @@
 '''
 Retrieve EC2 instance data for minions for ec2_tags and ec2_tags_list
 
-The minion id must be the AWS instance-id or value in 'tag_match_key'.
-For example set 'tag_match_key' to 'Name', to have the minion-id matched against the
-tag 'Name'. The tag contents must be unique.  The value of tag_match_value can
-be 'uqdn' or 'asis'. if 'uqdn' strips any domain before comparison.
+The minion id must be the AWS instance-id or value in ``tag_match_key``.  For
+example set ``tag_match_key`` to ``Name`` to have the minion-id matched against
+the tag 'Name'. The tag contents must be unique. The value of
+``tag_match_value`` can be 'uqdn' or 'asis'. if 'uqdn', then the domain will be
+stripped before comparison.
 
-The option use_grain can be set to True.  This allows the use of an
-instance-id grain instead of the minion-id.  Since this is a potential
-security risk, the configuration can be further expanded to include
-a list of minions that are trusted to only allow the alternate id
-of the instances to specific hosts.  There is no glob matching at
-this time.
+Additionally, the ``use_grain`` option can be set to ``True``. This allows the
+use of an instance-id grain instead of the minion-id. Since this is a potential
+security risk, the configuration can be further expanded to include a list of
+minions that are trusted to only allow the alternate id of the instances to
+specific hosts. There is no glob matching at this time.
 
-The optional 'tag_list_key' indicates which keys should be added to
-'ec2_tags_list' and be split by tag_list_sep (default `;`). If a tag key is
-included in 'tag_list_key' it is removed from ec2_tags. If a tag does not
-exist it is still included as an empty list.
+.. note::
+    If you are using ``use_grain: True`` in the configuration for this external
+    pillar module, the minion must have :conf_minion:`metadata_server_grains`
+    enabled in the minion config file (see also :py:mod:`here
+    <salt.grains.metadata>`).
 
+    It is important to also note that enabling the ``use_grain`` option allows
+    the minion to manipulate the pillar data returned, as described above.
 
-  Note: restart the salt-master for changes to take effect.
+The optional ``tag_list_key`` indicates which keys should be added to
+``ec2_tags_list`` and be split by ``tag_list_sep`` (by default ``;``). If a tag
+key is included in ``tag_list_key`` it is removed from ec2_tags. If a tag does
+not exist it is still included as an empty list.
 
 
+..note::
+    As with any master configuration change, restart the salt-master daemon for
+    changes to take effect.
+
 .. code-block:: yaml
 
     ext_pillar:
@@ -38,11 +48,10 @@
             - trusted-minion-2
             - trusted-minion-3
 
-This is a very simple pillar that simply retrieves the instance data
-from AWS.  Currently the only portion implemented are EC2 tags, which
-returns a list of key/value pairs for all of the EC2 tags assigned to
-the instance.
-
+This is a very simple pillar configuration that simply retrieves the instance
+data from AWS. Currently the only portion implemented are EC2 tags, which
+returns a list of key/value pairs for all of the EC2 tags assigned to the
+instance.
 '''
 
 # Import python libs

diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
@@ -183,7 +183,7 @@ class TestDaemon(object):
     '''
     Set up the master and minion daemons, and run related cases
     '''
-    MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 120
+    MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 300
 
     def __init__(self, parser):
         self.parser = parser
@@ -219,6 +219,8 @@ def __enter__(self):
         if getattr(self.parser.options, 'ssh', False):
             self.prep_ssh()
 
+        self.wait_for_minions(time.time(), self.MINIONS_CONNECT_TIMEOUT)
+
         if self.parser.options.sysinfo:
             try:
                 print_header(
@@ -1185,84 +1187,6 @@ def __client_job_running(self, targets, jid):
             k for (k, v) in six.iteritems(running) if v and v[0]['jid'] == jid
         ]
 
-    def wait_for_minion_connections(self, targets, timeout):
-        salt.utils.process.appendproctitle('WaitForMinionConnections')
-        sys.stdout.write(
-            ' {LIGHT_BLUE}*{ENDC} Waiting at most {0} for minions({1}) to '
-            'connect back\n'.format(
-                (timeout > 60 and
-                 timedelta(seconds=timeout) or
-                 '{0} secs'.format(timeout)),
-                ', '.join(targets),
-                **self.colors
-            )
-        )
-        sys.stdout.flush()
-        expected_connections = set(targets)
-        now = datetime.now()
-        expire = now + timedelta(seconds=timeout)
-        while now <= expire:
-            sys.stdout.write(
-                '\r{0}\r'.format(
-                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
-                )
-            )
-            sys.stdout.write(
-                ' * {LIGHT_YELLOW}[Quit in {0}]{ENDC} Waiting for {1}'.format(
-                    '{0}'.format(expire - now).rsplit('.', 1)[0],
-                    ', '.join(expected_connections),
-                    **self.colors
-                )
-            )
-            sys.stdout.flush()
-
-            try:
-                responses = self.client.cmd(
-                    list(expected_connections), 'test.ping', tgt_type='list',
-                )
-            # we'll get this exception if the master process hasn't finished starting yet
-            except SaltClientError:
-                time.sleep(0.1)
-                now = datetime.now()
-                continue
-            for target in responses:
-                if target not in expected_connections:
-                    # Someone(minion) else "listening"?
-                    continue
-                expected_connections.remove(target)
-                sys.stdout.write(
-                    '\r{0}\r'.format(
-                        ' ' * getattr(self.parser.options, 'output_columns',
-                                      PNUM)
-                    )
-                )
-                sys.stdout.write(
-                    '   {LIGHT_GREEN}*{ENDC} {0} connected.\n'.format(
-                        target, **self.colors
-                    )
-                )
-                sys.stdout.flush()
-
-            if not expected_connections:
-                return
-
-            time.sleep(1)
-            now = datetime.now()
-        else:  # pylint: disable=W0120
-            print(
-                '\n {LIGHT_RED}*{ENDC} WARNING: Minions failed to connect '
-                'back. Tests requiring them WILL fail'.format(**self.colors)
-            )
-            try:
-                print_header(
-                    '=', sep='=', inline=True,
-                    width=getattr(self.parser.options, 'output_columns', PNUM)
-
-                )
-            except TypeError:
-                print_header('=', sep='=', inline=True)
-            raise SystemExit()
-
     def sync_minion_modules_(self, modules_kind, targets, timeout=None):
         if not timeout:
             timeout = 120
@@ -1339,3 +1263,20 @@ def sync_minion_modules(self, targets, timeout=None):
     def sync_minion_grains(self, targets, timeout=None):
         salt.utils.process.appendproctitle('SyncMinionGrains')
         self.sync_minion_modules_('grains', targets, timeout=timeout)
+
+    def wait_for_minions(self, start, timeout, sleep=5):
+        '''
+        Ensure all minions and masters (including sub-masters) are connected.
+        '''
+        while True:
+            try:
+                ret = self.client.run_job('*', 'test.ping')
+            except salt.exceptions.SaltClientError:
+                ret = None
+            if ret and 'minions' not in ret:
+                continue
+            if ret and sorted(ret['minions']) == ['minion', 'sub_minion']:
+                break
+            if time.time() - start >= timeout:
+                raise RuntimeError("Ping Minions Failed")
+            time.sleep(sleep)