Merge remote-tracking branch 'upstream/2015.2' into merge-forward-dev…

…elop Conflicts: salt/modules/cmdmod.py salt/modules/debian_ip.py salt/modules/drac.py salt/modules/rpm.py salt/returners/postgres_local_cache.py
jraby · Mar 19, 2015 · d82d764 · d82d764
2 parents 3196cc8 + 7bbff8c
commit d82d764
Show file tree

Hide file tree

Showing 25 changed files with 431 additions and 104 deletions.
diff --git a/pkg/windows/BuildSalt.bat b/pkg/windows/BuildSalt.bat
@@ -0,0 +1,54 @@
+@ echo off
+:: Define Variables
+@ echo Defining Variables...
+Set "CurrDir=%cd%"
+Set "BinDir=%cd%\buildenv\bin"
+Set "InsDir=%cd%\installer"
+Set "PyDir=C:\Python27"
+
+:: Find the NSIS Installer
+If Exist "C:\Program Files\NSIS\" (
+    Set NSIS="C:\Program Files\NSIS\"
+) Else (
+    Set NSIS="C:\Program Files (x86)\NSIS\"
+)
+Set "PATH=%NSIS%;%PATH%"
+
+@ echo Copying C:\Python27 to bin...
+:: Check for existing bin directory and remove
+If Exist "%BinDir%\" rd "%BinDir%" /S /Q
+
+:: Copy the Python27 directory to bin
+xcopy /S /E %PyDir% %CurrDir%\buildenv\bin\
+
+@ echo Cleaning up unused files and directories...
+:: Remove all Compiled Python files (.pyc)
+del /S /Q %BinDir%\*.pyc
+
+:: Delete Unused Docs and Modules
+rd /S /Q %BinDir%\Doc
+rd /S /Q %BinDir%\share
+rd /S /Q %BinDir%\tcl
+rd /S /Q %BinDir%\Lib\idlelib
+rd /S /Q %BinDir%\Lib\lib-tk
+rd /S /Q %BinDir%\Lib\test
+rd /S /Q %BinDir%\Lib\unit-test
+
+:: Delete Unused .dll files
+del /S /Q %BinDir%\DLLs\tcl85.dll
+del /S /Q %BinDir%\DLLs\tclpip85.dll
+del /S /Q %BinDir%\DLLs\tk85.dll
+
+:: Delete .txt files
+del /q %BinDir%\NEWS.txt
+del /q %BinDir%\README.txt
+
+@ echo Building the installer...
+makensis.exe "%InsDir%\Salt-Minion-Setup.nsi
+
+@ echo.
+@ echo Script completed...
+@ echo Installation file can be found in the following directory:
+@ echo %InsDir%
+pause
+cls
diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat
@@ -0,0 +1,2 @@
+net stop salt-minion
+.\bin\python.exe .\bin\Scripts\salt-minion -l debug -c C:\salt\conf
diff --git a/pkg/windows/buildenv/salt-minion-start-service.bat b/pkg/windows/buildenv/salt-minion-start-service.bat
@@ -0,0 +1 @@
+net start salt-minion
diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi
@@ -142,30 +142,30 @@ ShowUnInstDetails show
 ; See http://blogs.msdn.com/b/astebner/archive/2009/01/29/9384143.aspx for more info
 Section -Prerequisites
 
-  !define VC_REDIST_X64_GUID "{5FCE6D76-F5DC-37AB-B2B8-22AB8CEDB1D4}"
-  !define VC_REDIST_X86_GUID "{9BE518E6-ECC6-35A9-88E4-87755C07200F}"
-  !define VC_REDIST_X64_URI "http://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe"
-  !define VC_REDIST_X86_URI "http://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x86.exe"
-
-  Var /GLOBAL VcRedistGuid
-  Var /GLOBAL VcRedistUri
-  ${If} ${RunningX64}
-    StrCpy $VcRedistGuid ${VC_REDIST_X64_GUID}
-    StrCpy $VcRedistUri  ${VC_REDIST_X64_URI}
-  ${Else}
-    StrCpy $VcRedistGuid ${VC_REDIST_X86_GUID}
-    StrCpy $VcRedistUri  ${VC_REDIST_X86_URI}
-  ${EndIf}
-
-  Push $VcRedistGuid
+;  !define VC_REDIST_X64_GUID "{5FCE6D76-F5DC-37AB-B2B8-22AB8CEDB1D4}"
+;  !define VC_REDIST_X86_GUID "{9BE518E6-ECC6-35A9-88E4-87755C07200F}"
+;  !define VC_REDIST_X64_URI "http://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe"
+;  !define VC_REDIST_X86_URI "http://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x86.exe"
+
+;  Var /GLOBAL VcRedistGuid
+;  Var /GLOBAL VcRedistUri
+;  ${If} ${RunningX64}
+;    StrCpy $VcRedistGuid ${VC_REDIST_X64_GUID}
+;    StrCpy $VcRedistUri  ${VC_REDIST_X64_URI}
+;  ${Else}
+;    StrCpy $VcRedistGuid ${VC_REDIST_X86_GUID}
+;    StrCpy $VcRedistUri  ${VC_REDIST_X86_URI}
+;  ${EndIf}
+
+;  Push $VcRedistGuid
   Call MsiQueryProductState
-  ${If} $NeedVcRedist == "True"
-    NSISdl::download /TIMEOUT=30000 $VcRedistUri $TEMP\vcredist.exe
-    Pop $R0
-    StrCmp $R0 "success" +2
-      MessageBox MB_OK "VC redist package download failed: $R0" /SD IDOK    ; just report, do not break installation
-    Execwait '"$TEMP\vcredist.exe" /q'
-  ${EndIf}
+;  ${If} $NeedVcRedist == "True"
+;    NSISdl::download /TIMEOUT=30000 $VcRedistUri $TEMP\vcredist.exe
+;    Pop $R0
+;    StrCmp $R0 "success" +2
+;      MessageBox MB_OK "VC redist package download failed: $R0" /SD IDOK    ; just report, do not break installation
+;    Execwait '"$TEMP\vcredist.exe" /q'
+;  ${EndIf}
 
 SectionEnd
 
@@ -177,14 +177,13 @@ Section "MainSection" SEC01
   SetOverwrite try
   CreateDirectory $INSTDIR\conf\pki\minion
   File /r "..\buildenv\"
-  Exec 'icacls c:\salt /inheritance:r /grant:r "BUILTIN\Administrators":(OI)(CI)F /grant:r "NT AUTHORITY\SYSTEM":(OI)(CI)F' 
-
+  Exec 'icacls c:\salt /inheritance:r /grant:r "BUILTIN\Administrators":(OI)(CI)F /grant:r "NT AUTHORITY\SYSTEM":(OI)(CI)F'
 
 SectionEnd
 
 Section -Post
   WriteUninstaller "$INSTDIR\uninst.exe"
-  WriteRegStr HKLM "${PRODUCT_DIR_REGKEY}" "" "$INSTDIR\salt-minion.exe"
+  WriteRegStr HKLM "${PRODUCT_DIR_REGKEY}" "" "$INSTDIR\bin\Scripts\salt-minion.exe"
   WriteRegStr ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" "DisplayName" "$(^Name)"
   WriteRegStr ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" "UninstallString" "$INSTDIR\uninst.exe"
   WriteRegStr ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" "DisplayIcon" "$INSTDIR\salt.ico"
@@ -196,7 +195,7 @@ Section -Post
 SectionEnd
 
 Function .onInstSuccess
-  Exec "nssm.exe install salt-minion $INSTDIR\salt-minion.exe -c $INSTDIR\conf -l quiet"
+  Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet"
   RMDir /R "$INSTDIR\var\cache\salt" ; removing cache from old version
   ExecWait "net start salt-minion"
 FunctionEnd
@@ -238,7 +237,8 @@ Section Uninstall
   ExecWait "sc delete salt-minion"
   Delete "$INSTDIR\uninst.exe"
   Delete "$INSTDIR\nssm.exe"
-  Delete "$INSTDIR\python*"
+  Delete "$INSTDIR\salt*"
+  Delete "$INSTDIR\bin"
 
   #Delete "$SMPROGRAMS\Salt Minion\Uninstall.lnk"
   #RMDir /r "$SMPROGRAMS\Salt Minion"

diff --git a/salt/auth/ldap.py b/salt/auth/ldap.py
@@ -227,7 +227,7 @@ def groups(username, **kwargs):
     else:
         return False
     for _, entry in search_results:
-        if entry['memberUid'][0] == username:
+        if username in entry['memberUid']:
             group_list.append(entry['cn'][0])
     log.debug('User {0} is a member of groups: {1}'.format(username, group_list))
     return group_list
diff --git a/salt/client/__init__.py b/salt/client/__init__.py
@@ -798,23 +798,38 @@ def get_returns_no_block(
             self,
             jid,
             event=None,
-            gather_errors=False):
+            gather_errors=False,
+            additional_tags=None
+           ):
         '''
         Raw function to just return events of jid excluding timeout logic
 
         Yield either the raw event data or None
+
+       Pass a list of additional tags as `additional_tags` to search the
+       event bus for non-return data, such as minion lists returned from
+       syndics.
         '''
         if event is None:
             event = self.event
+
         jid_tag = 'salt/job/{0}'.format(jid)
+
+        tag_search = []
+        tag_search.append(jid_tag)
+        if isinstance(additional_tags, str):
+            tag_search.append(additional_tags)
+        elif isinstance(additional_tags, list):
+            for tag in additional_tags:
+                tag_search.append(tag)
         while True:
             if self.opts.get('transport') == 'zeromq':
                 try:
                     raw = event.get_event_noblock()
                     if gather_errors:
                         if (raw and
-                               (raw.get('tag', '').startswith('_salt_error') or
-                                raw.get('tag', '').startswith(jid_tag))):
+                                (raw.get('tag', '').startswith('_salt_error') or
+                                any([raw.get('tag', '').startswith(tag) for tag in tag_search]))):
                             yield raw
                     else:
                         if raw and raw.get('tag', '').startswith(jid_tag):
@@ -875,10 +890,15 @@ def get_iter_returns(
         syndic_wait = 0
         last_time = False
         # iterator for this job's return
-        ret_iter = self.get_returns_no_block(jid, gather_errors=gather_errors)
+        if self.opts['order_masters']:
+            # If we are a MoM, we need to gather expected minions from downstreams masters.
+            ret_iter = self.get_returns_no_block(jid, gather_errors=gather_errors, additional_tags='syndic')
+        else:
+            ret_iter = self.get_returns_no_block(jid, gather_errors=gather_errors)
         # iterator for the info of this job
         jinfo_iter = []
         timeout_at = time.time() + timeout
+        gather_syndic_wait = time.time() + self.opts['syndic_wait']
         # are there still minions running the job out there
         # start as True so that we ping at least once
         minions_running = True
@@ -900,9 +920,6 @@ def get_iter_returns(
                 if 'minions' in raw.get('data', {}):
                     minions.update(raw['data']['minions'])
                     continue
-                if 'syndic' in raw:
-                    minions.update(raw['syndic'])
-                    continue
                 if 'return' not in raw['data']:
                     continue
                 if kwargs.get('raw', False):
@@ -925,8 +942,19 @@ def get_iter_returns(
                 # All minions have returned, break out of the loop
                 log.debug('jid {0} found all minions {1}'.format(jid, found))
                 break
+            elif len(found.intersection(minions)) >= len(minions) and self.opts['order_masters']:
+                if len(found) >= len(minions) and len(minions) > 0 and time.time() > gather_syndic_wait:
+                    # There were some minions to find and we found them
+                    # However, this does not imply that *all* masters have yet responded with expected minion lists.
+                    # Therefore, continue to wait up to the syndic_wait period (calculated in gather_syndic_wait) to see
+                    # if additional lower-level masters deliver their lists of expected
+                    # minions.
+                    break
+           # If we get here we may not have gathered the minion list yet. Keep waiting
+           # for all lower-level masters to respond with their minion lists
 
             # let start the timeouts for all remaining minions
+
             for id_ in minions - found:
                 # if we have a new minion in the list, make sure it has a timeout
                 if id_ not in minion_timeouts:

diff --git a/salt/config.py b/salt/config.py
@@ -251,6 +251,7 @@
     'random_reauth_delay': int,
     'syndic_event_forward_timeout': float,
     'syndic_max_event_process_time': float,
+    'syndic_jid_forward_cache_hwm': int,
     'ssh_passwd': str,
     'ssh_port': str,
     'ssh_sudo': bool,
@@ -590,6 +591,7 @@
     'gather_job_timeout': 5,
     'syndic_event_forward_timeout': 0.5,
     'syndic_max_event_process_time': 0.5,
+    'syndic_jid_forward_cache_hwm': 100,
     'ssh_passwd': '',
     'ssh_port': '22',
     'ssh_sudo': False,
@@ -1026,6 +1028,7 @@ def syndic_config(master_config_path,
         'sock_dir': os.path.join(
             opts['cachedir'], opts.get('syndic_sock_dir', opts['sock_dir'])
         ),
+        'cachedir': master_opts['cachedir'],
     }
     opts.update(syndic_opts)
     # Prepend root_dir to other paths

diff --git a/salt/crypt.py b/salt/crypt.py
@@ -307,7 +307,7 @@ def __singleton_init__(self, opts):
 
         self.authenticate()
 
-    def authenticate(self):
+    def authenticate(self, timeout=None, safe=None):
         '''
         Authenticate with the master, this method breaks the functional
         paradigm, it will update the master information from a fresh sign
@@ -323,7 +323,7 @@ def authenticate(self):
             acceptance_wait_time_max = acceptance_wait_time
 
         while True:
-            creds = self.sign_in()
+            creds = self.sign_in(timeout, safe)
             if creds == 'retry':
                 if self.opts.get('caller'):
                     print('Minion failed to authenticate with the master, '

diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py
@@ -718,7 +718,10 @@ def _minion_event(self, load):
             for event in load['events']:
                 self.event.fire_event(event, event['tag'])  # old dup event
                 if load.get('pretag') is not None:
-                    self.event.fire_event(event, tagify(event['tag'], base=load['pretag']))
+                    if 'data' in event:
+                        self.event.fire_event(event['data'], tagify(event['tag'], base=load['pretag']))
+                    else:
+                        self.event.fire_event(event, tagify(event['tag'], base=load['pretag']))
         else:
             tag = load['tag']
             self.event.fire_event(load, tag)

diff --git a/salt/master.py b/salt/master.py
@@ -746,6 +746,13 @@ def __bind(self):
                     raise
                 # catch all other exceptions, so we don't go defunct
                 except Exception as exc:
+                    # since we are in an exceptional state, lets attempt to tell
+                    # the minion we have a problem, otherwise the minion will get
+                    # no response and be forced to wait for their max timeout
+                    try:
+                        socket.send('Unexpected Error in Mworker')
+                    except:  # pylint: disable=W0702
+                        pass
                     # Properly handle EINTR from SIGUSR1
                     if isinstance(exc, zmq.ZMQError) and exc.errno == errno.EINTR:
                         continue
@@ -1286,9 +1293,9 @@ def _syndic_return(self, load):
         if any(key not in load for key in ('return', 'jid', 'id')):
             return None
         # if we have a load, save it
-        if 'load' in load:
+        if load.get('load'):
             fstr = '{0}.save_load'.format(self.opts['master_job_cache'])
-            self.mminion.returners[fstr](load['jid'], load)
+            self.mminion.returners[fstr](load['jid'], load['load'])
 
         # Format individual return loads
         for key, item in six.iteritems(load['return']):
@@ -1297,6 +1304,10 @@ def _syndic_return(self, load):
                    'return': item}
             if 'master_id' in load:
                 ret['master_id'] = load['master_id']
+            if 'fun' in load:
+                ret['fun'] = load['fun']
+            if 'arg' in load:
+                ret['fun_args'] = load['arg']
             if 'out' in load:
                 ret['out'] = load['out']
             self._return(ret)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		net stop salt-minion
		.\bin\python.exe .\bin\Scripts\salt-minion -l debug -c C:\salt\conf