Add python 2 compatible string testing

maerteijn · Aug 18, 2017 · 8781d5b · 8781d5b
1 parent c1cf403
commit 8781d5b
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 34 deletions.
diff --git a/fyrd/batch_systems/local.py b/fyrd/batch_systems/local.py
@@ -42,6 +42,10 @@
 except ImportError:  # python2
     from Queue import Empty
 
+from six import text_type as _txt
+from six import string_types as _str
+from six import integer_types as _int
+
 import psutil as _psutil
 
 import Pyro4
@@ -300,7 +304,7 @@ def create_database(self, confirm=True):
 
     def __getitem__(self, x):
         """Quick access to jobs by ID."""
-        if isinstance(x, str):
+        if isinstance(x, (_str, _txt)):
             return self.query().filter(Job.jobno == x).all()
 
     def __len__(self):
@@ -555,10 +559,13 @@ def submit(self, command, name, threads=1, dependencies=None,
         Raises
         ------
         QueueError
-            If a dependency is not in the job db already
+            If a dependency is not in the job db already or command is invalid
         """
         session = self.db.get_session()
         threads = int(threads)
+        if not isinstance(command, (_str, _txt)):
+            raise ValueError('command is {0}, type{1}, cannot continue'
+                             .format(command, type(command)))
         depends = []
         if dependencies:
             for dep in dependencies:
@@ -614,7 +621,7 @@ def get(self, jobs=None, preclean=True):
             Job.exitcode, Job.runpath, Job.outfile, Job.errfile
         )
         if jobs:
-            jobs = [jobs] if isinstance(jobs, (int, str)) else jobs
+            jobs = [jobs] if isinstance(jobs, (_int, _str, _txt)) else jobs
             jobs = [int(j) for j in jobs]
             q = q.filter(Job.jobno.in_(jobs))
         res = q.all()
@@ -658,7 +665,7 @@ def kill(self, jobs):
         ----------
         jobs : list of int
         """
-        if isinstance(jobs, str):
+        if isinstance(jobs, (_str, _txt)):
             jobs = [int(jobs)]
         elif isinstance(jobs, int):
             jobs = [jobs]
@@ -728,7 +735,7 @@ def shutdown_jobs(self):
             self.inqueue.put('stop')
             print('waiting for jobs to terminate gracefully')
             try:
-                result = self.outqueue.get(STOP_WAIT)
+                result = self.outqueue.get(timeout=STOP_WAIT)
             except Empty:
                 pass
         print('killing runner')
@@ -927,6 +934,9 @@ def job_runner(inqueue, outqueue, max_jobs):
             if info[0] != 'queue':
                 raise QueueError('Invalid argument: {0}'.format(info[0]))
             jobno, command, threads, depends, stdout, stderr, runpath = info[1]
+            if not command:
+                raise QueueError('Job command is {0}, cannot continue'
+                                 .format(type(command)))
             jobno = int(jobno)
             threads = int(threads)
             # Run anyway
@@ -973,7 +983,6 @@ def job_runner(inqueue, outqueue, max_jobs):
                 if not_done:
                     continue
             if info['threads'] <= available_cores:
-                assert isinstance(info['command'], str)
                 if info['runpath']:
                     curpath = _os.path.abspath('.')
                     _os.chdir(info['runpath'])
@@ -1357,6 +1366,9 @@ def submit(file_name, dependencies=None, job=None, args=None, kwds=None):
             params[param] = None
     # Submit the job
     server = get_server()
+    if not _os.path.isfile(file_name):
+        raise QueueError('File {0} does not exist, cannot submit'
+                         .format(file_name))
     command = 'bash {0}'.format(_os.path.abspath(file_name))
     jobno = server.submit(
         command, params['name'], threads=params['cores'],

diff --git a/fyrd/batch_systems/options.py b/fyrd/batch_systems/options.py
@@ -30,6 +30,8 @@
 from collections import OrderedDict as _OD
 
 from six import reraise as _raise
+from six import text_type as _txt
+from six import string_types as _str
 from tabulate import tabulate as _tabulate
 
 from .. import logme
@@ -356,7 +358,7 @@ def check_arguments(kwargs):
                                    'it is formatted as {}'.format(opt))
 
         # Force memory into an integer of megabytes
-        elif arg == 'mem' and isinstance(opt, str):
+        elif arg == 'mem' and isinstance(opt, (_str, _txt)):
             if opt.isdigit():
                 opt = int(opt)
             else:

diff --git a/fyrd/batch_systems/slurm.py b/fyrd/batch_systems/slurm.py
@@ -8,6 +8,10 @@
 import pwd as _pwd     # Used to get usernames for queue
 from subprocess import CalledProcessError as _CalledProcessError
 
+from six import text_type as _txt
+from six import string_types as _str
+from six import integer_types as _int
+
 from .. import run as _run
 from .. import conf as _conf
 from .. import logme as _logme
@@ -323,15 +327,15 @@ def queue_parser(user=None, partition=None):
                                 .format(len(sinfo)))
         if partition and spartition != partition:
             continue
-        if not isinstance(sid, str):
+        if not isinstance(sid, (_str, _txt)):
             sid = str(sid) if sid else None
         else:
             sarr = None
-        if not isinstance(snodes, int):
+        if not isinstance(snodes, _int):
             snodes = int(snodes) if snodes else None
-        if not isinstance(scpus, int):
+        if not isinstance(scpus, _int):
             scpus = int(scpus) if snodes else None
-        if not isinstance(scode, int):
+        if not isinstance(scode, _int):
             scode = int(scode) if scode else None
         sstate = sstate.lower()
         # Convert user from ID to name

diff --git a/fyrd/helpers.py b/fyrd/helpers.py
@@ -12,6 +12,9 @@
     Split a file, run a command in parallel, return result.
 """
 import os as _os
+from six import text_type as _txt
+from six import string_types as _str
+from six import integer_types as _int
 
 ###############################################################################
 #                               Import Ourself                                #
@@ -240,15 +243,15 @@ def _parapply(jobs, df, func, args=(), profile=None, applymap=False,
               **kwds):
     """Direct running function for parapply, see parapply docstring."""
     # Handle arguments
-    if not isinstance(jobs, int):
+    if not isinstance(jobs, _int):
         raise ValueError('Jobs argument must be an integer.')
     if not isinstance(df, (_pd.core.frame.DataFrame, _np.ndarray)):
         raise ValueError('df must be a dataframe or numpy array, is {}'
                          .format(type(df)))
     if not callable(func):
         raise ValueError('function must be callable, current type is {}'
                          .format(type(func)))
-    if profile is not None and not isinstance(profile, str):
+    if profile is not None and not isinstance(profile, (_str, _txt)):
         raise ValueError('Profile must be a string, is {}'
                          .format(type(profile)))
     fyrd_kwds, pandas_kwds = _options.split_keywords(kwds)
@@ -596,9 +599,9 @@ def _splitrun(jobs, infile, inheader, command, args=None, kwargs=None,
     function directly.
     """
     # Handle arguments
-    if not isinstance(jobs, int):
+    if not isinstance(jobs, _int):
         raise ValueError('Jobs argument must be an integer.')
-    if profile is not None and not isinstance(profile, str):
+    if profile is not None and not isinstance(profile, (_str, _txt)):
         raise ValueError('Profile must be a string, is {}'
                          .format(type(profile)))
     kwds = _options.check_arguments(kwds)

diff --git a/fyrd/job.py b/fyrd/job.py
@@ -13,6 +13,9 @@
 # Try to use dill, revert to pickle if not found
 import dill as _pickle
 from six import reraise as _reraise
+from six import text_type as _txt
+from six import string_types as _str
+from six import integer_types as _int
 
 ###############################################################################
 #                                Our functions                                #
@@ -457,7 +460,7 @@ def initialize(self):
             self.dependencies = []
             errmsg = 'Dependencies must be number, numeric string or Job'
             for dependency in dependencies:
-                if not isinstance(dependency, (str, Job)):
+                if not isinstance(dependency, (_str, _txt, Job)):
                     raise _ClusterError(errmsg)
                 self.dependencies.append(dependency)
 

diff --git a/fyrd/queue.py b/fyrd/queue.py
@@ -31,6 +31,10 @@
 from time import time as _time
 from time import sleep as _sleep
 
+from six import text_type as _txt
+from six import string_types as _str
+from six import integer_types as _int
+
 ###############################################################################
 #                                Our functions                                #
 ###############################################################################
@@ -127,7 +131,7 @@ def __init__(self, user=None, partition=None, qtype=None,):
                 self.user = None
             else:
                 if isinstance(user, int) \
-                        or (isinstance(user, str) and user.isdigit()):
+                        or (isinstance(user, (_str, _txt)) and user.isdigit()):
                     self.uid  = int(user)
                 else:
                     self.uid = _pwd.getpwnam(str(user)).pw_uid
@@ -230,7 +234,7 @@ def wait(self, jobs, return_disp=False):
         if not isinstance(jobs, (list, tuple)):
             jobs = [jobs]
         for job in jobs:
-            if not isinstance(job, (str, int, QueueJob, self._Job)):
+            if not isinstance(job, (_str, _txt, _int, QueueJob, self._Job)):
                 raise _ClusterError('job must be int, string, or Job, ' +
                                     'is {}'.format(type(job)))
 
@@ -474,13 +478,7 @@ def get_user_jobs(self, users):
             A filtered job dictionary of `{job_id: QueueJob}` for all jobs
             owned by the queried users.
         """
-        try:
-            if isinstance(users, (str, int)):
-                users = [users]
-            else:
-                users = list(users)
-        except TypeError:
-            users = [users]
+        users = _run.listify(users)
         return {k: v for k, v in self.jobs.items() if v.owner in users}
 
     @property

diff --git a/fyrd/run.py b/fyrd/run.py
@@ -24,6 +24,9 @@
 from time import sleep
 from glob import glob as _glob
 
+from six import text_type as _txt
+from six import string_types as _str
+from six import integer_types as _int
 from six.moves import input as _get_input
 
 # Progress bar handling
@@ -345,7 +348,7 @@ def listify(iterable):
     """Try to force any iterable into a list sensibly."""
     if isinstance(iterable, list):
         return iterable
-    if isinstance(iterable, (str, int, float)):
+    if isinstance(iterable, (_str, _txt, _int, float)):
         return [iterable]
     if not iterable:
         return []
@@ -424,7 +427,7 @@ def open_zipped(infile, mode='r'):
     mode   = mode[0] + 't'
     if hasattr(infile, 'write'):
         return infile
-    if isinstance(infile, str):
+    if isinstance(infile, _str):
         if infile.endswith('.gz'):
             return gzip.open(infile, mode)
         if infile.endswith('.bz2'):
@@ -619,7 +622,7 @@ def cmd(command, args=None, stdout=None, stderr=None, tries=1):
             raise ValueError('Cannot submit list/tuple command as ' +
                              'well as args argument')
         command = ' '.join(command)
-    assert isinstance(command, str)
+    assert isinstance(command, _str)
     if args:
         if isinstance(args, (list, tuple)):
             args = ' '.join(args)
@@ -754,14 +757,14 @@ def replace_argument(args, find_string, replace_string, error=True):
     newargs = tuple()
     if args:
         for arg in listify(args):
-            if isinstance(arg, str) and find_string in arg:
+            if isinstance(arg, _str) and find_string in arg:
                 arg = arg.format(**{find_string.strip('{}'): replace_string})
                 found = True
             newargs += (arg,)
     newkwds = {}
     if kwargs:
         for arg, value in kwargs.items():
-            if isinstance(value, str) and find_string in value:
+            if isinstance(value, _str) and find_string in value:
                 value = replace_string
                 found = True
             newkwds[arg] = value
@@ -874,7 +877,7 @@ def get_input(message, valid_answers=None, default=None):
     if not message.endswith(' '):
         message = message + ' '
     if valid_answers:
-        if isinstance(valid_answers, str):
+        if isinstance(valid_answers, _str):
             if valid_answers.lower() == 'yesno':
                 valid_answers = ['yes', 'no', 'y', 'n']
             else:
@@ -952,7 +955,7 @@ def normalize_imports(imports, prot=True):
     if not imports:
         return []
     for imp in imports:
-        if not isinstance(imp, str):
+        if not isinstance(imp, _str):
             raise ValueError('All imports must be strings')
         if imp.startswith('try:'):
             prot_impts.append(imp.rstrip())

diff --git a/fyrd/script_runners.py b/fyrd/script_runners.py
@@ -124,7 +124,7 @@ def run_function(func_c, args=None, kwargs=None):
             iter(args)
         except TypeError:
             args = (args,)
-        if isinstance(args, str):
+        if isinstance(args, (six.string_types, six.text_type)):
             args = (args,)
         ot = func_c(*args)
     elif kwargs:

diff --git a/tests/test_requirements.txt b/tests/test_requirements.txt
@@ -9,6 +9,5 @@ codeclimate-test-reporter>=0.2.0
 codacy-coverage>=1.3.3
 tqdm>=4.15.0
 Pyro4>=4.62
-daemonocle>=1.0
 sqlalchemy>=1.1.13
 psutil>=5.2