From 183da818dfd4eaf74faebc6c042a1924596c14a2 Mon Sep 17 00:00:00 2001 From: practicalswift Date: Sat, 5 Mar 2016 13:07:51 +0100 Subject: [PATCH] [Python] Fix five classes of PEP-8 violations (E101/E111/E128/E302/W191) * E101: indentation contains mixed spaces and tabs * E111: indentation is not a multiple of four * E128: continuation line under-indented for visual indent * E302: expected 2 blank lines, found 1 * W191: indentation contains tabs --- .pep8 | 2 +- benchmark/scripts/Benchmark_DTrace.in | 11 +- benchmark/scripts/Benchmark_Driver | 148 +++-- benchmark/scripts/Benchmark_GuardMalloc.in | 11 +- .../scripts/Benchmark_RuntimeLeaksRunner.in | 7 +- benchmark/scripts/compare_perf_tests.py | 6 + .../perf_test_driver/perf_test_driver.py | 4 + docs/conf.py | 1 + docs/scripts/ns-html2rst | 1 + .../Inputs/fake-build-for-bitcode.py | 6 +- test/Driver/Dependencies/Inputs/touch.py | 4 +- .../Inputs/filelists/check-filelist-abc.py | 34 +- test/Driver/Inputs/filelists/fake-ld.py | 8 +- .../bindings/python/sourcekitd/capi.py | 332 +++++------ .../bindings/python/sourcekitd/request.py | 7 +- utils/GYBUnicodeDataUtils.py | 21 +- utils/SwiftBuildSupport.py | 8 +- utils/SwiftIntTypes.py | 20 +- utils/apply-fixit-edits.py | 3 + utils/build-script | 288 ++++++---- utils/cmpcodesize/cmpcodesize/compare.py | 10 +- utils/gyb.py | 80 ++- utils/line-directive | 4 + .../scripts/pipelines_build_script.py | 5 + utils/pass-pipeline/src/pass_pipeline.py | 4 + .../src/pass_pipeline_library.py | 8 + utils/protocol_graph.py | 6 + utils/recursive-lipo | 37 +- utils/resolve-crashes.py | 10 +- utils/rth | 7 +- utils/sil-opt-verify-all-modules.py | 13 +- utils/split_file.py | 1 + utils/swift-api-dump.py | 14 + utils/swift-bench.py | 539 +++++++++--------- .../swift_build_support/debug.py | 21 +- utils/swift_build_support/tests/test_tar.py | 21 +- utils/update-checkout | 18 +- utils/viewcfg | 14 +- 38 files changed, 1033 insertions(+), 701 deletions(-) diff --git a/.pep8 b/.pep8 index 14110bc35758c..28ac7fbde660f 100644 --- a/.pep8 +++ b/.pep8 @@ -1,3 +1,3 @@ [flake8] filename = *.py,Benchmark_Driver,Benchmark_DTrace.in,Benchmark_GuardMalloc.in,Benchmark_RuntimeLeaksRunner.in,build-script,gyb,line-directive,ns-html2rst,recursive-lipo,rth,submit-benchmark-results,update-checkout,viewcfg -ignore = D100,D101,D102,D103,D104,D105,E101,E111,E128,E302,E402,E501,W191 +ignore = D100,D101,D102,D103,D104,D105,E402,E501 diff --git a/benchmark/scripts/Benchmark_DTrace.in b/benchmark/scripts/Benchmark_DTrace.in index 1e0067337988e..59549c111576d 100644 --- a/benchmark/scripts/Benchmark_DTrace.in +++ b/benchmark/scripts/Benchmark_DTrace.in @@ -27,6 +27,7 @@ import perf_test_driver XFAIL_LIST = [ ] + class DTraceResult(perf_test_driver.Result): def __init__(self, name, status, output, csv_output): @@ -59,11 +60,14 @@ class DTraceResult(perf_test_driver.Result): print(DTraceResult.data_format(max_test_len).format(*result)) + class DTraceBenchmarkDriver(perf_test_driver.BenchmarkDriver): + def __init__(self, binary, xfail_list, csv_output): - perf_test_driver.BenchmarkDriver.__init__(self, binary, xfail_list, - enable_parallel=False, - opt_levels=['O']) + perf_test_driver.BenchmarkDriver.__init__( + self, binary, xfail_list, + enable_parallel=False, + opt_levels=['O']) self.csv_output = csv_output def print_data_header(self, max_test_len): @@ -94,6 +98,7 @@ class DTraceBenchmarkDriver(perf_test_driver.BenchmarkDriver): SWIFT_BIN_DIR = os.path.dirname(os.path.abspath(__file__)) + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-filter', type=str, default=None, diff --git a/benchmark/scripts/Benchmark_Driver b/benchmark/scripts/Benchmark_Driver index f730da31cdbf9..67e0382030484 100755 --- a/benchmark/scripts/Benchmark_Driver +++ b/benchmark/scripts/Benchmark_Driver @@ -27,6 +27,7 @@ import glob DRIVER_DIR = os.path.dirname(os.path.realpath(__file__)) + def parse_results(res, optset): # Parse lines like this # #,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),PEAK_MEMORY(B) @@ -58,10 +59,12 @@ def parse_results(res, optset): mem_test = {} mem_test['Data'] = [mem_testresult] mem_test['Info'] = {} - mem_test['Name'] = "nts.swift/mem_maxrss." + optset + "." + testname + ".mem" + mem_test['Name'] = "nts.swift/mem_maxrss." + \ + optset + "." + testname + ".mem" tests.append(mem_test) return tests + def submit_to_lnt(data, url): print "\nSubmitting results to LNT server..." json_report = {'input_data': json.dumps(data), 'commit': '1'} @@ -75,6 +78,7 @@ def submit_to_lnt(data, url): print "Error:\t", response['error'] sys.exit(1) + def instrument_test(driver_path, test, num_samples): """Run a test and instrument its peak memory use""" test_outputs = [] @@ -113,14 +117,18 @@ def instrument_test(driver_path, test, num_samples): return avg_test_output + def get_tests(driver_path): """Return a list of available performance tests""" return subprocess.check_output([driver_path, '--list']).split()[2:] + def get_current_git_branch(git_repo_path): """Return the selected branch for the repo `git_repo_path`""" - return subprocess.check_output(['git', '-C', git_repo_path, 'rev-parse', - '--abbrev-ref', 'HEAD'], stderr=subprocess.STDOUT).strip() + return subprocess.check_output( + ['git', '-C', git_repo_path, 'rev-parse', + '--abbrev-ref', 'HEAD'], stderr=subprocess.STDOUT).strip() + def log_results(log_directory, driver, formatted_output, swift_repo=None): """Log `formatted_output` to a branch specific directory in @@ -146,6 +154,7 @@ def log_results(log_directory, driver, formatted_output, swift_repo=None): with open(log_file, 'w') as f: f.write(formatted_output) + def run_benchmarks(driver, benchmarks=[], num_samples=10, verbose=False, log_directory=None, swift_repo=None): """Run perf tests individually and return results in a format that's @@ -192,6 +201,7 @@ def run_benchmarks(driver, benchmarks=[], num_samples=10, verbose=False, log_results(log_directory, driver, formatted_output, swift_repo) return formatted_output + def submit(args): print "SVN revision:\t", args.revision print "Machine name:\t", args.machine @@ -208,8 +218,9 @@ def submit(args): print "Opt level:\t", optset file = os.path.join(args.tests, "Benchmark_" + optset) try: - res = run_benchmarks(file, benchmarks=args.benchmark, - num_samples=args.iterations) + res = run_benchmarks( + file, benchmarks=args.benchmark, + num_samples=args.iterations) data['Tests'].extend(parse_results(res, optset)) except subprocess.CalledProcessError as e: print "Execution failed.. Test results are empty." @@ -227,24 +238,29 @@ def submit(args): submit_to_lnt(data, args.lnt_host) return 0 + def run(args): optset = args.optimization file = os.path.join(args.tests, "Benchmark_" + optset) - run_benchmarks(file, benchmarks=args.benchmarks, - num_samples=args.iterations, verbose=True, - log_directory=args.output_dir, - swift_repo=args.swift_repo) + run_benchmarks( + file, benchmarks=args.benchmarks, + num_samples=args.iterations, verbose=True, + log_directory=args.output_dir, + swift_repo=args.swift_repo) return 0 + def format_name(log_path): """Return the filename and directory for a log file""" return '/'.join(log_path.split('/')[-2:]) + def compare_logs(compare_script, new_log, old_log): """Return diff of log files at paths `new_log` and `old_log`""" print 'Comparing %s %s ...' % (format_name(old_log), format_name(new_log)) subprocess.call([compare_script, old_log, new_log]) + def compare(args): log_dir = args.log_dir swift_repo = args.swift_repo @@ -263,7 +279,8 @@ def compare(args): for branch_dir in [current_branch_dir, master_branch_dir]: for opt in ['O', 'Onone']: recent_logs[os.path.basename(branch_dir) + '_' + opt] = sorted( - glob.glob(os.path.join(branch_dir, 'Benchmark_' + opt + '-*.log')), + glob.glob(os.path.join( + branch_dir, 'Benchmark_' + opt + '-*.log')), key=os.path.getctime, reverse=True) if current_branch == 'master': @@ -311,65 +328,86 @@ def compare(args): return 0 + def positive_int(value): ivalue = int(value) if not (ivalue > 0): raise ValueError return ivalue + def main(): parser = argparse.ArgumentParser(description='Swift benchmarks driver') subparsers = parser.add_subparsers() - submit_parser = subparsers.add_parser('submit', - help='run benchmarks and submit results to LNT') - submit_parser.add_argument('-t', '--tests', - help='directory containing Benchmark_O{,none,unchecked} ' + - '(default: DRIVER_DIR)', - default=DRIVER_DIR) - submit_parser.add_argument('-m', '--machine', required=True, - help='LNT machine name') - submit_parser.add_argument('-r', '--revision', required=True, - help='SVN revision of compiler to identify the LNT run', type=int) - submit_parser.add_argument('-l', '--lnt_host', required=True, - help='LNT host to submit results to') - submit_parser.add_argument('-i', '--iterations', - help='number of times to run each test (default: 10)', - type=positive_int, default=10) - submit_parser.add_argument('-o', '--optimization', nargs='+', - help='optimization levels to use (default: O Onone Ounchecked)', - default=['O', 'Onone', 'Ounchecked']) - submit_parser.add_argument('benchmark', - help='benchmark to run (default: all)', nargs='*') + submit_parser = subparsers.add_parser( + 'submit', + help='run benchmarks and submit results to LNT') + submit_parser.add_argument( + '-t', '--tests', + help='directory containing Benchmark_O{,none,unchecked} ' + + '(default: DRIVER_DIR)', + default=DRIVER_DIR) + submit_parser.add_argument( + '-m', '--machine', required=True, + help='LNT machine name') + submit_parser.add_argument( + '-r', '--revision', required=True, + help='SVN revision of compiler to identify the LNT run', type=int) + submit_parser.add_argument( + '-l', '--lnt_host', required=True, + help='LNT host to submit results to') + submit_parser.add_argument( + '-i', '--iterations', + help='number of times to run each test (default: 10)', + type=positive_int, default=10) + submit_parser.add_argument( + '-o', '--optimization', nargs='+', + help='optimization levels to use (default: O Onone Ounchecked)', + default=['O', 'Onone', 'Ounchecked']) + submit_parser.add_argument( + 'benchmark', + help='benchmark to run (default: all)', nargs='*') submit_parser.set_defaults(func=submit) - run_parser = subparsers.add_parser('run', - help='run benchmarks and output results to stdout') - run_parser.add_argument('-t', '--tests', - help='directory containing Benchmark_O{,none,unchecked} ' + - '(default: DRIVER_DIR)', - default=DRIVER_DIR) - run_parser.add_argument('-i', '--iterations', - help='number of times to run each test (default: 1)', - type=positive_int, default=1) - run_parser.add_argument('-o', '--optimization', - help='optimization level to use (default: O)', default='O') - run_parser.add_argument('--output-dir', - help='log results to directory (default: no logging)') - run_parser.add_argument('--swift-repo', - help='absolute path to Swift source repo for branch comparison') - run_parser.add_argument('benchmarks', - help='benchmark to run (default: all)', nargs='*') + run_parser = subparsers.add_parser( + 'run', + help='run benchmarks and output results to stdout') + run_parser.add_argument( + '-t', '--tests', + help='directory containing Benchmark_O{,none,unchecked} ' + + '(default: DRIVER_DIR)', + default=DRIVER_DIR) + run_parser.add_argument( + '-i', '--iterations', + help='number of times to run each test (default: 1)', + type=positive_int, default=1) + run_parser.add_argument( + '-o', '--optimization', + help='optimization level to use (default: O)', default='O') + run_parser.add_argument( + '--output-dir', + help='log results to directory (default: no logging)') + run_parser.add_argument( + '--swift-repo', + help='absolute path to Swift source repo for branch comparison') + run_parser.add_argument( + 'benchmarks', + help='benchmark to run (default: all)', nargs='*') run_parser.set_defaults(func=run) - compare_parser = subparsers.add_parser('compare', - help='compare benchmark results') - compare_parser.add_argument('--log-dir', required=True, - help='directory containing benchmark logs') - compare_parser.add_argument('--swift-repo', required=True, - help='absolute path to Swift source repo') - compare_parser.add_argument('--compare-script', required=True, - help='absolute path to compare script') + compare_parser = subparsers.add_parser( + 'compare', + help='compare benchmark results') + compare_parser.add_argument( + '--log-dir', required=True, + help='directory containing benchmark logs') + compare_parser.add_argument( + '--swift-repo', required=True, + help='absolute path to Swift source repo') + compare_parser.add_argument( + '--compare-script', required=True, + help='absolute path to compare script') compare_parser.set_defaults(func=compare) args = parser.parse_args() diff --git a/benchmark/scripts/Benchmark_GuardMalloc.in b/benchmark/scripts/Benchmark_GuardMalloc.in index 30a9826fd3c46..0b36495326d61 100644 --- a/benchmark/scripts/Benchmark_GuardMalloc.in +++ b/benchmark/scripts/Benchmark_GuardMalloc.in @@ -24,15 +24,19 @@ import perf_test_driver XFAIL_LIST = [ ] + class GuardMallocResult(perf_test_driver.Result): def __init__(self, name, status): perf_test_driver.Result.__init__(self, name, status, "", XFAIL_LIST) + class GuardMallocBenchmarkDriver(perf_test_driver.BenchmarkDriver): + def __init__(self, binary, xfail_list): - perf_test_driver.BenchmarkDriver.__init__(self, binary, xfail_list, - enable_parallel=True) + perf_test_driver.BenchmarkDriver.__init__( + self, binary, xfail_list, + enable_parallel=True) self.new_env = os.environ.copy() self.new_env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib' @@ -43,7 +47,8 @@ class GuardMallocBenchmarkDriver(perf_test_driver.BenchmarkDriver): test_name = '({},{})'.format(data['opt'], data['test_name']) print "Running {}...".format(test_name) sys.stdout.flush() - status = subprocess.call([data['path'], data['test_name'], '--num-iters=2'], + status = subprocess.call( + [data['path'], data['test_name'], '--num-iters=2'], env=data['env'], stderr=open('/dev/null', 'w'), stdout=open('/dev/null', 'w')) return GuardMallocResult(test_name, status) diff --git a/benchmark/scripts/Benchmark_RuntimeLeaksRunner.in b/benchmark/scripts/Benchmark_RuntimeLeaksRunner.in index 512847eb98ec6..22c2fabbc4e97 100644 --- a/benchmark/scripts/Benchmark_RuntimeLeaksRunner.in +++ b/benchmark/scripts/Benchmark_RuntimeLeaksRunner.in @@ -53,16 +53,19 @@ IGNORABLE_GLOBAL_OBJC_CLASSES = set([ '_NSJSONReader' ]) + class LeaksRunnerResult(perf_test_driver.Result): def __init__(self, name, status): perf_test_driver.Result.__init__(self, name, status, "", XFAIL_LIST) + class LeaksRunnerBenchmarkDriver(perf_test_driver.BenchmarkDriver): def __init__(self, binary, xfail_list): - perf_test_driver.BenchmarkDriver.__init__(self, binary, xfail_list, - enable_parallel=True) + perf_test_driver.BenchmarkDriver.__init__( + self, binary, xfail_list, + enable_parallel=True) def prepare_input(self, name): return {} diff --git a/benchmark/scripts/compare_perf_tests.py b/benchmark/scripts/compare_perf_tests.py index eb0d4c95cfa27..ad03dea4b86a3 100755 --- a/benchmark/scripts/compare_perf_tests.py +++ b/benchmark/scripts/compare_perf_tests.py @@ -35,12 +35,14 @@ ShowSpeedup = 1 PrintAllScores = 0 + def parse_int(word): try: return int(word) except: raise Exception("Expected integer value, not " + word) + def get_scores(fname): scores = {} worstscores = {} @@ -77,9 +79,11 @@ def get_scores(fname): f.close() return scores, worstscores, runs, nums + def is_max_score(newscore, maxscore, invert): return not maxscore or (newscore > maxscore if not invert else newscore < maxscore) + def compare_scores(key, score1, worstsample1, score2, worstsample2, runs, num): print num.rjust(3), print key.ljust(25), @@ -144,6 +148,7 @@ def compare_scores(key, score1, worstsample1, score2, worstsample2, runs, num): print "(?)", print + def print_best_scores(key, scores): print key, bestscore = None @@ -153,6 +158,7 @@ def print_best_scores(key, scores): bestscore = score print ", %d" % bestscore + def usage(): print "repeat.sh Benchmark_O[none|unchecked] > file.times" print "compare_perf_tests.py []" diff --git a/benchmark/scripts/perf_test_driver/perf_test_driver.py b/benchmark/scripts/perf_test_driver/perf_test_driver.py index b7a1afecbc2ed..ef144094ea0f0 100644 --- a/benchmark/scripts/perf_test_driver/perf_test_driver.py +++ b/benchmark/scripts/perf_test_driver/perf_test_driver.py @@ -17,7 +17,9 @@ import multiprocessing import re + class Result(object): + def __init__(self, name, status, output, xfail_list): self.name = name self.status = status @@ -56,11 +58,13 @@ def print_data(self, max_test_len): fmt = '{:<%d}{:}' % (max_test_len + 5) print(fmt.format(self.get_name(), self.get_result())) + def _unwrap_self(args): return type(args[0]).process_input(*args) BenchmarkDriver_OptLevels = ['Onone', 'O', 'Ounchecked'] + class BenchmarkDriver(object): def __init__(self, binary_dir, xfail_list, enable_parallel=False, opt_levels=BenchmarkDriver_OptLevels): diff --git a/docs/conf.py b/docs/conf.py index e7c7888928df7..af9f6d53ab454 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -277,6 +277,7 @@ # Monkeypatch pygments.lexers.get_lexer_by_name to return our lexers from pygments.lexers import get_lexer_by_name as original_get_lexer_by_name + def swift_get_lexer_by_name(_alias, *args, **kw): if _alias == 'swift': return swift_pygments_lexers.SwiftLexer() diff --git a/docs/scripts/ns-html2rst b/docs/scripts/ns-html2rst index 82d8d92d70a4d..932a40f14b8d3 100755 --- a/docs/scripts/ns-html2rst +++ b/docs/scripts/ns-html2rst @@ -5,6 +5,7 @@ import re import subprocess import sys + def run(): if len(sys.argv) > 1: print(""" diff --git a/test/Driver/Dependencies/Inputs/fake-build-for-bitcode.py b/test/Driver/Dependencies/Inputs/fake-build-for-bitcode.py index 5e14ed8d148ff..462435aa615c3 100755 --- a/test/Driver/Dependencies/Inputs/fake-build-for-bitcode.py +++ b/test/Driver/Dependencies/Inputs/fake-build-for-bitcode.py @@ -32,8 +32,8 @@ os.utime(outputFile, None) if '-emit-bc' in sys.argv: - print("Handled", os.path.basename(primaryFile)) + print("Handled", os.path.basename(primaryFile)) elif '-c' in sys.argv: - print("Produced", os.path.basename(outputFile)) + print("Produced", os.path.basename(outputFile)) else: - assert False, "unknown action" + assert False, "unknown action" diff --git a/test/Driver/Dependencies/Inputs/touch.py b/test/Driver/Dependencies/Inputs/touch.py index df02b1ccca6b3..e618225fd1c06 100755 --- a/test/Driver/Dependencies/Inputs/touch.py +++ b/test/Driver/Dependencies/Inputs/touch.py @@ -27,5 +27,5 @@ # Update the output file mtime, or create it if necessary. # From http://stackoverflow.com/a/1160227. for outputFile in sys.argv[1:]: - with open(outputFile, 'a'): - os.utime(outputFile, (timeVal, timeVal)) + with open(outputFile, 'a'): + os.utime(outputFile, (timeVal, timeVal)) diff --git a/test/Driver/Inputs/filelists/check-filelist-abc.py b/test/Driver/Inputs/filelists/check-filelist-abc.py index f5487763bfac4..6e04a583ddc7b 100755 --- a/test/Driver/Inputs/filelists/check-filelist-abc.py +++ b/test/Driver/Inputs/filelists/check-filelist-abc.py @@ -19,33 +19,33 @@ assert sys.argv[1] == '-frontend' if '-primary-file' in sys.argv: - primaryFile = sys.argv[sys.argv.index('-primary-file') + 1] + primaryFile = sys.argv[sys.argv.index('-primary-file') + 1] else: - primaryFile = None + primaryFile = None if primaryFile and primaryFile.endswith(".bc"): - sys.exit() + sys.exit() filelistFile = sys.argv[sys.argv.index('-filelist') + 1] with open(filelistFile, 'r') as f: - lines = f.readlines() - assert lines[0].endswith("/a.swift\n") or lines[0].endswith("/a.swiftmodule\n") - assert lines[1].endswith("/b.swift\n") or lines[1].endswith("/b.swiftmodule\n") - assert lines[2].endswith("/c.swift\n") or lines[2].endswith("/c.swiftmodule\n") + lines = f.readlines() + assert lines[0].endswith("/a.swift\n") or lines[0].endswith("/a.swiftmodule\n") + assert lines[1].endswith("/b.swift\n") or lines[1].endswith("/b.swiftmodule\n") + assert lines[2].endswith("/c.swift\n") or lines[2].endswith("/c.swiftmodule\n") if primaryFile: - print("Handled", os.path.basename(primaryFile)) + print("Handled", os.path.basename(primaryFile)) elif lines[0].endswith(".swiftmodule\n"): - print("Handled modules") + print("Handled modules") else: - print("Handled all") + print("Handled all") if '-num-threads' in sys.argv: - outputListFile = sys.argv[sys.argv.index('-output-filelist') + 1] - with open(outputListFile, 'r') as f: - lines = f.readlines() - assert lines[0].endswith("/a.o\n") or lines[0].endswith("/a.bc\n") - assert lines[1].endswith("/b.o\n") or lines[1].endswith("/b.bc\n") - assert lines[2].endswith("/c.o\n") or lines[2].endswith("/c.bc\n") - print("...with output!") + outputListFile = sys.argv[sys.argv.index('-output-filelist') + 1] + with open(outputListFile, 'r') as f: + lines = f.readlines() + assert lines[0].endswith("/a.o\n") or lines[0].endswith("/a.bc\n") + assert lines[1].endswith("/b.o\n") or lines[1].endswith("/b.bc\n") + assert lines[2].endswith("/c.o\n") or lines[2].endswith("/c.bc\n") + print("...with output!") diff --git a/test/Driver/Inputs/filelists/fake-ld.py b/test/Driver/Inputs/filelists/fake-ld.py index 92bf033bfeb60..d299b82072cf6 100755 --- a/test/Driver/Inputs/filelists/fake-ld.py +++ b/test/Driver/Inputs/filelists/fake-ld.py @@ -18,9 +18,9 @@ filelistFile = sys.argv[sys.argv.index('-filelist') + 1] with open(filelistFile, 'r') as f: - lines = f.readlines() - assert lines[0].endswith("/a.o\n") - assert lines[1].endswith("/b.o\n") - assert lines[2].endswith("/c.o\n") + lines = f.readlines() + assert lines[0].endswith("/a.o\n") + assert lines[1].endswith("/b.o\n") + assert lines[2].endswith("/c.o\n") print("Handled link") diff --git a/tools/SourceKit/bindings/python/sourcekitd/capi.py b/tools/SourceKit/bindings/python/sourcekitd/capi.py index 31cbfffbac47a..7e94b0dc1c02b 100644 --- a/tools/SourceKit/bindings/python/sourcekitd/capi.py +++ b/tools/SourceKit/bindings/python/sourcekitd/capi.py @@ -35,6 +35,7 @@ # Structures and Utility Classes + class CachedProperty(object): """Decorator that lazy-loads the value of a property. @@ -61,6 +62,7 @@ def __get__(self, instance, instance_type=None): class Object(object): + def __init__(self, obj): if isinstance(obj, Object): self._obj = conf.lib.sourcekitd_request_retain(obj) @@ -75,14 +77,16 @@ def __init__(self, obj): POINTER(c_void_p)(), POINTER(c_void_p)(), 0) self._as_parameter_ = self._obj for k, v in obj.iteritems(): - conf.lib.sourcekitd_request_dictionary_set_value(self, + conf.lib.sourcekitd_request_dictionary_set_value( + self, UIdent(k), Object(v)) elif isinstance(obj, (list, tuple)): self._obj = conf.lib.sourcekitd_request_array_create( POINTER(c_void_p)(), 0) self._as_parameter_ = self._obj for v in obj: - conf.lib.sourcekitd_request_array_set_value(self, -1, Object(v)) + conf.lib.sourcekitd_request_array_set_value( + self, -1, Object(v)) else: raise ValueError("wrong init parameter (%s)" % type(obj)) self._as_parameter_ = self._obj @@ -100,7 +104,9 @@ def __repr__(self): conf.free(ptr) return s + class Response(object): + def __init__(self, obj): if isinstance(obj, c_object_p): self._obj = self._as_parameter_ = obj @@ -123,7 +129,9 @@ def __repr__(self): conf.free(ptr) return s + class UIdent(object): + def __init__(self, obj): if isinstance(obj, c_object_p): self._obj = obj @@ -156,6 +164,7 @@ def __ne__(self, other): def __hash__(self): return hash(self._ptr()) + class ErrorKind(object): """Describes the kind of type.""" @@ -199,6 +208,7 @@ def __repr__(self): ErrorKind.REQUEST_FAILED = ErrorKind(3) ErrorKind.REQUEST_CANCELLED = ErrorKind(4) + class Variant(Structure): _fields_ = [ ("data", c_uint64 * 3)] @@ -227,7 +237,8 @@ def applier(index, value, arr): # continue return 1 arr = [] - conf.lib.sourcekitd_variant_array_apply_f(self, callbacks['array_applier'](applier), arr) + conf.lib.sourcekitd_variant_array_apply_f( + self, callbacks['array_applier'](applier), arr) return arr def to_python_dictionary(self): @@ -236,9 +247,11 @@ def applier(cobj, value, d): # continue return 1 d = {} - conf.lib.sourcekitd_variant_dictionary_apply_f(self, callbacks['dictionary_applier'](applier), d) + conf.lib.sourcekitd_variant_dictionary_apply_f( + self, callbacks['dictionary_applier'](applier), d) return d + class VariantType(object): """Describes the kind of type.""" @@ -248,7 +261,8 @@ class VariantType(object): def __init__(self, value): if value >= len(VariantType._kinds): - VariantType._kinds += [None] * (value - len(VariantType._kinds) + 1) + VariantType._kinds += [None] * \ + (value - len(VariantType._kinds) + 1) if VariantType._kinds[value] is not None: raise ValueError('VariantType already loaded') self.value = value @@ -289,228 +303,224 @@ def __repr__(self): # Register callback types in common container. callbacks['array_applier'] = CFUNCTYPE(c_int, c_size_t, Variant, py_object) -callbacks['dictionary_applier'] = CFUNCTYPE(c_int, c_object_p, Variant, py_object) +callbacks['dictionary_applier'] = CFUNCTYPE( + c_int, c_object_p, Variant, py_object) # Functions strictly alphabetical order. functionList = [ - # FIXME: Fix PEP8 violation "continuation line under-indented for hanging - # indent" (E121) and remove "noqa" marker. - ("sourcekitd_cancel_request", # noqa - [c_void_p]), - - ("sourcekitd_initialize", - None), - - ("sourcekitd_request_array_create", - [POINTER(c_object_p), c_size_t], - c_object_p), - - ("sourcekitd_request_array_set_int64", - [Object, c_size_t, c_int64]), + ("sourcekitd_cancel_request", + [c_void_p]), - ("sourcekitd_request_array_set_string", - [Object, c_size_t, c_char_p]), + ("sourcekitd_initialize", + None), - ("sourcekitd_request_array_set_stringbuf", - [Object, c_size_t, c_char_p, c_size_t]), + ("sourcekitd_request_array_create", + [POINTER(c_object_p), c_size_t], + c_object_p), - ("sourcekitd_request_array_set_uid", - [Object, c_size_t, UIdent]), + ("sourcekitd_request_array_set_int64", + [Object, c_size_t, c_int64]), - ("sourcekitd_request_array_set_value", - [Object, c_size_t, Object]), + ("sourcekitd_request_array_set_string", + [Object, c_size_t, c_char_p]), - ("sourcekitd_request_create_from_yaml", - [c_char_p, POINTER(c_char_p)], - c_object_p), + ("sourcekitd_request_array_set_stringbuf", + [Object, c_size_t, c_char_p, c_size_t]), - ("sourcekitd_request_description_copy", - [Object], - c_void_p), + ("sourcekitd_request_array_set_uid", + [Object, c_size_t, UIdent]), - ("sourcekitd_request_description_dump", - [Object]), + ("sourcekitd_request_array_set_value", + [Object, c_size_t, Object]), - ("sourcekitd_request_dictionary_create", - [POINTER(c_object_p), POINTER(c_object_p), c_size_t], - c_object_p), + ("sourcekitd_request_create_from_yaml", + [c_char_p, POINTER(c_char_p)], + c_object_p), - ("sourcekitd_request_dictionary_set_int64", - [Object, UIdent, c_int64]), + ("sourcekitd_request_description_copy", + [Object], + c_void_p), - ("sourcekitd_request_dictionary_set_string", - [Object, UIdent, c_char_p]), + ("sourcekitd_request_description_dump", + [Object]), - ("sourcekitd_request_dictionary_set_stringbuf", - [Object, UIdent, c_char_p, c_size_t]), + ("sourcekitd_request_dictionary_create", + [POINTER(c_object_p), POINTER(c_object_p), c_size_t], + c_object_p), - ("sourcekitd_request_dictionary_set_uid", - [Object, UIdent, UIdent]), + ("sourcekitd_request_dictionary_set_int64", + [Object, UIdent, c_int64]), - ("sourcekitd_request_dictionary_set_value", - [Object, UIdent, Object]), + ("sourcekitd_request_dictionary_set_string", + [Object, UIdent, c_char_p]), - ("sourcekitd_request_int64_create", - [c_int64], - c_object_p), + ("sourcekitd_request_dictionary_set_stringbuf", + [Object, UIdent, c_char_p, c_size_t]), - ("sourcekitd_request_retain", - [Object], - c_object_p), + ("sourcekitd_request_dictionary_set_uid", + [Object, UIdent, UIdent]), - ("sourcekitd_request_release", - [Object]), + ("sourcekitd_request_dictionary_set_value", + [Object, UIdent, Object]), - ("sourcekitd_request_string_create", - [c_char_p], - c_object_p), + ("sourcekitd_request_int64_create", + [c_int64], + c_object_p), - ("sourcekitd_request_uid_create", - [UIdent], - c_object_p), + ("sourcekitd_request_retain", + [Object], + c_object_p), - ("sourcekitd_response_description_copy", - [Response], - c_char_p), + ("sourcekitd_request_release", + [Object]), - ("sourcekitd_response_description_dump", - [Response]), + ("sourcekitd_request_string_create", + [c_char_p], + c_object_p), - ("sourcekitd_response_description_dump_filedesc", - [Response, c_int]), + ("sourcekitd_request_uid_create", + [UIdent], + c_object_p), - ("sourcekitd_response_dispose", - [Response]), + ("sourcekitd_response_description_copy", + [Response], + c_char_p), - ("sourcekitd_response_error_get_description", - [Response], - c_char_p), + ("sourcekitd_response_description_dump", + [Response]), - ("sourcekitd_response_error_get_kind", - [Response], - ErrorKind.from_id), + ("sourcekitd_response_description_dump_filedesc", + [Response, c_int]), - ("sourcekitd_response_get_value", - [Response], - Variant), + ("sourcekitd_response_dispose", + [Response]), - ("sourcekitd_response_is_error", - [Response], - c_bool), + ("sourcekitd_response_error_get_description", + [Response], + c_char_p), - # ("sourcekitd_send_request", + ("sourcekitd_response_error_get_kind", + [Response], + ErrorKind.from_id), - ("sourcekitd_send_request_sync", - [Object], - c_object_p), + ("sourcekitd_response_get_value", + [Response], + Variant), - # ("sourcekitd_set_interrupted_connection_handler", + ("sourcekitd_response_is_error", + [Response], + c_bool), - ("sourcekitd_shutdown", - None), + ("sourcekitd_send_request_sync", + [Object], + c_object_p), - ("sourcekitd_uid_get_from_buf", - [c_char_p, c_size_t], - c_object_p), + ("sourcekitd_shutdown", + None), - ("sourcekitd_uid_get_from_cstr", - [c_char_p], - c_object_p), + ("sourcekitd_uid_get_from_buf", + [c_char_p, c_size_t], + c_object_p), - ("sourcekitd_uid_get_length", - [UIdent], - c_size_t), + ("sourcekitd_uid_get_from_cstr", + [c_char_p], + c_object_p), - ("sourcekitd_uid_get_string_ptr", - [UIdent], - c_char_p), + ("sourcekitd_uid_get_length", + [UIdent], + c_size_t), - ("sourcekitd_variant_array_apply_f", - [Variant, callbacks['array_applier'], py_object], - c_bool), + ("sourcekitd_uid_get_string_ptr", + [UIdent], + c_char_p), + ("sourcekitd_variant_array_apply_f", + [Variant, callbacks['array_applier'], py_object], + c_bool), - ("sourcekitd_variant_array_get_bool", - [Variant, c_size_t], - c_bool), + ("sourcekitd_variant_array_get_bool", + [Variant, c_size_t], + c_bool), - ("sourcekitd_variant_array_get_count", - [Variant], - c_size_t), + ("sourcekitd_variant_array_get_count", + [Variant], + c_size_t), - ("sourcekitd_variant_array_get_int64", - [Variant, c_size_t], - c_int64), + ("sourcekitd_variant_array_get_int64", + [Variant, c_size_t], + c_int64), - ("sourcekitd_variant_array_get_string", - [Variant, c_size_t], - c_char_p), + ("sourcekitd_variant_array_get_string", + [Variant, c_size_t], + c_char_p), - ("sourcekitd_variant_array_get_uid", - [Variant, c_size_t], - c_object_p), + ("sourcekitd_variant_array_get_uid", + [Variant, c_size_t], + c_object_p), - ("sourcekitd_variant_array_get_value", - [Variant, c_size_t], - Variant), + ("sourcekitd_variant_array_get_value", + [Variant, c_size_t], + Variant), - ("sourcekitd_variant_bool_get_value", - [Variant], - c_bool), + ("sourcekitd_variant_bool_get_value", + [Variant], + c_bool), - ("sourcekitd_variant_dictionary_apply_f", - [Variant, callbacks['dictionary_applier'], py_object], - c_bool), + ("sourcekitd_variant_dictionary_apply_f", + [Variant, callbacks['dictionary_applier'], py_object], + c_bool), - ("sourcekitd_variant_dictionary_get_bool", - [Variant, UIdent], - c_bool), + ("sourcekitd_variant_dictionary_get_bool", + [Variant, UIdent], + c_bool), - ("sourcekitd_variant_dictionary_get_int64", - [Variant, UIdent], - c_int64), + ("sourcekitd_variant_dictionary_get_int64", + [Variant, UIdent], + c_int64), - ("sourcekitd_variant_dictionary_get_string", - [Variant, UIdent], - c_char_p), + ("sourcekitd_variant_dictionary_get_string", + [Variant, UIdent], + c_char_p), - ("sourcekitd_variant_dictionary_get_value", - [Variant, UIdent], - Variant), + ("sourcekitd_variant_dictionary_get_value", + [Variant, UIdent], + Variant), - ("sourcekitd_variant_dictionary_get_uid", - [Variant, UIdent], - c_object_p), + ("sourcekitd_variant_dictionary_get_uid", + [Variant, UIdent], + c_object_p), - ("sourcekitd_variant_get_type", - [Variant], - VariantType.from_id), + ("sourcekitd_variant_get_type", + [Variant], + VariantType.from_id), - ("sourcekitd_variant_string_get_length", - [Variant], - c_size_t), + ("sourcekitd_variant_string_get_length", + [Variant], + c_size_t), - ("sourcekitd_variant_string_get_ptr", - [Variant], - c_char_p), + ("sourcekitd_variant_string_get_ptr", + [Variant], + c_char_p), - ("sourcekitd_variant_int64_get_value", - [Variant], - c_int64), + ("sourcekitd_variant_int64_get_value", + [Variant], + c_int64), - ("sourcekitd_variant_uid_get_value", - [Variant], - c_object_p), + ("sourcekitd_variant_uid_get_value", + [Variant], + c_object_p), ] class LibsourcekitdError(Exception): + def __init__(self, message): self.m = message def __str__(self): return self.m + def register_function(lib, item, ignore_errors): # A function may not exist, if these bindings are used with an older or # incompatible version of sourcekitd. @@ -532,6 +542,7 @@ def register_function(lib, item, ignore_errors): if len(item) == 4: func.errcheck = item[3] + def register_functions(lib, ignore_errors): """Register function prototypes with a sourcekitd library instance. @@ -543,6 +554,7 @@ def register(item): map(register, functionList) + class Config: library_path = None library_file = None diff --git a/tools/SourceKit/bindings/python/sourcekitd/request.py b/tools/SourceKit/bindings/python/sourcekitd/request.py index 72030a75bdf44..b97d26be60111 100644 --- a/tools/SourceKit/bindings/python/sourcekitd/request.py +++ b/tools/SourceKit/bindings/python/sourcekitd/request.py @@ -10,15 +10,19 @@ import capi + def request_sync(req): ptr = capi.conf.lib.sourcekitd_send_request_sync(capi.Object(req)) resp = capi.Response(ptr) if capi.conf.lib.sourcekitd_response_is_error(resp): - raise SourceKitError(capi.conf.lib.sourcekitd_response_error_get_kind(resp), + raise SourceKitError( + capi.conf.lib.sourcekitd_response_error_get_kind(resp), capi.conf.lib.sourcekitd_response_error_get_description(resp)) return resp + class SourceKitError(Exception): + def __init__(self, kind, message): self.kind = kind self.msg = message @@ -26,6 +30,7 @@ def __init__(self, kind, message): def __str__(self): return "%s (%s)" % (self.msg, self.kind) + def syntax_annotate_text(text): req = { 'key.request': capi.UIdent('source.request.editor.open'), diff --git a/utils/GYBUnicodeDataUtils.py b/utils/GYBUnicodeDataUtils.py index 3cd40e0528560..8d4266b6da140 100644 --- a/utils/GYBUnicodeDataUtils.py +++ b/utils/GYBUnicodeDataUtils.py @@ -11,6 +11,7 @@ import re import codecs + class UnicodeProperty(object): """Abstract base class for Unicode properties.""" @@ -29,6 +30,7 @@ def to_numeric_value(self, value): def get_numeric_value(self, cp): raise NotImplemented + class GraphemeClusterBreakPropertyTable(UnicodeProperty): """Grapheme_Cluster_Break property.""" @@ -217,10 +219,12 @@ def get_bmp_data_offset(self, cp): return cp & ((1 << self.bmp_data_offset_bits) - 1) def get_supp_first_level_index(self, cp): - return cp >> (self.supp_second_level_index_bits + self.supp_data_offset_bits) + return cp >> \ + (self.supp_second_level_index_bits + self.supp_data_offset_bits) def get_supp_second_level_index(self, cp): - return (cp >> self.supp_data_offset_bits) & ((1 << self.supp_second_level_index_bits) - 1) + return (cp >> self.supp_data_offset_bits) & \ + ((1 << self.supp_second_level_index_bits) - 1) def get_supp_data_offset(self, cp): return cp & ((1 << self.supp_data_offset_bits) - 1) @@ -246,8 +250,8 @@ def create_tables(self): # maximum Unicode code point value is not 2^21-1 (0x1fffff), it is # 0x10ffff. self.supp_first_level_index_max = \ - 0x10ffff >> (self.supp_second_level_index_bits + - self.supp_data_offset_bits) + 0x10ffff >> \ + (self.supp_second_level_index_bits + self.supp_data_offset_bits) # A mapping from BMP first-level index to BMP data block index. self.bmp_lookup = [i for i in range(0, 1 << self.bmp_first_level_index_bits)] @@ -359,8 +363,7 @@ def map_index(idx): self.supp_data.pop(j) for k in range(0, len(self.supp_lookup2)): self.supp_lookup2[k] = \ - remap_indexes(self.supp_lookup2[k], old_idx=j, - new_idx=i) + remap_indexes(self.supp_lookup2[k], old_idx=j, new_idx=i) else: j += 1 i += 1 @@ -444,6 +447,7 @@ def serialize(self, unicode_property): self.supp_data_bytes_offset = len(self.trie_bytes) self.trie_bytes += supp_data_bytes + def get_extended_grapheme_cluster_rules_matrix(grapheme_cluster_break_property_table): any_value = \ grapheme_cluster_break_property_table.symbolic_values @@ -492,8 +496,7 @@ def get_extended_grapheme_cluster_rules_matrix(grapheme_cluster_break_property_t row = rules_matrix[first] # Change strings into bits. - bits = [row[second] == 'no_boundary' - for second in any_value] + bits = [row[second] == 'no_boundary' for second in any_value] # Pack bits into an integer. packed = sum([bits[i] * pow(2, i) for i in range(0, len(bits))]) @@ -502,6 +505,7 @@ def get_extended_grapheme_cluster_rules_matrix(grapheme_cluster_break_property_t return result + def get_grapheme_cluster_break_tests_as_utf8(grapheme_break_test_file_name): def _convert_line(line): # Strip comments. @@ -553,6 +557,7 @@ def _convert_line(line): return result + def get_grapheme_cluster_break_tests_as_unicode_scalars(grapheme_break_test_file_name): def _convert_line(line): # Strip comments. diff --git a/utils/SwiftBuildSupport.py b/utils/SwiftBuildSupport.py index d13f552ae4935..0b47385f66428 100644 --- a/utils/SwiftBuildSupport.py +++ b/utils/SwiftBuildSupport.py @@ -87,7 +87,8 @@ def check_call(args, print_command=False, verbose=False): sys.stdout.flush() sys.exit(1) except OSError as e: - print_with_argv0("could not execute '" + quote_shell_command(args) + + print_with_argv0( + "could not execute '" + quote_shell_command(args) + "': " + e.strerror) sys.stdout.flush() sys.exit(1) @@ -108,7 +109,8 @@ def check_output(args, print_command=False, verbose=False): sys.stdout.flush() sys.exit(1) except OSError as e: - print_with_argv0("could not execute '" + quote_shell_command(args) + + print_with_argv0( + "could not execute '" + quote_shell_command(args) + "': " + e.strerror) sys.stdout.flush() sys.exit(1) @@ -125,6 +127,7 @@ def _load_preset_files_impl(preset_file_names, substitutions={}): _PRESET_PREFIX = "preset: " + def _get_preset_options_impl(config, substitutions, preset_name): section_name = _PRESET_PREFIX + preset_name if section_name not in config.sections(): @@ -204,6 +207,7 @@ def get_all_preset_names(preset_file_names): # with WorkingDirectory('/tmp'): # ... do work in /tmp... class WorkingDirectory(object): + def __init__(self, new_cwd): self.new_cwd = new_cwd diff --git a/utils/SwiftIntTypes.py b/utils/SwiftIntTypes.py index 10ce4b62d131b..0b47ad332f464 100644 --- a/utils/SwiftIntTypes.py +++ b/utils/SwiftIntTypes.py @@ -14,7 +14,9 @@ # Number of bits in the biggest int type int_max_bits = max(_all_integer_type_bitwidths) + class SwiftIntegerType(object): + def __init__(self, is_word, bits, is_signed): self.is_word = is_word self.bits = bits @@ -44,18 +46,23 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) + def all_integer_types(word_bits): for bitwidth in _all_integer_type_bitwidths: for is_signed in [False, True]: - yield SwiftIntegerType(is_word=False, bits=bitwidth, + yield SwiftIntegerType( + is_word=False, bits=bitwidth, is_signed=is_signed) for is_signed in [False, True]: - yield SwiftIntegerType(is_word=True, bits=word_bits, + yield SwiftIntegerType( + is_word=True, bits=word_bits, is_signed=is_signed) # 'truncatingBitPattern' initializer is defined if the conversion is truncating # on any platform that Swift supports. + + def should_define_truncating_bit_pattern_init(src_ty, dst_ty): # Don't define a truncating conversion between a type and itself. if src_ty == dst_ty: @@ -72,32 +79,41 @@ def should_define_truncating_bit_pattern_init(src_ty, dst_ty): return False + def all_integer_type_names(): return [self_ty.stdlib_name for self_ty in all_integer_types(0)] + def all_real_number_type_names(): # FIXME , 'Float80' Revert until I figure out a test failure # Float80 for i386 & x86_64 return ['Float', 'Double'] + def all_numeric_type_names(): return all_integer_type_names() + all_real_number_type_names() + def numeric_type_names_macintosh_only(): return ['Float80'] # Swift_Programming_Language/Expressions.html + def all_integer_binary_operator_names(): return ['<<', '>>', '&*', '&', '&+', '&-', '|', '^'] + def all_integer_or_real_binary_operator_names(): return ['*', '/', '%', '+', '-', '..<', '...'] + def all_arithmetic_comparison_operator_names(): return ['<', '<=', '>', '>=', '==', '!='] + def all_integer_assignment_operator_names(): return ['<<=', '>>=', '&=', '^=', '|='] + def all_integer_or_real_assignment_operator_names(): return ['=', '*=', '/=', '%=', '+=', '-='] diff --git a/utils/apply-fixit-edits.py b/utils/apply-fixit-edits.py index 3efa7488b7a9d..9dd9f7052a07e 100755 --- a/utils/apply-fixit-edits.py +++ b/utils/apply-fixit-edits.py @@ -16,6 +16,7 @@ import sys import os + def find_remap_files(path): for root, dirs, files in os.walk(path): for filename in files: @@ -23,6 +24,7 @@ def find_remap_files(path): continue yield os.path.join(root, filename) + def apply_edits(path): remap_files = find_remap_files(path) if not remap_files: @@ -65,6 +67,7 @@ def apply_edits(path): f.write(file_data) return 0 + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, diff --git a/utils/build-script b/utils/build-script index 6dbbf530bf8f0..b4249d64f9b5d 100755 --- a/utils/build-script +++ b/utils/build-script @@ -51,22 +51,27 @@ def main_preset(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="""Builds Swift using a preset.""") - parser.add_argument("--preset-file", + parser.add_argument( + "--preset-file", help="load presets from the specified file", metavar="PATH", action="append", dest="preset_file_names", default=[]) - parser.add_argument("--preset", + parser.add_argument( + "--preset", help="use the specified option preset", metavar="NAME") - parser.add_argument("--show-presets", + parser.add_argument( + "--show-presets", help="list all presets and exit", action="store_true") - parser.add_argument("--distcc", + parser.add_argument( + "--distcc", help="use distcc", action="store_true") - parser.add_argument("preset_substitutions_raw", + parser.add_argument( + "preset_substitutions_raw", help="'name=value' pairs that are substituted in the preset", nargs="*", metavar="SUBSTITUTION") @@ -305,46 +310,56 @@ details of the setups of other systems or automated environments.""") projects_group = parser.add_argument_group( title="Options to select projects") - projects_group.add_argument("-l", "--lldb", + projects_group.add_argument( + "-l", "--lldb", help="build LLDB", action="store_true", dest="build_lldb") - projects_group.add_argument("-b", "--llbuild", + projects_group.add_argument( + "-b", "--llbuild", help="build llbuild", action="store_true", dest="build_llbuild") - projects_group.add_argument("-p", "--swiftpm", + projects_group.add_argument( + "-p", "--swiftpm", help="build swiftpm", action="store_true", dest="build_swiftpm") - projects_group.add_argument("--xctest", + projects_group.add_argument( + "--xctest", help="build xctest", action="store_true", dest="build_xctest") - projects_group.add_argument("--foundation", + projects_group.add_argument( + "--foundation", help="build foundation", action="store_true", dest="build_foundation") - projects_group.add_argument("--libdispatch", + projects_group.add_argument( + "--libdispatch", help="build libdispatch", action="store_true", dest="build_libdispatch") extra_actions_group = parser.add_argument_group( title="Extra actions to perform before or in addition to building") - extra_actions_group.add_argument("-c", "--clean", + extra_actions_group.add_argument( + "-c", "--clean", help="do a clean build", action="store_true") - extra_actions_group.add_argument("--export-compile-commands", + extra_actions_group.add_argument( + "--export-compile-commands", help="generate compilation databases in addition to building", action="store_true") - extra_actions_group.add_argument("--symbols-package", + extra_actions_group.add_argument( + "--symbols-package", metavar="PATH", help="if provided, an archive of the symbols directory will be " - "generated at this path") + "generated at this path") build_variant_group = parser.add_mutually_exclusive_group(required=False) - build_variant_group.add_argument("-d", "--debug", + build_variant_group.add_argument( + "-d", "--debug", help=""" build the Debug variant of everything (LLVM, Clang, Swift host tools, target Swift standard libraries, LLDB (if enabled) @@ -352,13 +367,15 @@ Swift standard libraries, LLDB (if enabled) action="store_const", const="Debug", dest="build_variant") - build_variant_group.add_argument("-r", "--release-debuginfo", + build_variant_group.add_argument( + "-r", "--release-debuginfo", help=""" build the RelWithDebInfo variant of everything (default is Debug)""", action="store_const", const="RelWithDebInfo", dest="build_variant") - build_variant_group.add_argument("-R", "--release", + build_variant_group.add_argument( + "-R", "--release", help="build the Release variant of everything (default is Debug)", action="store_const", const="Release", @@ -366,50 +383,59 @@ build the RelWithDebInfo variant of everything (default is Debug)""", build_variant_override_group = parser.add_argument_group( title="Override build variant for a specific project") - build_variant_override_group.add_argument("--debug-llvm", + build_variant_override_group.add_argument( + "--debug-llvm", help="build the Debug variant of LLVM", action="store_const", const="Debug", dest="llvm_build_variant") - build_variant_override_group.add_argument("--debug-swift", + build_variant_override_group.add_argument( + "--debug-swift", help="build the Debug variant of Swift host tools", action="store_const", const="Debug", dest="swift_build_variant") - build_variant_override_group.add_argument("--debug-swift-stdlib", + build_variant_override_group.add_argument( + "--debug-swift-stdlib", help=""" build the Debug variant of the Swift standard library and SDK overlay""", action="store_const", const="Debug", dest="swift_stdlib_build_variant") - build_variant_override_group.add_argument("--debug-lldb", + build_variant_override_group.add_argument( + "--debug-lldb", help="build the Debug variant of LLDB", action="store_const", const="Debug", dest="lldb_build_variant") - build_variant_override_group.add_argument("--debug-cmark", + build_variant_override_group.add_argument( + "--debug-cmark", help="build the Debug variant of CommonMark", action="store_const", const="Debug", dest="cmark_build_variant") - build_variant_override_group.add_argument("--debug-foundation", + build_variant_override_group.add_argument( + "--debug-foundation", help="build the Debug variant of Foundation", action="store_const", const="Debug", dest="foundation_build_variant") - build_variant_override_group.add_argument("--debug-libdispatch", + build_variant_override_group.add_argument( + "--debug-libdispatch", help="build the Debug variant of libdispatch", action="store_const", const="Debug", dest="libdispatch_build_variant") assertions_group = parser.add_mutually_exclusive_group(required=False) - assertions_group.add_argument("--assertions", + assertions_group.add_argument( + "--assertions", help="enable assertions in all projects", action="store_const", const=True, dest="assertions") - assertions_group.add_argument("--no-assertions", + assertions_group.add_argument( + "--no-assertions", help="disable assertions in all projects", action="store_const", const=False, @@ -417,47 +443,56 @@ build the Debug variant of the Swift standard library and SDK overlay""", assertions_override_group = parser.add_argument_group( title="Control assertions in a specific project") - assertions_override_group.add_argument("--cmark-assertions", + assertions_override_group.add_argument( + "--cmark-assertions", help="enable assertions in CommonMark", action="store_const", const=True, dest="cmark_assertions") - assertions_override_group.add_argument("--llvm-assertions", + assertions_override_group.add_argument( + "--llvm-assertions", help="enable assertions in LLVM", action="store_const", const=True, dest="llvm_assertions") - assertions_override_group.add_argument("--no-llvm-assertions", + assertions_override_group.add_argument( + "--no-llvm-assertions", help="disable assertions in LLVM", action="store_const", const=False, dest="llvm_assertions") - assertions_override_group.add_argument("--swift-assertions", + assertions_override_group.add_argument( + "--swift-assertions", help="enable assertions in Swift", action="store_const", const=True, dest="swift_assertions") - assertions_override_group.add_argument("--no-swift-assertions", + assertions_override_group.add_argument( + "--no-swift-assertions", help="disable assertions in Swift", action="store_const", const=False, dest="swift_assertions") - assertions_override_group.add_argument("--swift-stdlib-assertions", + assertions_override_group.add_argument( + "--swift-stdlib-assertions", help="enable assertions in the Swift standard library", action="store_const", const=True, dest="swift_stdlib_assertions") - assertions_override_group.add_argument("--no-swift-stdlib-assertions", + assertions_override_group.add_argument( + "--no-swift-stdlib-assertions", help="disable assertions in the Swift standard library", action="store_const", const=False, dest="swift_stdlib_assertions") - assertions_override_group.add_argument("--lldb-assertions", + assertions_override_group.add_argument( + "--lldb-assertions", help="enable assertions in LLDB", action="store_const", const=True, dest="lldb_assertions") - assertions_override_group.add_argument("--no-lldb-assertions", + assertions_override_group.add_argument( + "--no-lldb-assertions", help="disable assertions in LLDB", action="store_const", const=False, @@ -465,22 +500,26 @@ build the Debug variant of the Swift standard library and SDK overlay""", cmake_generator_group = parser.add_argument_group( title="Select the CMake generator") - cmake_generator_group.add_argument("-x", "--xcode", + cmake_generator_group.add_argument( + "-x", "--xcode", help="use CMake's Xcode generator (default is Ninja)", action="store_const", const="Xcode", dest="cmake_generator") - cmake_generator_group.add_argument("-X", "--xcode-ide-only", + cmake_generator_group.add_argument( + "-X", "--xcode-ide-only", help="use CMake's Xcode generator, with only IDE support (no build)", action="store_const", const="XcodeIDEOnly", dest="cmake_generator") - cmake_generator_group.add_argument("-m", "--make", + cmake_generator_group.add_argument( + "-m", "--make", help="use CMake's Makefile generator (default is Ninja)", action="store_const", const="Unix Makefiles", dest="cmake_generator") - cmake_generator_group.add_argument("-e", "--eclipse", + cmake_generator_group.add_argument( + "-e", "--eclipse", help="use CMake's Eclipse generator (default is Ninja)", action="store_const", const="Eclipse CDT4 - Ninja", @@ -488,147 +527,182 @@ build the Debug variant of the Swift standard library and SDK overlay""", run_tests_group = parser.add_argument_group( title="Run tests") - run_tests_group.add_argument("-t", "--test", + run_tests_group.add_argument( + "-t", "--test", help="test Swift after building", action="store_true") - run_tests_group.add_argument("-T", "--validation-test", + run_tests_group.add_argument( + "-T", "--validation-test", help="run the validation test suite (implies --test)", action="store_true") - run_tests_group.add_argument("--host-test", + run_tests_group.add_argument( + "--host-test", help="run executable tests on host devices (such as iOS or tvOS)", action="store_true") - run_tests_group.add_argument("-B", "--benchmark", + run_tests_group.add_argument( + "-B", "--benchmark", help="run the Swift Benchmark Suite after building", action="store_true") - run_tests_group.add_argument("--skip-test-linux", + run_tests_group.add_argument( + "--skip-test-linux", help="skip testing Swift stdlibs for Linux", action="store_true") - run_tests_group.add_argument("--skip-test-freebsd", + run_tests_group.add_argument( + "--skip-test-freebsd", help="skip testing Swift stdlibs for FreeBSD", action="store_true") - run_tests_group.add_argument("--skip-test-cygwin", + run_tests_group.add_argument( + "--skip-test-cygwin", help="skip testing Swift stdlibs for Cygwin", action="store_true") - parser.add_argument("-o", "--test-optimized", + parser.add_argument( + "-o", "--test-optimized", help="run the test suite in optimized mode too (implies --test)", action="store_true") run_build_group = parser.add_argument_group( title="Run build") - run_build_group.add_argument("-S", "--skip-build", + run_build_group.add_argument( + "-S", "--skip-build", help="generate build directory only without building", action="store_true") - run_build_group.add_argument("--skip-build-linux", + run_build_group.add_argument( + "--skip-build-linux", help="skip building Swift stdlibs for Linux", action="store_true") - run_build_group.add_argument("--skip-build-freebsd", + run_build_group.add_argument( + "--skip-build-freebsd", help="skip building Swift stdlibs for FreeBSD", action="store_true") - run_build_group.add_argument("--skip-build-cygwin", + run_build_group.add_argument( + "--skip-build-cygwin", help="skip building Swift stdlibs for Cygwin", action="store_true") - run_build_group.add_argument("--skip-build-benchmarks", + run_build_group.add_argument( + "--skip-build-benchmarks", help="skip building Swift Benchmark Suite", action="store_true") skip_test_group = parser.add_argument_group( title="Skip testing specified targets") - skip_test_group.add_argument("--skip-test-ios", + skip_test_group.add_argument( + "--skip-test-ios", help="skip testing all iOS targets. Equivalent to specifying both " - "--skip-test-ios-simulator and --skip-test-ios-host", + "--skip-test-ios-simulator and --skip-test-ios-host", action="store_true") - skip_test_group.add_argument("--skip-test-ios-simulator", + skip_test_group.add_argument( + "--skip-test-ios-simulator", help="skip testing iOS simulator targets", action="store_true") - skip_test_group.add_argument("--skip-test-ios-host", + skip_test_group.add_argument( + "--skip-test-ios-host", help="skip testing iOS device targets on the host machine (the phone " - "itself)", + "itself)", action="store_true") - skip_test_group.add_argument("--skip-test-tvos", + skip_test_group.add_argument( + "--skip-test-tvos", help="skip testing all tvOS targets. Equivalent to specifying both " - "--skip-test-tvos-simulator and --skip-test-tvos-host", + "--skip-test-tvos-simulator and --skip-test-tvos-host", action="store_true") - skip_test_group.add_argument("--skip-test-tvos-simulator", + skip_test_group.add_argument( + "--skip-test-tvos-simulator", help="skip testing tvOS simulator targets", action="store_true") - skip_test_group.add_argument("--skip-test-tvos-host", + skip_test_group.add_argument( + "--skip-test-tvos-host", help="skip testing tvOS device targets on the host machine (the TV " - "itself)", + "itself)", action="store_true") - skip_test_group.add_argument("--skip-test-watchos", + skip_test_group.add_argument( + "--skip-test-watchos", help="skip testing all tvOS targets. Equivalent to specifying both " - "--skip-test-watchos-simulator and --skip-test-watchos-host", + "--skip-test-watchos-simulator and --skip-test-watchos-host", action="store_true") - skip_test_group.add_argument("--skip-test-watchos-simulator", + skip_test_group.add_argument( + "--skip-test-watchos-simulator", help="skip testing watchOS simulator targets", action="store_true") - skip_test_group.add_argument("--skip-test-watchos-host", + skip_test_group.add_argument( + "--skip-test-watchos-host", help="skip testing watchOS device targets on the host machine (the " - "watch itself)", + "watch itself)", action="store_true") - parser.add_argument("-i", "--ios", + parser.add_argument( + "-i", "--ios", help=""" also build for iOS, but disallow tests that require an iOS device""", action="store_true") - parser.add_argument("--tvos", + parser.add_argument( + "--tvos", help=""" also build for tvOS, but disallow tests that require a tvos device""", action="store_true") - parser.add_argument("--watchos", + parser.add_argument( + "--watchos", help=""" -also build for Apple watchos, but disallow tests that require an watchOS device""", +also build for Apple watchOS, but disallow tests that require an watchOS device""", action="store_true") - parser.add_argument("--swift-analyze-code-coverage", + parser.add_argument( + "--swift-analyze-code-coverage", help="""enable code coverage analysis in Swift (false, not-merged, merged).""", choices=["false", "not-merged", "merged"], default="false", # so CMake can see the inert mode as a false value dest="swift_analyze_code_coverage") - parser.add_argument("--build-subdir", + parser.add_argument( + "--build-subdir", help=""" name of the directory under $SWIFT_BUILD_ROOT where the build products will be placed""", metavar="PATH") - parser.add_argument("--install-prefix", + parser.add_argument( + "--install-prefix", help="The installation prefix. This is where built Swift products " - "(like bin, lib, and include) will be installed.", + "(like bin, lib, and include) will be installed.", metavar="PATH", default=swift_build_support.targets.install_prefix()) - parser.add_argument("--install-symroot", + parser.add_argument( + "--install-symroot", help="the path to install debug symbols into", metavar="PATH") - parser.add_argument("-j", "--jobs", + parser.add_argument( + "-j", "--jobs", help=""" the number of parallel build jobs to use""", type=int, dest="build_jobs", default=multiprocessing.cpu_count()) - parser.add_argument("--darwin-xcrun-toolchain", - help="the name of the toolchain to use on Darwin", - default="default") - parser.add_argument("--cmake", - help="the path to a CMake executable that will be " - "used to build Swift") - parser.add_argument("--show-sdks", - help="print installed Xcode and SDK versions", - action="store_true") - - parser.add_argument("--extra-swift-args", help=textwrap.dedent(""" + parser.add_argument( + "--darwin-xcrun-toolchain", + help="the name of the toolchain to use on Darwin", + default="default") + parser.add_argument( + "--cmake", + help="the path to a CMake executable that will be " + "used to build Swift") + parser.add_argument( + "--show-sdks", + help="print installed Xcode and SDK versions", + action="store_true") + + parser.add_argument( + "--extra-swift-args", help=textwrap.dedent(""" Pass through extra flags to swift in the form of a cmake list 'module_regexp;flag'. Can be called multiple times to add multiple such module_regexp flag pairs. All semicolons in flags must be escaped with a '\'"""), - action="append", dest="extra_swift_args", default=[]) + action="append", dest="extra_swift_args", default=[]) - parser.add_argument("build_script_impl_args", + parser.add_argument( + "build_script_impl_args", help="", nargs="*") @@ -658,12 +732,14 @@ the number of parallel build jobs to use""", if args.symbols_package: if not os.path.isabs(args.symbols_package): - print('--symbols-package must be an absolute path ' - '(was \'{}\')'.format(args.symbols_package)) + print( + '--symbols-package must be an absolute path ' + '(was \'{}\')'.format(args.symbols_package)) return 1 if not args.install_symroot: - print_with_argv0("--install-symroot is required when specifying " - "--symbols-package.") + print_with_argv0( + "--install-symroot is required when specifying " + "--symbols-package.") return 1 # Build cmark if any cmark-related options were specified. @@ -885,7 +961,8 @@ the number of parallel build jobs to use""", if platform.system() == 'Darwin': build_script_impl_inferred_args += [ "--toolchain-prefix", - swift_build_support.targets.darwin_toolchain_prefix(args.install_prefix), + swift_build_support.targets.darwin_toolchain_prefix( + args.install_prefix), ] if args.build_subdir is None: @@ -913,7 +990,8 @@ the number of parallel build jobs to use""", if (llvm_build_dir_label == swift_build_dir_label and llvm_build_dir_label == swift_stdlib_build_dir_label and llvm_build_dir_label == cmark_build_dir_label): - # Use a simple directory name if all projects use the same build type. + # Use a simple directory name if all projects use the same build + # type. args.build_subdir += "-" + llvm_build_dir_label elif (llvm_build_dir_label != swift_build_dir_label and llvm_build_dir_label == swift_stdlib_build_dir_label and @@ -976,8 +1054,10 @@ the number of parallel build jobs to use""", "--lldb-build-type", args.lldb_build_variant, "--llvm-enable-assertions", str(args.llvm_assertions).lower(), "--swift-enable-assertions", str(args.swift_assertions).lower(), - "--swift-stdlib-enable-assertions", str(args.swift_stdlib_assertions).lower(), - "--swift-analyze-code-coverage", str(args.swift_analyze_code_coverage).lower(), + "--swift-stdlib-enable-assertions", str( + args.swift_stdlib_assertions).lower(), + "--swift-analyze-code-coverage", str( + args.swift_analyze_code_coverage).lower(), "--cmake-generator", args.cmake_generator, "--build-jobs", str(args.build_jobs), "--workspace", SWIFT_SOURCE_ROOT @@ -1022,7 +1102,9 @@ the number of parallel build jobs to use""", # If we have extra_swift_args, combine all of them together and then add # them as one command. if args.extra_swift_args: - build_script_impl_args += ["--extra-swift-args", ";".join(args.extra_swift_args)] + build_script_impl_args += [ + "--extra-swift-args", + ";".join(args.extra_swift_args)] build_script_impl_args += args.build_script_impl_args @@ -1062,12 +1144,14 @@ the number of parallel build jobs to use""", def main(): if not SWIFT_SOURCE_ROOT: - print_with_argv0("Could not infer source root directory. " + + print_with_argv0( + "Could not infer source root directory. " + "Forgot to set $SWIFT_SOURCE_ROOT environment variable?") return 1 if not os.path.isdir(SWIFT_SOURCE_ROOT): - print_with_argv0("Source root directory \'" + SWIFT_SOURCE_ROOT + + print_with_argv0( + "Source root directory \'" + SWIFT_SOURCE_ROOT + "\' does not exist. " + "Forgot to set $SWIFT_SOURCE_ROOT environment variable?") return 1 diff --git a/utils/cmpcodesize/cmpcodesize/compare.py b/utils/cmpcodesize/cmpcodesize/compare.py index e7f0d7a7ffafb..a1e99cea87be8 100644 --- a/utils/cmpcodesize/cmpcodesize/compare.py +++ b/utils/cmpcodesize/cmpcodesize/compare.py @@ -60,9 +60,9 @@ def add_function(sizes, function, start_addr, end_addr, group_by_prefix): if group_by_prefix: for infix in sorted_infixes: if infix in function: - if generic_function_prefix not in function: - sizes[infixes[infix]] += size - return + if generic_function_prefix not in function: + sizes[infixes[infix]] += size + return for prefix in sorted_prefixes: if function.startswith(prefix): # Special handling for function signature specializations @@ -99,9 +99,9 @@ def read_sizes(sizes, file_name, function_details, group_by_prefix): if "arm64" in arch: arch = "arm64" if arch is not None: - arch_params = ["-arch", arch] + arch_params = ["-arch", arch] else: - arch_params = [] + arch_params = [] if function_details: content = subprocess.check_output(flatten(["otool", arch_params, "-l", "-v", "-t", file_name])).split("\n") diff --git a/utils/gyb.py b/utils/gyb.py index 5194b16390cd8..ad41552a27be5 100755 --- a/utils/gyb.py +++ b/utils/gyb.py @@ -14,6 +14,7 @@ from bisect import bisect import os + def get_line_starts(s): """Return a list containing the start index of each line in s. @@ -29,10 +30,12 @@ def get_line_starts(s): starts[-1] -= 1 return starts + def strip_trailing_nl(s): """If s ends with a newline, drop it; else return s intact""" return s[:-1] if s.endswith('\n') else s + def split_lines(s): """Split s into a list of lines, each of which has a trailing newline @@ -90,6 +93,7 @@ def split_lines(s): gyb_block_close = re.compile('\}%[ \t]*\n?') + def token_pos_to_index(token_pos, start, line_starts): """Translate a tokenize (line, column) pair into an absolute position in source text given the position where we started @@ -115,6 +119,7 @@ def token_pos_to_index(token_pos, start, line_starts): return line_starts[abs_token_line] + token_col + def tokenize_python_to_unmatched_close_curly(source_text, start, line_starts): """Apply Python's tokenize to source_text starting at index start while matching open and close curly braces. When an unmatched @@ -143,6 +148,7 @@ def tokenize_python_to_unmatched_close_curly(source_text, start, line_starts): return len(source_text) + def tokenize_template(template_text): r"""Given the text of a template, returns an iterator over (tokenType,token,match) tuples. @@ -223,7 +229,8 @@ def tokenize_template(template_text): while pos < end: m = tokenize_re.match(template_text, pos, end) - # pull out the one matched key (ignoring internal patterns starting with _) + # pull out the one matched key (ignoring internal patterns starting + # with _) ((kind, text), ) = ( (kind, text) for (kind, text) in m.groupdict().items() if text is not None and kind[0] != '_') @@ -255,6 +262,7 @@ def tokenize_template(template_text): if saved_literal != []: yield 'literal', ''.join(saved_literal), literal_first_match + def split_gyb_lines(source_lines): r"""Return a list of lines at which to split the incoming source @@ -338,6 +346,7 @@ def split_gyb_lines(source_lines): return unmatched_indents + def code_starts_with_dedent_keyword(source_lines): r"""Return True iff the incoming Python source_lines begin with "else", "elif", "except", or "finally". @@ -360,6 +369,7 @@ def code_starts_with_dedent_keyword(source_lines): return token_text in ('else', 'elif', 'except', 'finally') + class ParseContext: """State carried through a parse of a template""" @@ -491,7 +501,8 @@ def token_generator(self, base_tokens): elif kind == 'gybLines': - self.code_start_line = self.pos_to_line(self.token_match.start('gybLines')) + self.code_start_line = self.pos_to_line( + self.token_match.start('gybLines')) indentation = self.token_match.group('_indent') # Strip off the leading indentation and %-sign @@ -526,6 +537,7 @@ def next_token(self): self.token_kind = None + class ExecutionContext: """State we pass around during execution of a template""" @@ -543,12 +555,15 @@ def append_text(self, text, file, line): # We can only insert the line directive at a line break if len(self.result_text) == 0 \ or self.result_text[-1].endswith('\n'): - self.result_text.append('%s %d "%s"\n' % (self.line_directive, line + 1, file)) - # But if the new text contains any line breaks, we can create one + self.result_text.append('%s %d "%s"\n' % ( + self.line_directive, line + 1, file)) + # But if the new text contains any line breaks, we can create + # one elif '\n' in text: i = text.find('\n') self.result_text.append(text[:i + 1]) - self.last_file_line = (self.last_file_line[0], self.last_file_line[1] + 1) + self.last_file_line = ( + self.last_file_line[0], self.last_file_line[1] + 1) # and try again self.append_text(text[i + 1:], file, line) return @@ -556,6 +571,7 @@ def append_text(self, text, file, line): self.result_text.append(text) self.last_file_line = (file, line + text.count('\n')) + class ASTNode(object): """Abstract base class for template AST nodes""" @@ -600,6 +616,7 @@ def execute(self, context): def __str__(self, indent=''): return indent + 'Block:' + self.format_children(indent) + class Literal(ASTNode): """An AST node that generates literal text""" @@ -617,6 +634,7 @@ def __str__(self, indent=''): return '\n'.join( [indent + x for x in ['Literal:'] + strip_trailing_nl(self.text).split('\n')]) + class Code(ASTNode): """An AST node that is evaluated as Python""" @@ -632,7 +650,8 @@ def __init__(self, context): def accumulate_code(): s = source + (context.code_start_line - source_line_count) * '\n' \ + textwrap.dedent(context.code_text) - line_count = context.code_start_line + context.code_text.count('\n') + line_count = context.code_start_line + \ + context.code_text.count('\n') context.next_token() return s, line_count @@ -645,7 +664,8 @@ def accumulate_code(): else: while context.token_kind == 'gybLinesOpen': source, source_line_count = accumulate_code() - source += ' __children__[%d].execute(__context__)\n' % len(self.children) + source += ' __children__[%d].execute(__context__)\n' % len( + self.children) source_line_count += 1 self.children += (Block(context),) @@ -683,10 +703,12 @@ def execute(self, context): # If we got a result, the code was an expression, so append # its value if result is not None and result != '': - context.append_text(str(result), self.filename, self.start_line_number) + context.append_text( + str(result), self.filename, self.start_line_number) def __str__(self, indent=''): - source_lines = re.sub(r'^\n', '', strip_trailing_nl(self.source), flags=re.MULTILINE).split('\n') + source_lines = re.sub(r'^\n', '', strip_trailing_nl( + self.source), flags=re.MULTILINE).split('\n') if len(source_lines) == 1: s = indent + 'Code: {' + source_lines[0] + '}' else: @@ -695,6 +717,7 @@ def __str__(self, indent=''): ) + '\n' + indent + '}' return s + self.format_children(indent) + def parse_template(filename, text=None): r"""Return an AST corresponding to the given template file. @@ -941,6 +964,7 @@ def parse_template(filename, text=None): """ return Block(ParseContext(filename, text)) + def execute_template(ast, line_directive='', **local_bindings): r"""Return the text generated by executing the given template AST. @@ -979,10 +1003,12 @@ def execute_template(ast, line_directive='', **local_bindings): //#setline 6 "/dummy.file" [0, 1, 2] """ - execution_context = ExecutionContext(line_directive=line_directive, **local_bindings) + execution_context = ExecutionContext( + line_directive=line_directive, **local_bindings) ast.execute(execution_context) return ''.join(execution_context.result_text) + def main(): import argparse import sys @@ -1050,16 +1076,30 @@ def succ(a): - The End. - ''' ) - parser.add_argument('-D', action='append', dest='defines', metavar='NAME=VALUE', - default=[], - help='''Bindings to be set in the template's execution context''') - - parser.add_argument('file', type=argparse.FileType(), help='Path to GYB template file (defaults to stdin)', nargs='?', default=sys.stdin) - parser.add_argument('-o', dest='target', type=argparse.FileType('w'), help='Output file (defaults to stdout)', default=sys.stdout) - parser.add_argument('--test', action='store_true', default=False, help='Run a self-test') - parser.add_argument('--verbose-test', action='store_true', default=False, help='Run a verbose self-test') - parser.add_argument('--dump', action='store_true', default=False, help='Dump the parsed template to stdout') - parser.add_argument('--line-directive', default='// ###setline', help='Line directive prefix; empty => no line markers') + parser.add_argument( + '-D', action='append', dest='defines', metavar='NAME=VALUE', + default=[], + help='''Bindings to be set in the template's execution context''') + + parser.add_argument( + 'file', type=argparse.FileType(), + help='Path to GYB template file (defaults to stdin)', nargs='?', + default=sys.stdin) + parser.add_argument( + '-o', dest='target', type=argparse.FileType('w'), + help='Output file (defaults to stdout)', default=sys.stdout) + parser.add_argument( + '--test', action='store_true', + default=False, help='Run a self-test') + parser.add_argument( + '--verbose-test', action='store_true', + default=False, help='Run a verbose self-test') + parser.add_argument( + '--dump', action='store_true', + default=False, help='Dump the parsed template to stdout') + parser.add_argument( + '--line-directive', default='// ###setline', + help='Line directive prefix; empty => no line markers') args = parser.parse_args(sys.argv[1:]) diff --git a/utils/line-directive b/utils/line-directive index a7a9db7e2a2da..5f013f6ca6ddd 100755 --- a/utils/line-directive +++ b/utils/line-directive @@ -23,6 +23,7 @@ import subprocess line_pattern = re.compile(r'^// ###setline ([0-9]+) "([^"]+)"\s*') + def _make_line_map(filename, stream=None): """ >>> from StringIO import StringIO @@ -47,6 +48,7 @@ def _make_line_map(filename, stream=None): _line_maps = {} + def fline_map(filename): map = _line_maps.get(filename) if map is None: @@ -54,6 +56,7 @@ def fline_map(filename): _line_maps[filename] = map return map + def map_line(filename, line_num): assert(line_num > 0) map = fline_map(filename) @@ -61,6 +64,7 @@ def map_line(filename, line_num): base = map[index - 1] return base[1], base[2] + (line_num - base[0] - 1) + def run(): if len(sys.argv) <= 1: import doctest diff --git a/utils/pass-pipeline/scripts/pipelines_build_script.py b/utils/pass-pipeline/scripts/pipelines_build_script.py index ae1efc29af070..2873b11391690 100755 --- a/utils/pass-pipeline/scripts/pipelines_build_script.py +++ b/utils/pass-pipeline/scripts/pipelines_build_script.py @@ -16,6 +16,7 @@ PASSES = [p.name for p in passes.PASSES] DEFAULT_PRESENTS = "--preset=buildbot_incremental_extra_swift_args,tools=RA,stdlib=RD" + def run_build_script_with_data_file(build_script, data_file, verbose=False): build_script_args = [build_script, DEFAULT_PRESENTS, 'extra_swift_args=^Swift$;-Xfrontend\;-external-pass-pipeline-filename\;-Xfrontend\;%s' % data_file] sys.stdout.write("Running build script with: %s..." % ' '.join(build_script_args)) @@ -39,6 +40,7 @@ def run_build_script_with_data_file(build_script, data_file, verbose=False): else: sys.stdout.write(" Failure:\n") + def build_disable_slice_pipelines(**kwargs): pipeline_range = range(len(PIPELINES)) @@ -55,6 +57,7 @@ def get_pipeline_args(script, iter): f.write(subprocess.check_output(pipeline_args)) run_build_script_with_data_file(kwargs['build_script'], data_file, verbose=kwargs['verbose']) + def build_disable_individual_pass(**kwargs): pass_name = kwargs['pass_name'] data_file = os.path.join(kwargs['output_dir'], "%s-disabled-pass.json" % pass_name) @@ -62,12 +65,14 @@ def build_disable_individual_pass(**kwargs): f.write(subprocess.check_output([kwargs['pipeline_script'], '--disable-pass', pass_name])) run_build_script_with_data_file(kwargs['build_script'], data_file, verbose=kwargs['verbose']) + def build_disable_individual_passes(**kwargs): for p in PASSES: d = dict(kwargs) d['pass_name'] = p build_disable_individual_pass(**d) + def add_default_parser_args(p): p.add_argument('pipeline_script', help=textwrap.dedent(""" The path to normal_pipeline.py. In the future could be generalized to take other files. diff --git a/utils/pass-pipeline/src/pass_pipeline.py b/utils/pass-pipeline/src/pass_pipeline.py index 12cddc58f789f..0204c6422c81d 100644 --- a/utils/pass-pipeline/src/pass_pipeline.py +++ b/utils/pass-pipeline/src/pass_pipeline.py @@ -1,4 +1,5 @@ class Pass(object): + def __init__(self, name): self.name = name @@ -9,6 +10,7 @@ def __repr__(self): class PassList(object): + def __init__(self, transforms): global PassListId self.pass_id = PassListId @@ -39,7 +41,9 @@ def generate(self): stack.append(child) return result + class PassPipeline(object): + def __init__(self, identifier, action): self.identifier = identifier self.action = action diff --git a/utils/pass-pipeline/src/pass_pipeline_library.py b/utils/pass-pipeline/src/pass_pipeline_library.py index b5eec16db38a3..43036d7671ba8 100644 --- a/utils/pass-pipeline/src/pass_pipeline_library.py +++ b/utils/pass-pipeline/src/pass_pipeline_library.py @@ -2,6 +2,7 @@ import pass_pipeline as ppipe import passes as p + def diagnostic_passlist(): return ppipe.PassList([ p.CapturePromotion, @@ -16,6 +17,7 @@ def diagnostic_passlist(): p.SplitNonCondBrCriticalEdges, ]) + def simplifycfg_silcombine_passlist(): return ppipe.PassList([ p.SimplifyCFG, @@ -23,6 +25,7 @@ def simplifycfg_silcombine_passlist(): p.SimplifyCFG, ]) + def highlevel_loopopt_passlist(): return ppipe.PassList([ p.LowerAggregateInstrs, @@ -44,6 +47,7 @@ def highlevel_loopopt_passlist(): p.SwiftArrayOpts, ]) + def lowlevel_loopopt_passlist(): return ppipe.PassList([ p.LICM, @@ -53,6 +57,7 @@ def lowlevel_loopopt_passlist(): p.SimplifyCFG, ]) + def inliner_for_optlevel(optlevel): if optlevel == 'high': return p.EarlyInliner @@ -63,6 +68,7 @@ def inliner_for_optlevel(optlevel): else: raise RuntimeError('Unknown opt level') + def ssapass_passlist(optlevel): return ppipe.PassList([ simplifycfg_silcombine_passlist(), @@ -89,6 +95,7 @@ def ssapass_passlist(optlevel): p.GlobalARCOpts, ]) + def lower_passlist(): return ppipe.PassList([ p.DeadFunctionElimination, @@ -100,6 +107,7 @@ def lower_passlist(): p.FunctionSignatureOpts, ]) + def normal_passpipelines(): result = [] diff --git a/utils/protocol_graph.py b/utils/protocol_graph.py index b869a4b119bc8..e09f2929af9bd 100644 --- a/utils/protocol_graph.py +++ b/utils/protocol_graph.py @@ -50,6 +50,8 @@ ''' # substitute local variables into the string + + def interpolate(string): import inspect frame = inspect.currentframe() @@ -57,6 +59,8 @@ def interpolate(string): # Given the body_text of a protocol definition, return a list of # associated type and operator requirements. + + def body_lines(body_text): return [ cgi.escape(b.group(0)) for b in @@ -82,6 +86,7 @@ def body_lines(body_text): generic_parameter_constraint = interpolate(r' (%(identifier)s) \s* : \s* (%(identifier)s) ') + def parse_generic_operator(m): generic_params = m.group(5) generic_operator = cgi.escape(m.group(0).strip()) @@ -104,6 +109,7 @@ def parse_generic_operator(m): generic_operators.setdefault(protocol, set()).add(abbreviated_signature) + def parse_protocol(m): child = m.group(1) # skip irrelevant protocols diff --git a/utils/recursive-lipo b/utils/recursive-lipo index feac8d9cf5be3..52313ec296c9e 100755 --- a/utils/recursive-lipo +++ b/utils/recursive-lipo @@ -10,6 +10,7 @@ import shutil from SwiftBuildSupport import check_call + def merge_file_lists(src_root_dirs, skip_files, skip_subpaths): """Merges the file lists recursively from all src_root_dirs supplied, returning the union of all file paths found. @@ -20,12 +21,13 @@ def merge_file_lists(src_root_dirs, skip_files, skip_subpaths): for src_root_dir in src_root_dirs: for src_dir, dirs, files in os.walk(src_root_dir): rel_dir = os.path.relpath(src_dir, src_root_dir) - rel_files = [os.path.join(rel_dir, file) for file in files + dirs - if file not in skip_files] - file_list.extend(filter(lambda file: - file not in file_list, rel_files)) - dirs[:] = filter(lambda dir: - os.path.join(rel_dir, dir) not in skip_subpaths, dirs) + rel_files = [ + os.path.join(rel_dir, file) for file in files + dirs + if file not in skip_files] + file_list.extend( + filter(lambda file: file not in file_list, rel_files)) + dirs[:] = filter( + lambda dir: os.path.join(rel_dir, dir) not in skip_subpaths, dirs) return file_list @@ -46,14 +48,15 @@ def merge_lipo_files(src_root_dirs, file_list, copy_verbatim_subpaths, lipo_cmd = [lipo_executable, "-create"] for file in file_list: - file_paths = [os.path.join(dir, file) for dir in src_root_dirs - if os.path.exists(os.path.join(dir, file))] + file_paths = [ + os.path.join(dir, file) for dir in src_root_dirs + if os.path.exists(os.path.join(dir, file))] if len(file_paths) == 0: print("-- Warning: Can't locate source file %s" % file) continue - dest_path = os.path.join(dest_root_dir, os.path.relpath(file_paths[0], - src_root_dirs[0])) + dest_path = os.path.join( + dest_root_dir, os.path.relpath(file_paths[0], src_root_dirs[0])) if all([os.path.islink(item) for item in file_paths]): # It's a symlink in all found instances, copy the link. @@ -72,7 +75,8 @@ def merge_lipo_files(src_root_dirs, file_list, copy_verbatim_subpaths, if not os.path.isdir(dest_path): os.makedirs(dest_path) elif all([os.path.isfile(item) for item in file_paths]): - # It's a regular file in all found instances, see if they're identical. + # It's a regular file in all found instances, see if they're + # identical. if all([filecmp.cmp(item, file_paths[0]) for item in file_paths]): # All instances are identical, just copy the unique file. print("-- Copying file %s" % dest_path) @@ -87,7 +91,8 @@ def merge_lipo_files(src_root_dirs, file_list, copy_verbatim_subpaths, print_command=verbose, verbose=verbose) else: # Multiple instances are different, and they're not executable. - print("-- Warning: non-executable source files are different, skipping: %s" % file_paths) + print( + "-- Warning: non-executable source files are different, skipping: %s" % file_paths) else: print("-- Warning: Unsupport file type, skipping: %s" % file_paths) @@ -131,7 +136,8 @@ verbatim. This is useful if some subdirectories already contain fat binaries. args = parser.parse_args() skip_files = args.skip_files.split() - copy_verbatim_subpaths = [subdir.strip('/') for subdir in args.copy_subdirs.split()] + copy_verbatim_subpaths = [ + subdir.strip('/') for subdir in args.copy_subdirs.split()] file_list = merge_file_lists(args.src_root_dirs, skip_files, copy_verbatim_subpaths) @@ -139,8 +145,9 @@ verbatim. This is useful if some subdirectories already contain fat binaries. if args.verbose: print("Discovered files and dirs: %s" % file_list) - merge_lipo_files(args.src_root_dirs, file_list, copy_verbatim_subpaths, - args.destination, args.verbose, args.lipo) + merge_lipo_files( + args.src_root_dirs, file_list, copy_verbatim_subpaths, + args.destination, args.verbose, args.lipo) return 0 diff --git a/utils/resolve-crashes.py b/utils/resolve-crashes.py index e843c6c68c2f1..eb0ee738fcdc6 100755 --- a/utils/resolve-crashes.py +++ b/utils/resolve-crashes.py @@ -8,12 +8,14 @@ import sys import os + def execute_cmd(cmd): print(cmd) os.system(cmd) # The regular expression we use to match compiler-crasher lines. -regex = re.compile('.*Swift :: (compiler_crashers|compiler_crashers_2|IDE/crashers)/(.*\.swift).*') +regex = re.compile( + '.*Swift :: (compiler_crashers|compiler_crashers_2|IDE/crashers)/(.*\.swift).*') # Take the output of lit as standard input. for line in sys.stdin: @@ -29,11 +31,13 @@ def execute_cmd(cmd): execute_cmd(git_mv_cmd) # Replace "not --crash" with "not", and remove XFAIL lines. - sed_replace_not_cmd = 'sed -e "s/not --crash/not/" -i "" %s' % (to_filename) + sed_replace_not_cmd = 'sed -e "s/not --crash/not/" -i "" %s' % ( + to_filename) execute_cmd(sed_replace_not_cmd) # Remove "// XFAIL: whatever" lines. - sed_remove_xfail_cmd = 'sed -e "s/^\\/\\/.*XFAIL.*$//g" -i "" %s' % (to_filename) + sed_remove_xfail_cmd = 'sed -e "s/^\\/\\/.*XFAIL.*$//g" -i "" %s' % ( + to_filename) execute_cmd(sed_remove_xfail_cmd) # "git add" the result. diff --git a/utils/rth b/utils/rth index 2a53e9c8f01eb..cabfd2b209b4b 100755 --- a/utils/rth +++ b/utils/rth @@ -19,12 +19,15 @@ import pipes VERBOSE = True + def verbose_print_command(command): if VERBOSE: print " ".join(pipes.quote(c) for c in command) sys.stdout.flush() + class ResilienceTest(object): + def __init__(self, target_build_swift, target_run, tmp_dir, test_dir, test_src, additional_compile_flags_library, no_backward_deployment): @@ -89,8 +92,7 @@ class ResilienceTest(object): for config2 in self.config_dir_map: # --no-backward-deployment skips testing a new application # linked against an old library. - if config1 == "BEFORE" and config2 == "AFTER" and \ - self.no_backward_deployment: + if config1 == "BEFORE" and config2 == "AFTER" and self.no_backward_deployment: continue yield (config1, config2) @@ -124,6 +126,7 @@ class ResilienceTest(object): returncode = subprocess.call(command) assert returncode == 0, str(command) + def main(): parser = argparse.ArgumentParser(description='Resilience test helper') parser.add_argument('--target-build-swift', required=True) diff --git a/utils/sil-opt-verify-all-modules.py b/utils/sil-opt-verify-all-modules.py index ea3f7eae270d3..ccd82b9c301ca 100755 --- a/utils/sil-opt-verify-all-modules.py +++ b/utils/sil-opt-verify-all-modules.py @@ -20,6 +20,7 @@ import tempfile import pipes + def get_verify_toolchain_modules_commands(toolchain_dir, sil_opt): if sil_opt is None: sil_opt = os.path.join(toolchain_dir, 'usr', 'bin', 'sil-opt') @@ -71,7 +72,8 @@ def get_verify_resource_dir_modules_commands( ] commands = [] - module_cache_dir = tempfile.mkdtemp(prefix="swift-testsuite-clang-module-cache") + module_cache_dir = tempfile.mkdtemp( + prefix="swift-testsuite-clang-module-cache") for (subdir, arch, triple) in known_platforms: modules_dir = os.path.join(resource_dir, subdir, arch) print(modules_dir) @@ -127,13 +129,16 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="""Verifies Swift modules.""") - parser.add_argument("--sil-opt", + parser.add_argument( + "--sil-opt", help="use the specified 'sil-opt' binary", metavar="PATH") - parser.add_argument("--verify-build-dir", + parser.add_argument( + "--verify-build-dir", help="verify the Swift resource directory under the given build directory", metavar="PATH") - parser.add_argument("--verify-xcode", + parser.add_argument( + "--verify-xcode", help="verify the Xcode.app that is currently xcode-select'ed", action="store_true") args = parser.parse_args() diff --git a/utils/split_file.py b/utils/split_file.py index f0d499f78f49f..19f6bc7eee9b2 100755 --- a/utils/split_file.py +++ b/utils/split_file.py @@ -14,6 +14,7 @@ import re import getopt + def usage(): sys.stderr.write(__doc__.strip() + "\n") sys.exit(1) diff --git a/utils/swift-api-dump.py b/utils/swift-api-dump.py index 5b3d692eb9a30..efbc148f1b60c 100755 --- a/utils/swift-api-dump.py +++ b/utils/swift-api-dump.py @@ -65,6 +65,7 @@ 'vecLib', } + def create_parser(): script_path = os.path.dirname(sys.argv[0]) script_path = os.path.abspath(script_path) @@ -87,10 +88,12 @@ def create_parser(): parser.add_argument('-I', '--include-dir', action='append', help='Add additional include directories') return parser + def output_command_result_to_file(command_args, filename): with open(filename, 'w') as output_file: subprocess.call(command_args, stdout=output_file) + def run_command(args): proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() @@ -98,6 +101,8 @@ def run_command(args): return (exitcode, out, err) # Collect the set of submodules for the given module. + + def collect_submodules(common_args, module): # Execute swift-ide-test to print the interface. my_args = ['-module-print-submodules', '-module-to-print=%s' % (module)] @@ -117,6 +122,8 @@ def collect_submodules(common_args, module): return sorted(list(submodules)) # Print out the command we're about to execute + + def print_command(cmd, outfile=""): str = " ".join(cmd) if outfile != "": @@ -124,6 +131,8 @@ def print_command(cmd, outfile=""): print(str) # Dump the API for the given module. + + def dump_module_api((cmd, extra_dump_args, output_dir, module, quiet, verbose)): # Collect the submodules submodules = collect_submodules(cmd, module) @@ -158,6 +167,7 @@ def dump_module_api((cmd, extra_dump_args, output_dir, module, quiet, verbose)): return + def pretty_sdk_name(sdk): if sdk.find("macosx") == 0: return 'OSX' @@ -170,6 +180,8 @@ def pretty_sdk_name(sdk): return 'unknownOS' # Collect the set of frameworks we should dump + + def collect_frameworks(sdk): (exitcode, sdk_path, err) = run_command(["xcrun", "--show-sdk-path", "-sdk", sdk]) if exitcode != 0: @@ -198,6 +210,7 @@ def collect_frameworks(sdk): return (sorted(list(frameworks)), sdk_path) + def create_dump_module_api_args(cmd_common, cmd_extra_args, sdk, module, target, source_filename, output_dir, quiet, verbose): # Determine the SDK root and collect the set of frameworks. @@ -224,6 +237,7 @@ def create_dump_module_api_args(cmd_common, cmd_extra_args, sdk, module, target, return results + def main(): source_filename = 'swift-api-dump.swift' parser = create_parser() diff --git a/utils/swift-bench.py b/utils/swift-bench.py index 9706291ec0808..bff1f6394a95b 100644 --- a/utils/swift-bench.py +++ b/utils/swift-bench.py @@ -48,59 +48,66 @@ def pstdev(l): class SwiftBenchHarness: - sources = [] - verbose_level = 0 - compiler = "" - tests = {} - time_limit = 1000 - min_sample_time = 100 - min_iter_time = 1 - opt_flags = [] - - def log(self, str, level): - if self.verbose_level >= level: - for _ in range(1, level): - sys.stdout.write(' ') - print(str) - - def run_command(self, cmd): - self.log(' Executing: ' + ' '.join(cmd), 1) - return subprocess.check_output(cmd, stderr=subprocess.STDOUT) - - def parse_arguments(self): - self.log("Parsing arguments.", 2) - parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbosity", help="increase output verbosity", type=int) - parser.add_argument("files", help="input files", nargs='+') - parser.add_argument('-c', '--compiler', help="compiler to use") - parser.add_argument('-t', '--timelimit', help="Time limit for every test", type=int) - parser.add_argument('-s', '--sampletime', help="Minimum time for every sample", type=int) - parser.add_argument('-f', '--flags', help="Compilation flags", nargs='+') - args = parser.parse_args() - if args.verbosity: - self.verbose_level = args.verbosity - self.sources = args.files - if args.flags: - self.opt_flags = args.flags - if args.compiler: - self.compiler = args.compiler - else: - self.compiler = 'swiftc' - if args.timelimit and args.timelimit > 0: - self.time_limit = args.timelimit - if args.sampletime and args.sampletime > 0: - self.min_sample_time = args.sampletime - self.log("Sources: %s." % ', '.join(self.sources), 3) - self.log("Compiler: %s." % self.compiler, 3) - self.log("Opt flags: %s." % ', '.join(self.opt_flags), 3) - self.log("Verbosity: %s." % self.verbose_level, 3) - self.log("Time limit: %s." % self.time_limit, 3) - self.log("Min sample time: %s." % self.min_sample_time, 3) - - def process_source(self, name): - self.log("Processing source file: %s." % name, 2) - - header = """ + sources = [] + verbose_level = 0 + compiler = "" + tests = {} + time_limit = 1000 + min_sample_time = 100 + min_iter_time = 1 + opt_flags = [] + + def log(self, str, level): + if self.verbose_level >= level: + for _ in range(1, level): + sys.stdout.write(' ') + print(str) + + def run_command(self, cmd): + self.log(' Executing: ' + ' '.join(cmd), 1) + return subprocess.check_output(cmd, stderr=subprocess.STDOUT) + + def parse_arguments(self): + self.log("Parsing arguments.", 2) + parser = argparse.ArgumentParser() + parser.add_argument( + "-v", "--verbosity", + help="increase output verbosity", type=int) + parser.add_argument("files", help="input files", nargs='+') + parser.add_argument('-c', '--compiler', help="compiler to use") + parser.add_argument( + '-t', '--timelimit', + help="Time limit for every test", type=int) + parser.add_argument( + '-s', '--sampletime', + help="Minimum time for every sample", type=int) + parser.add_argument( + '-f', '--flags', help="Compilation flags", nargs='+') + args = parser.parse_args() + if args.verbosity: + self.verbose_level = args.verbosity + self.sources = args.files + if args.flags: + self.opt_flags = args.flags + if args.compiler: + self.compiler = args.compiler + else: + self.compiler = 'swiftc' + if args.timelimit and args.timelimit > 0: + self.time_limit = args.timelimit + if args.sampletime and args.sampletime > 0: + self.min_sample_time = args.sampletime + self.log("Sources: %s." % ', '.join(self.sources), 3) + self.log("Compiler: %s." % self.compiler, 3) + self.log("Opt flags: %s." % ', '.join(self.opt_flags), 3) + self.log("Verbosity: %s." % self.verbose_level, 3) + self.log("Time limit: %s." % self.time_limit, 3) + self.log("Min sample time: %s." % self.min_sample_time, 3) + + def process_source(self, name): + self.log("Processing source file: %s." % name, 2) + + header = """ @_silgen_name("mach_absolute_time") func __mach_absolute_time__() -> UInt64 @_silgen_name("opaqueGetInt32") func _opaqueGetInt32(x: Int) -> Int @@ -122,13 +129,13 @@ def process_source(self, name): @inline(never) func Consume(x: Int) { if False() { println(x) } } """ - before_bench = """ + before_bench = """ @inline(never) """ - into_bench = """ + into_bench = """ if False() { return 0 } """ - main_begin = """ + main_begin = """ func main() { var N = 1 var name = "" @@ -136,7 +143,7 @@ def process_source(self, name): N = Process.arguments[1].toInt()! } """ - main_body = """ + main_body = """ name = "%s" if Process.arguments.count <= 2 || Process.arguments[2] == name { let start = __mach_absolute_time__() @@ -147,225 +154,241 @@ def process_source(self, name): println("\(name),\(N),\(end - start)") } """ - main_end = """ + main_end = """ } main() """ - bench_re = re.compile("^\s*func\s\s*bench_([a-zA-Z0-9_]+)\s*\(\s*\)\s*->\s*Int\s*({)?\s*$") - with open(name) as f: - lines = list(f) - output = header - looking_for_curly_brace = False - test_names = [] - for l in lines: - if looking_for_curly_brace: - output += l - if "{" not in l: - continue + bench_re = re.compile( + "^\s*func\s\s*bench_([a-zA-Z0-9_]+)\s*\(\s*\)\s*->\s*Int\s*({)?\s*$") + with open(name) as f: + lines = list(f) + output = header looking_for_curly_brace = False - output += into_bench - continue - - m = bench_re.match(l) - if m: - output += before_bench - output += l - bench_name = m.group(1) - # TODO: Keep track of the line number as well - self.log("Benchmark found: %s" % bench_name, 3) - self.tests[name + ":" + bench_name] = Test(bench_name, name, "", "") - test_names.append(bench_name) - if m.group(2): - output += into_bench - else: - looking_for_curly_brace = True - else: - output += l - - output += main_begin - for n in test_names: - output += main_body % (n, n) - processed_name = 'processed_' + os.path.basename(name) - output += main_end - with open(processed_name, 'w') as f: - f.write(output) - for n in test_names: - self.tests[name + ":" + n].processed_source = processed_name - - def process_sources(self): - self.log("Processing sources: %s." % self.sources, 2) - for s in self.sources: - self.process_source(s) - - def compile_opaque_cfile(self): - self.log("Generating and compiling C file with opaque functions.", 3) - file_body = """ + test_names = [] + for l in lines: + if looking_for_curly_brace: + output += l + if "{" not in l: + continue + looking_for_curly_brace = False + output += into_bench + continue + + m = bench_re.match(l) + if m: + output += before_bench + output += l + bench_name = m.group(1) + # TODO: Keep track of the line number as well + self.log("Benchmark found: %s" % bench_name, 3) + self.tests[ + name + ":" + + bench_name] = Test(bench_name, name, "", "") + test_names.append(bench_name) + if m.group(2): + output += into_bench + else: + looking_for_curly_brace = True + else: + output += l + + output += main_begin + for n in test_names: + output += main_body % (n, n) + processed_name = 'processed_' + os.path.basename(name) + output += main_end + with open(processed_name, 'w') as f: + f.write(output) + for n in test_names: + self.tests[name + ":" + n].processed_source = processed_name + + def process_sources(self): + self.log("Processing sources: %s." % self.sources, 2) + for s in self.sources: + self.process_source(s) + + def compile_opaque_cfile(self): + self.log("Generating and compiling C file with opaque functions.", 3) + file_body = """ #include extern "C" int32_t opaqueGetInt32(int32_t x) { return x; } extern "C" int64_t opaqueGetInt64(int64_t x) { return x; } """ - with open('opaque.cpp', 'w') as f: - f.write(file_body) - # TODO: Handle subprocess.CalledProcessError for this call: - self.run_command(['clang++', 'opaque.cpp', '-o', 'opaque.o', '-c', '-O2']) - - compiled_files = {} - - def compile_source(self, name): - self.tests[name].binary = "./" + self.tests[name].processed_source.split(os.extsep)[0] - if not self.tests[name].processed_source in self.compiled_files: - try: - self.run_command([self.compiler, self.tests[name].processed_source, "-o", self.tests[name].binary + '.o', '-c'] + self.opt_flags) - self.run_command([self.compiler, '-o', self.tests[name].binary, self.tests[name].binary + '.o', 'opaque.o']) - self.compiled_files[self.tests[name].processed_source] = ('', '') - except subprocess.CalledProcessError as e: - self.compiled_files[self.tests[name].processed_source] = ('COMPFAIL', e.output) - - (status, output) = self.compiled_files[self.tests[name].processed_source] - self.tests[name].status = status - self.tests[name].output = output - - def compile_sources(self): - self.log("Compiling processed sources.", 2) - self.compile_opaque_cfile() - for t in self.tests: - self.compile_source(t) - - def run_benchmarks(self): - self.log("Running benchmarks.", 2) - for t in self.tests: - self.run_bench(t) - - def parse_benchmark_output(self, res): - # Parse lines like - # TestName,NNN,MMM - # where NNN - performed iterations number, MMM - execution time (in ns) - results_re = re.compile(r"(\w+),[ \t]*(\d+),[ \t]*(\d+)") - m = results_re.match(res) - if not m: - return ("", 0, 0) - return (m.group(1), m.group(2), m.group(3)) - - def compute_iters_number(self, name): - scale = 1 - spent = 0 - # Measure time for one iteration - # If it's too small, increase number of iteration until it's measurable - while (spent <= self.min_iter_time): - try: - r = self.run_command([self.tests[name].binary, str(scale), - self.tests[name].name]) - (test_name, iters_computed, exec_time) = self.parse_benchmark_output(r) - # Convert ns to ms - spent = int(exec_time) / 1000000 - if spent <= self.min_iter_time: - scale *= 2 - if scale > sys.maxint: - return (0, 0) - except subprocess.CalledProcessError as e: - r = e.output - break - if spent == 0: - spent = 1 - # Now compute number of samples we can take in the given time limit - mult = int(self.min_sample_time / spent) - if mult == 0: - mult = 1 - scale *= mult - spent *= mult - samples = int(self.time_limit / spent) - if samples == 0: - samples = 1 - return (samples, scale) - - def run_bench(self, name): - if not self.tests[name].status == "": - return - (num_samples, iter_scale) = self.compute_iters_number(name) - if (num_samples, iter_scale) == (0, 0): - self.tests[name].status = "CAN'T MEASURE" - self.tests[name].output = "Can't find number of iterations for the test to last longer than %d ms." % self.min_iter_time - return - samples = [] - self.log("Running bench: %s, numsamples: %d" % (name, num_samples), 2) - for _ in range(0, num_samples): - try: - r = self.run_command([self.tests[name].binary, str(iter_scale), - self.tests[name].name]) - (test_name, iters_computed, exec_time) = self.parse_benchmark_output(r) - # TODO: Verify test_name and iters_computed - samples.append(int(exec_time) / iter_scale) - self.tests[name].output = r - except subprocess.CalledProcessError as e: - self.tests[name].status = "RUNFAIL" - self.tests[name].output = e.output - break - res = TestResults(name, samples) - self.tests[name].results = res - - def report_results(self): - self.log("\nReporting results.", 2) - print("==================================================") - for t in self.tests: - self.tests[t].do_print() + with open('opaque.cpp', 'w') as f: + f.write(file_body) + # TODO: Handle subprocess.CalledProcessError for this call: + self.run_command( + ['clang++', 'opaque.cpp', '-o', 'opaque.o', '-c', '-O2']) + + compiled_files = {} + + def compile_source(self, name): + self.tests[name].binary = "./" + \ + self.tests[name].processed_source.split(os.extsep)[0] + if not self.tests[name].processed_source in self.compiled_files: + try: + self.run_command([ + self.compiler, self.tests[name].processed_source, + "-o", self.tests[name].binary + '.o', '-c'] + self.opt_flags) + self.run_command([ + self.compiler, '-o', self.tests[name].binary, + self.tests[name].binary + '.o', 'opaque.o']) + self.compiled_files[ + self.tests[name].processed_source] = ('', '') + except subprocess.CalledProcessError as e: + self.compiled_files[self.tests[name].processed_source] = ( + 'COMPFAIL', e.output) + + (status, output) = self.compiled_files[ + self.tests[name].processed_source] + self.tests[name].status = status + self.tests[name].output = output + + def compile_sources(self): + self.log("Compiling processed sources.", 2) + self.compile_opaque_cfile() + for t in self.tests: + self.compile_source(t) + + def run_benchmarks(self): + self.log("Running benchmarks.", 2) + for t in self.tests: + self.run_bench(t) + + def parse_benchmark_output(self, res): + # Parse lines like + # TestName,NNN,MMM + # where NNN - performed iterations number, MMM - execution time (in ns) + results_re = re.compile(r"(\w+),[ \t]*(\d+),[ \t]*(\d+)") + m = results_re.match(res) + if not m: + return ("", 0, 0) + return (m.group(1), m.group(2), m.group(3)) + + def compute_iters_number(self, name): + scale = 1 + spent = 0 + # Measure time for one iteration + # If it's too small, increase number of iteration until it's measurable + while (spent <= self.min_iter_time): + try: + r = self.run_command([ + self.tests[name].binary, str(scale), + self.tests[name].name]) + (test_name, iters_computed, exec_time) = self.parse_benchmark_output(r) + # Convert ns to ms + spent = int(exec_time) / 1000000 + if spent <= self.min_iter_time: + scale *= 2 + if scale > sys.maxint: + return (0, 0) + except subprocess.CalledProcessError as e: + r = e.output + break + if spent == 0: + spent = 1 + # Now compute number of samples we can take in the given time limit + mult = int(self.min_sample_time / spent) + if mult == 0: + mult = 1 + scale *= mult + spent *= mult + samples = int(self.time_limit / spent) + if samples == 0: + samples = 1 + return (samples, scale) + + def run_bench(self, name): + if not self.tests[name].status == "": + return + (num_samples, iter_scale) = self.compute_iters_number(name) + if (num_samples, iter_scale) == (0, 0): + self.tests[name].status = "CAN'T MEASURE" + self.tests[name].output = ( + "Can't find number of iterations for the test to last longer than %d ms." % self.min_iter_time) + return + samples = [] + self.log("Running bench: %s, numsamples: %d" % (name, num_samples), 2) + for _ in range(0, num_samples): + try: + r = self.run_command([self.tests[name].binary, str(iter_scale), + self.tests[name].name]) + (test_name, iters_computed, exec_time) = self.parse_benchmark_output(r) + # TODO: Verify test_name and iters_computed + samples.append(int(exec_time) / iter_scale) + self.tests[name].output = r + except subprocess.CalledProcessError as e: + self.tests[name].status = "RUNFAIL" + self.tests[name].output = e.output + break + res = TestResults(name, samples) + self.tests[name].results = res + + def report_results(self): + self.log("\nReporting results.", 2) + print("==================================================") + for t in self.tests: + self.tests[t].do_print() class Test: - def __init__(self, name, source, processed_source, binary): - self.name = name - self.source = source - self.processed_source = processed_source - self.binary = binary - self.status = "" - - def do_print(self): - print("NAME: %s" % self.name) - print("SOURCE: %s" % self.source) - if self.status == "": - self.results.do_print() - else: - print("STATUS: %s" % self.status) - print("OUTPUT:") - print(self.output) - print("END OF OUTPUT") - print("") + + def __init__(self, name, source, processed_source, binary): + self.name = name + self.source = source + self.processed_source = processed_source + self.binary = binary + self.status = "" + + def do_print(self): + print("NAME: %s" % self.name) + print("SOURCE: %s" % self.source) + if self.status == "": + self.results.do_print() + else: + print("STATUS: %s" % self.status) + print("OUTPUT:") + print(self.output) + print("END OF OUTPUT") + print("") class TestResults: - def __init__(self, name, samples): - self.name = name - self.samples = samples - if len(samples) > 0: - self.process() - - def process(self): - self.minimum = min(self.samples) - self.maximum = max(self.samples) - self.avg = sum(self.samples) / len(self.samples) - self.std = pstdev(self.samples) - self.err = self.std / math.sqrt(len(self.samples)) - self.int_min = self.avg - self.err * 1.96 - self.int_max = self.avg + self.err * 1.96 - - def do_print(self): - print("SAMPLES: %d" % len(self.samples)) - print("MIN: %3.2e" % self.minimum) - print("MAX: %3.2e" % self.maximum) - print("AVG: %3.2e" % self.avg) - print("STD: %3.2e" % self.std) - print("ERR: %3.2e (%2.1f%%)" % (self.err, self.err * 100 / self.avg)) - print("CONF INT 0.95: (%3.2e, %3.2e)" % (self.int_min, self.int_max)) - print("") + + def __init__(self, name, samples): + self.name = name + self.samples = samples + if len(samples) > 0: + self.process() + + def process(self): + self.minimum = min(self.samples) + self.maximum = max(self.samples) + self.avg = sum(self.samples) / len(self.samples) + self.std = pstdev(self.samples) + self.err = self.std / math.sqrt(len(self.samples)) + self.int_min = self.avg - self.err * 1.96 + self.int_max = self.avg + self.err * 1.96 + + def do_print(self): + print("SAMPLES: %d" % len(self.samples)) + print("MIN: %3.2e" % self.minimum) + print("MAX: %3.2e" % self.maximum) + print("AVG: %3.2e" % self.avg) + print("STD: %3.2e" % self.std) + print("ERR: %3.2e (%2.1f%%)" % (self.err, self.err * 100 / self.avg)) + print("CONF INT 0.95: (%3.2e, %3.2e)" % (self.int_min, self.int_max)) + print("") def main(): - harness = SwiftBenchHarness() - harness.parse_arguments() - harness.process_sources() - harness.compile_sources() - harness.run_benchmarks() - harness.report_results() + harness = SwiftBenchHarness() + harness.parse_arguments() + harness.process_sources() + harness.compile_sources() + harness.run_benchmarks() + harness.report_results() main() diff --git a/utils/swift_build_support/swift_build_support/debug.py b/utils/swift_build_support/swift_build_support/debug.py index c008d48bd6790..884e12bd1e0f6 100644 --- a/utils/swift_build_support/swift_build_support/debug.py +++ b/utils/swift_build_support/swift_build_support/debug.py @@ -29,13 +29,14 @@ def _output(args): def print_xcodebuild_versions(sdks, file=sys.stdout): - """ - Print the host machine's `xcodebuild` version, as well as version - information for each of the given SDKs (for a full list of available - SDKs, invoke `xcodebuild -showsdks` on the command line). - """ - print(u'--- SDK versions ---', file=file) - print(u'{}\n'.format(_output(['xcodebuild', '-version'])), file=file) - for sdk in sdks: - print(u'{}\n'.format(_output(['xcodebuild', '-version', '-sdk', sdk])), - file=file) + """ + Print the host machine's `xcodebuild` version, as well as version + information for each of the given SDKs (for a full list of available + SDKs, invoke `xcodebuild -showsdks` on the command line). + """ + print(u'--- SDK versions ---', file=file) + print(u'{}\n'.format(_output(['xcodebuild', '-version'])), file=file) + for sdk in sdks: + print( + u'{}\n'.format(_output(['xcodebuild', '-version', '-sdk', sdk])), + file=file) diff --git a/utils/swift_build_support/tests/test_tar.py b/utils/swift_build_support/tests/test_tar.py index 4b04a0e91761c..5758109c0926c 100644 --- a/utils/swift_build_support/tests/test_tar.py +++ b/utils/swift_build_support/tests/test_tar.py @@ -17,17 +17,18 @@ class TarTestCase(unittest.TestCase): - def test_tar_this_file_succeeds(self): - # `tar` complains about absolute paths, so use a relative path here. - source = os.path.relpath(__file__) - _, destination = tempfile.mkstemp() - tar(source=source, destination=destination) - def test_tar_nonexistent_file_raises(self): - with self.assertRaises(subprocess.CalledProcessError): - tar(source='/path/to/something/that/surely/doesnt/exist', - destination='/another/path/that/shouldnt/exist') + def test_tar_this_file_succeeds(self): + # `tar` complains about absolute paths, so use a relative path here. + source = os.path.relpath(__file__) + _, destination = tempfile.mkstemp() + tar(source=source, destination=destination) + + def test_tar_nonexistent_file_raises(self): + with self.assertRaises(subprocess.CalledProcessError): + tar(source='/path/to/something/that/surely/doesnt/exist', + destination='/another/path/that/shouldnt/exist') if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/utils/update-checkout b/utils/update-checkout index 417f3c9d14b09..31a378aa64729 100755 --- a/utils/update-checkout +++ b/utils/update-checkout @@ -38,6 +38,7 @@ REPOSITORIES = { 'swift-integration-tests': 'apple/swift-integration-tests', } + def update_working_copy(repo_path, branch): if not os.path.isdir(repo_path): return @@ -58,6 +59,7 @@ def update_working_copy(repo_path, branch): check_call(["git", "fetch"]) check_call(["git", "rebase", "FETCH_HEAD"]) + def obtain_additional_swift_sources( with_ssh, branch, skip_history, skip_repositories): for dir_name, repo in REPOSITORIES.items(): @@ -79,6 +81,7 @@ def obtain_additional_swift_sources( src_path = SWIFT_SOURCE_ROOT + "/" + dir_name + "/" + ".git" check_call(['git', '--git-dir', src_path, '--work-tree', os.path.join(SWIFT_SOURCE_ROOT, dir_name), 'checkout', branch]) + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -86,21 +89,26 @@ def main(): repositories. By default, updates your checkouts of Swift, SourceKit, LLDB, and SwiftPM.""") - parser.add_argument("--clone", + parser.add_argument( + "--clone", help="Obtain Sources for Swift and Related Projects", action="store_true") - parser.add_argument("--clone-with-ssh", + parser.add_argument( + "--clone-with-ssh", help="Obtain Sources for Swift and Related Projects via SSH", action="store_true") - parser.add_argument("--skip-history", + parser.add_argument( + "--skip-history", help="Skip histories when obtaining sources", action="store_true") - parser.add_argument("--skip-repository", + parser.add_argument( + "--skip-repository", metavar="DIRECTORY", default=[], help="Skip the specified repository", action="append") - parser.add_argument("--branch", + parser.add_argument( + "--branch", help="Obtain Sources for specific branch") args = parser.parse_args() diff --git a/utils/viewcfg b/utils/viewcfg index 21491b438268a..b6c9174914c43 100755 --- a/utils/viewcfg +++ b/utils/viewcfg @@ -33,8 +33,9 @@ import sys import tempfile import subprocess + def help(): - print("""\ + print("""\ Usage: viewcfg [output-suffix] < file @@ -43,6 +44,7 @@ By default all CFGs are opened in the same window. Use the a unique output-suffix to open a CFG in a new window. """) + class Block: current_index = 0 @@ -81,6 +83,7 @@ class Block: return self.succs + def main(): suffix = "" if len(sys.argv) >= 2: @@ -150,16 +153,19 @@ def main(): out_file.write('digraph "CFG" {\n') for name, block in blocks.iteritems(): if block.content is not None: - out_file.write("\tNode" + str(block.index) + + out_file.write( + "\tNode" + str(block.index) + " [shape=record,label=\"{" + block.content + "}\"];\n") else: - out_file.write("\tNode" + str(block.index) + + out_file.write( + "\tNode" + str(block.index) + " [shape=record,color=gray,fontcolor=gray,label=\"{" + block.name + "}\"];\n") for succ_name in block.get_succs(): succ_block = blocks[succ_name] - out_file.write("\tNode" + str(block.index) + " -> Node" + + out_file.write( + "\tNode" + str(block.index) + " -> Node" + str(succ_block.index) + ";\n") out_file.write("}\n")