Skip to content

Commit

Permalink
run mats configurations in parallel (cisco#561)
Browse files Browse the repository at this point in the history 
* refactor mats to allow different configurations to run in parallel.

The {partial,all,bully}x targets now support being run in parallel should
make decide to do so (e.g., via the -j flag)

* fix mats ignoring "rmg" parameter

* Update travis-ci build scripts to use new partialx target and run multiple
jobs in parallel, based on the number of processors available.

* Add a way to only run particular machines in travis-ci by including
a line that starts with "travis:only:" and lists the machine types in the
commit message.
  • Loading branch information
@jltaylor-us
jltaylor-us authored May 2, 2021
1 parent 728813f commit 96ee63d
Show file tree
Hide file tree
Showing 39 changed files with 465 additions and 357 deletions.
12 changes: 9 additions & 3 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,15 @@ jobs:
# macOS
- env: TARGET_MACHINE=i3osx
os: osx
- env: TARGET_MACHINE=ti3osx
- env:
- TARGET_MACHINE=ti3osx
- PARALLEL_MATS=1
os: osx
- env: TARGET_MACHINE=a6osx
os: osx
- env: TARGET_MACHINE=ta6osx
- env:
- TARGET_MACHINE=ta6osx
- PARALLEL_MATS=1
os: osx

# Linux
Expand Down Expand Up @@ -59,6 +63,8 @@ jobs:
- git config core.autocrlf false; rm .git/index; git reset --hard
- choco install make -y
dist: bionic
before_install:
- .travis/maybe-skip-build.sh
script:
- .travis/build.sh
- .travis/test.sh
- travis_wait 60 .travis/test.sh
2 changes: 1 addition & 1 deletion .travis/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ set -e -o pipefail
echo 'travis_fold:start:build'
echo Building Chez Scheme...
./configure -m="$TARGET_MACHINE"
make
make -j $(getconf _NPROCESSORS_ONLN)
case "$TARGET_MACHINE" in
*a6nt)
curl -Ls https://github.com/burgerrg/win-iconv/releases/download/v0.0.9/iconv-x64.dll > "$TARGET_MACHINE"/bin/"$TARGET_MACHINE"/iconv.dll
Expand Down
24 changes: 24 additions & 0 deletions .travis/maybe-skip-build.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/bin/bash

git branch -v
echo "TRAVIS_COMMIT: $TRAVIS_COMMIT"
echo "TRAVIS_COMMIT_RANGE: $TRAVIS_COMMIT_RANGE"
echo "TRAVIS_PULL_REQUEST: $TRAVIS_PULL_REQUEST"

if test -z "$TRAVIS_PULL_REQUEST"; then
msg="$TRAVIS_COMMIT_MESSAGE"
else
msg="$(echo "$TRAVIS_COMMIT_RANGE" | cut -d. -f4 |xargs git log --format=%B -n 1)"
fi

echo "Checking to see whether to skip build for $TARGET_MACHINE due to commit message"
echo "$msg"
msg_only="$(echo "$msg" | grep "^travis:only:")"
if test -n "$msg_only" ; then
if ! echo "$msg_only" | grep -q "\<${TARGET_MACHINE}\>" ; then
echo "Skipping builds for target machine type $TARGET_MACHINE due to commit message:"
echo "$msg_only"
exit 1
fi
fi

2 changes: 1 addition & 1 deletion .travis/summary
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
-------- o=0 --------
-------- o=3 --------
-------- o=3 cp0=t --------
-------- o=3 cp0=t eval=interpret --------
-------- o=3 cp0=t eval=interpret rmg=2 --------
32 changes: 20 additions & 12 deletions .travis/test.sh
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,28 @@
#!/bin/bash
runmats() {
echo travis_fold:start:mats
echo make allxhelp "$@"
make -C "$TARGET_MACHINE"/mats allxhelp "$@" 2>&1 | .travis/matting.sh
echo travis_fold:end:mats
}

runmats o=0
runmats o=3
runmats o=3 cp0=t
runmats o=3 cp0=t eval=interpret
if test -n "$PARALLEL_MATS" ; then
njobs="$PARALLEL_MATS"
else
njobs="$(getconf _NPROCESSORS_ONLN)"
fi
make -C "$TARGET_MACHINE"/mats -j "$njobs" partialx

if [ -f "$TARGET_MACHINE"/mats/summary ]; then
cat "$TARGET_MACHINE"/mats/summary
diff -q .travis/summary "$TARGET_MACHINE"/mats/summary
exit $?
rc=$?
if [ $rc -gt 0 ]; then
echo 'travis_fold:start:make_prereqs_output'
echo "Make output from $TARGET_MACHINE/mats/Make.out:"
cat "$TARGET_MACHINE"/mats/Make.out
echo 'travis_fold:end:make_prereqs_output'
for log in "$TARGET_MACHINE"/mats/output*/Make.out ; do
echo 'travis_fold:start:make_output'
echo "Make output from ${log}:"
cat "${log}"
echo 'travis_fold:end:make_output'
done
fi
exit $rc
else
exit 1
fi
12 changes: 12 additions & 0 deletions LOG
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2173,3 +2173,15 @@
foreign.stex
- New spellings #true and #false for #t and #f are recognized
read.ss 6.ms
- refactor mats to allow different configurations to run in parallel.
The {partial,all,bully}x targets in Mats/Mf-base now support running
in parallel if make chooses to do so (e.g., if instructed via -j).
Update travis-ci build scripts to use new partialx target and run
jobs in parallel (based on the number of cores available). Also
add the ability to "skip" (i.e., error before building) travis targets
by using a line (or lines) beginning with "travis:only:" and listing
the desired target machine type(s) in the commit message.
.travis.yml .travis/{build,test,maybe-skip-build}.sh
mats/{5_4,6,7,8,bytevector,examples,foreign}.ms
mats/{ftype,hash,io,misc,primvars,profile,record}.ms
mats/Mf-base mats/Mf-*nt mats/mat.ss mats/patch-interpret*
4 changes: 2 additions & 2 deletions mats/5_4.ms
View file
Original file line number Diff line number Diff line change
Expand Up @@ -992,7 +992,7 @@
(error? (string-normalize-nfkc "hello" "goodbye"))
(error? (string-normalize-nfkc 'ouch))
(begin
(load "../unicode/unicode-data.ss")
(load (format "~a/../unicode/unicode-data.ss" *mats-dir*))
#t)
(let ()
(import (unicode-data))
Expand All @@ -1014,7 +1014,7 @@
(let ([data (map (lambda (x) (map conv (list-head x 5)))
(filter (lambda (x) (>= (length x) 5))
(get-unicode-data
"../unicode/UNIDATA/NormalizationTest.txt")))])
(format "~a/../unicode/UNIDATA/NormalizationTest.txt" *mats-dir*))))])
(define NFD string-normalize-nfd)
(define NFKD string-normalize-nfkd)
(define NFC string-normalize-nfc)
Expand Down
61 changes: 33 additions & 28 deletions mats/6.ms
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

;;; sections 6-1 and 6-2:

(define prettytest.ss (format "~a/prettytest.ss" *mats-dir*))

(mat current-input-port
(port? (current-input-port))
(input-port? (current-input-port))
Expand Down Expand Up @@ -422,21 +424,21 @@
mode2))
mode1))))
(and
(cmp '() "prettytest.ss" '() "prettytest.ss")
(cmp '(compressed) "prettytest.ss" '() "prettytest.ss")
(cmp '() "prettytest.ss" '(compressed) "prettytest.ss")
(cmp '(compressed) "prettytest.ss" '(compressed) "prettytest.ss")
(cmp '() prettytest.ss '() prettytest.ss)
(cmp '(compressed) prettytest.ss '() prettytest.ss)
(cmp '() prettytest.ss '(compressed) prettytest.ss)
(cmp '(compressed) prettytest.ss '(compressed) prettytest.ss)
(begin
(cp '(replace compressed) "prettytest.ss" "testfile.ss")
(cp '(replace compressed) prettytest.ss "testfile.ss")
#t)
(cmp '(compressed) "testfile.ss" '() "prettytest.ss")
(not (= (call-with-input-file "testfile.ss" file-length) (call-with-input-file "prettytest.ss" file-length)))
(cmp '(compressed) "testfile.ss" '() prettytest.ss)
(not (= (call-with-input-file "testfile.ss" file-length) (call-with-input-file prettytest.ss file-length)))
; the following test could cause an error with anything but latin-1 codec
#;(not (cmp '() "testfile.ss" '() "prettytest.ss"))
#;(not (cmp '() "testfile.ss" '() prettytest.ss))
(begin
(cp '(compressed append) "prettytest.ss" "testfile.ss")
(cp '(compressed append) prettytest.ss "testfile.ss")
#t)
(not (cmp '(compressed) "testfile.ss" '() "prettytest.ss"))
(not (cmp '(compressed) "testfile.ss" '() prettytest.ss))
))
(error? (open-output-file "testfile.ss" '(replace append)))
(error? (open-output-file "testfile.ss" '(append truncate)))
Expand Down Expand Up @@ -812,8 +814,8 @@
(lambda ()
(close-input-port ip)
(close-output-port op)))))])
(pretty-copy "prettytest.ss" "testfile.ss"))
(let ([p1 (open-input-file "prettytest.ss")]
(pretty-copy prettytest.ss "testfile.ss"))
(let ([p1 (open-input-file prettytest.ss)]
[p2 (open-input-file "testfile.ss")])
(dynamic-wind
(lambda () #f)
Expand Down Expand Up @@ -870,8 +872,8 @@
(lambda ()
(close-input-port ip)
(close-output-port op)))))])
(unpretty-copy "prettytest.ss" "testfile.ss"))
(let ([p1 (open-input-file "prettytest.ss")]
(unpretty-copy prettytest.ss "testfile.ss"))
(let ([p1 (open-input-file prettytest.ss)]
[p2 (open-input-file "testfile.ss")])
(dynamic-wind
(lambda () #f)
Expand Down Expand Up @@ -914,7 +916,7 @@
(lambda (p) (fasl-write +nan.0 p)))
(call-with-port (open-file-input-port "testfile.ss") fasl-read))
(/ 0.0 0.0))
(let ([ls (with-input-from-file "prettytest.ss"
(let ([ls (with-input-from-file prettytest.ss
(rec f
(lambda ()
(let ([x (read)])
Expand Down Expand Up @@ -964,8 +966,8 @@
(open-bytevector-input-port
(call-with-bytevector-output-port put-stuff))
(get-stuff fasl-read)))))
(eqv? (fasl-file "prettytest.ss" "testfile.ss") (void))
(let ([ls (with-input-from-file "prettytest.ss"
(eqv? (fasl-file prettytest.ss "testfile.ss") (void))
(let ([ls (with-input-from-file prettytest.ss
(rec f
(lambda ()
(let ([x (read)])
Expand Down Expand Up @@ -2622,8 +2624,8 @@
(eq? '\x23;foo\x7C;bar '\#foo\|bar)
)

(mat with-source-path
(equal? (source-directories) '("."))
(mat with-source-path (parameters [current-directory *mats-dir*] [source-directories '(".")] [library-directories '(".")])
(equal? (separate-eval '(source-directories)) "(\".\")\n")
(equal?
(with-source-path 'test "I should not be here" list)
'("I should not be here"))
Expand Down Expand Up @@ -2958,6 +2960,18 @@
(error? (get-mode "probably/not/there"))
(error? (get-mode "probably/not/there" #f))
(error? (get-mode "probably/not/there" #t))
(error? (file-access-time "probably/not/there"))
(error? (file-access-time "probably/not/there" #f))
(error? (file-access-time "probably/not/there" #t))
(error? (file-change-time "probably/not/there"))
(error? (file-change-time "probably/not/there" #f))
(error? (file-change-time "probably/not/there" #t))
(error? (file-modification-time "probably/not/there"))
(error? (file-modification-time "probably/not/there" #f))
(error? (file-modification-time "probably/not/there" #t))
)

(mat filesystem-operations2 (parameters [current-directory *mats-dir*])
(if (or (windows?) (embedded?))
(fixnum? (get-mode "mat.ss"))
(let ([m (get-mode "mat.ss")])
Expand Down Expand Up @@ -3008,15 +3022,6 @@
(time=? (file-change-time "Makefile") (file-change-time (format "Mf-~a" (machine-type)))))
(or (windows?) (embedded?)
(time=? (file-modification-time "Makefile") (file-modification-time (format "Mf-~a" (machine-type)))))
(error? (file-access-time "probably/not/there"))
(error? (file-access-time "probably/not/there" #f))
(error? (file-access-time "probably/not/there" #t))
(error? (file-change-time "probably/not/there"))
(error? (file-change-time "probably/not/there" #f))
(error? (file-change-time "probably/not/there" #t))
(error? (file-modification-time "probably/not/there"))
(error? (file-modification-time "probably/not/there" #f))
(error? (file-modification-time "probably/not/there" #t))
)

(mat unicode-filesystem-operations
Expand Down
6 changes: 3 additions & 3 deletions mats/7.ms
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1115,12 +1115,12 @@
"(aye captain)\n")
(equal?
(begin
(unless (or (embedded?) (equal? *scheme* (format "../bin/~a/scheme~a" (machine-type) (if (windows?) ".exe" ""))))
(unless (or (embedded?) (equal? *scheme* (format "~a/bin/~a/scheme~a" (path-parent *mats-dir*) (machine-type) (if (windows?) ".exe" ""))))
(errorf #f "not testing boot file based on ../boot/~a/petite.boot, since *scheme* isn't ../bin/~a/scheme~a"
(machine-type) (machine-type) (if (windows?) ".exe" "")))
(parameterize ([optimize-level 2])
(make-boot-file "testfile.boot" '()
(format "../boot/~a/petite.boot" (machine-type))
(format "~a/boot/~a/petite.boot" (path-parent *mats-dir*) (machine-type))
"testfile-1.so"
"testfile-2.so"
"testfile-3.ss"
Expand All @@ -1144,7 +1144,7 @@
; by the static-generation collection.
(equal?
(begin
(unless (or (embedded?) (equal? *scheme* (format "../bin/~a/scheme~a" (machine-type) (if (windows?) ".exe" ""))))
(unless (or (embedded?) (equal? *scheme* (format "~a/bin/~a/scheme~a" (path-parent *mats-dir*) (machine-type) (if (windows?) ".exe" ""))))
(errorf #f "not testing boot file based on ../boot/~a/petite.boot, since *scheme* isn't ../bin/~a/scheme~a"
(machine-type) (machine-type) (if (windows?) ".exe" "")))
(mkfile "testfile.ss"
Expand Down
4 changes: 3 additions & 1 deletion mats/8.ms
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10569,7 +10569,9 @@
(separate-compile 'imno1)
#t)
(equal?
(parameterize ([console-output-port (open-output-string)])
(parameterize ([source-directories '(".")]
[library-directories '(".")]
[console-output-port (open-output-string)])
(eval '(lambda () (import (testfile-imno2)) y))
(get-output-string (console-output-port)))
"import: did not find source file \"testfile-imno2.chezscheme.sls\"\nimport: found source file \"testfile-imno2.ss\"\nimport: did not find corresponding object file \"testfile-imno2.so\"\nimport: loading source file \"testfile-imno2.ss\"\nimport: did not find source file \"testfile-imno1.chezscheme.sls\"\nimport: found source file \"testfile-imno1.ss\"\nimport: found corresponding object file \"testfile-imno1.so\"\nimport: object file is not older\nimport: visiting object file \"testfile-imno1.so\"\nattempting to 'revisit' previously 'visited' \"testfile-imno1.so\" for library (testfile-imno1) run-time info\n")
Expand Down
4 changes: 2 additions & 2 deletions mats/Mf-a6nt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Mf-a6nt
# Copyright 1984-2017 Cisco Systems, Inc.
# Copyright 1984-2021 Cisco Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -27,5 +27,5 @@ export MSYS2_ARG_CONV_EXCL=*
foreign1.so: $(fsrc)
cmd.exe /c "vs.bat amd64 && cl /DWIN32 /DX86_64 /Fe$@ /I${Include} /LD /MD /nologo ../bin/$m/csv955.lib $(fsrc)"

cat_flush: cat_flush.c
cat_flush.exe: cat_flush.c
cmd.exe /c "vs.bat amd64 && cl /DWIN32 /DX86_64 /MD /nologo $<"
Loading

0 comments on commit 96ee63d

Please sign in to comment.