Build: Add perf check into Travis build to better monitor performance

regressions (fixes eslint#732)
suisho · Mar 29, 2014 · 69bc27c · 69bc27c
1 parent daaf7ba
commit 69bc27c
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 6 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,6 +2,14 @@ language: node_js
 node_js:
     - "0.8"
     - "0.10"
+    - "0.11"
+
+matrix:
+    allowed_failures:
+        node: "0.8"
+        node: "0.11"
+
+script: "npm test && npm run perf"
 
 before_install:
     - npm link

diff --git a/Makefile.js b/Makefile.js
@@ -14,7 +14,20 @@ require("shelljs/make");
 
 var path = require("path"),
     dateformat = require("dateformat"),
-    nodeCLI = require("shelljs-nodecli");
+    nodeCLI = require("shelljs-nodecli"),
+    os = require("os");
+
+//------------------------------------------------------------------------------
+// Settings
+//------------------------------------------------------------------------------
+
+/*
+ * A little bit fuzzy. My computer has a first OS speed of 3093 and the perf test
+ * always completes in < 2000ms. However, Travis is less predictable due to
+ * multiple different VM types. So I'm fudging this for now in the hopes that it
+ * at least provides some sort of useful signal.
+ */
+var PERF_MULTIPLIER = 1.5;
 
 //------------------------------------------------------------------------------
 // Data
@@ -252,15 +265,23 @@ target.checkRuleFiles = function() {
 };
 
 target.perf = function() {
-    var start = process.hrtime(),
-        diff;
+    var start = process.hrtime();
 
     exec(ESLINT + "./tests/performance/jshint.js", { silent: true }, function() {
-        diff = process.hrtime(start);
+        var diff = process.hrtime(start),
+            actual = (diff[0] * 1e9 + diff[1]) / 1000000,
+            cpuSpeed = os.cpus()[0].speed,
+            max = cpuSpeed * PERF_MULTIPLIER;
 
-        echo("Took %dms", (diff[0] * 1e9 + diff[1]) / 1000);
-    });
+        echo("CPU Speed is %d with multiplier %d", cpuSpeed, PERF_MULTIPLIER);
 
+        if (actual > max) {
+            echo("Performance budget exceeded: %dms (limit: %dms)", actual, max);
+            exit(1);
+        } else {
+            echo("Performance budget ok:  %dms (limit: %dms)", actual, max);
+        }
+    });
 };
 
 target.patch = function() {

diff --git a/package.json b/package.json
@@ -15,6 +15,7 @@
     "major": "node Makefile.js major",
     "gensite": "node Makefile.js gensite",
     "browserify": "node Makefile.js browserify",
+    "perf": "node Makefile.js perf",
     "profile": "beefy tests/bench/bench.js --open -- -t brfs -t ./tests/bench/xform-rules.js"
   },
   "files": [