Skip to content

Commit

Permalink
Move configuration into consul.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ryanbreen
ryanbreen committed Aug 13, 2014
1 parent 7e2fda4 commit 681957a
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 84 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
node_modules
config.json
172 changes: 88 additions & 84 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,130 +6,134 @@ var bodyParser = require('body-parser');

var ConsulClass = require('consul-node');

// TODO: Make these aspects configurable

var consul = new ConsulClass();

var GIT_ROOT = '/tmp/git_cache';
var GIT_URL = 'ssh://git@devgit.vistaprint.net:7999/~rbreen/configuration.git';
var BRANCHES = ['development', 'staging', 'production'];
require('./utils/read_config.js')(function(err, config) {
if (err) return console.error(err);

var git_manager = require('./lib/git_manager.js')(GIT_ROOT, GIT_URL, BRANCHES);
var GitManagerClass = require('./lib/git_manager.js');

console.log(config);

var git_manager = new GitManagerClass(config.local_store, config.git_url, config.branches);

/**
* Process a file node. If it's a directory, process each file in the directory.
* If it's a file, add the file as a KV to Consul with filename as key and file content
* as value.
*/
var process_file = function(data_dir, branch, f) {
/**
* Process a file node. If it's a directory, process each file in the directory.
* If it's a file, add the file as a KV to Consul with filename as key and file content
* as value.
*/
var process_file = function(data_dir, branch, f) {

if (!f) f = '';
if (!f) f = '';

var fqf = data_dir + f;
var fqf = data_dir + f;

// Don't recurse into .git directories.
if (f.indexOf('.git') != -1) return;
// Don't recurse into .git directories.
if (f.indexOf('.git') != -1) return;

if (fs.statSync(fqf).isDirectory()) {
if (fs.statSync(fqf).isDirectory()) {

console.log('Reading directory %s', fqf);
console.log('Reading directory %s', fqf);

fs.readdir(fqf, function(err, files) {
if (err) return console.error('Failed to read directory %s', err);
fs.readdir(fqf, function(err, files) {
if (err) return console.error('Failed to read directory %s', err);

files.forEach(function(file) {
process_file(data_dir, branch, f + '/' + file);
files.forEach(function(file) {
process_file(data_dir, branch, f + '/' + file);
});
});
});

} else {
} else {

fs.readFile(fqf, {encoding:'utf8'}, function(err, body) {
fs.readFile(fqf, {encoding:'utf8'}, function(err, body) {

// Prepend branch name to the KV so that the subtree is properly namespaced in Consul
var key_name = branch + '/' + f.substring(1);
var body = body ? body.trim() : '';
// Prepend branch name to the KV so that the subtree is properly namespaced in Consul
var key_name = branch + '/' + f.substring(1);
var body = body ? body.trim() : '';

if (err) return console.error('Failed to read directory %s', err);
console.log('Adding key %s, value %s', key_name, body);
consul.kv.put(key_name, body, function(err) {
if (err) return console.error('Failed to put key %s: %s', key_name, require('util').inspect(err));
if (err) return console.error('Failed to read directory %s', err);
console.log('Adding key %s, value %s', key_name, body);
consul.kv.put(key_name, body, function(err) {
if (err) return console.error('Failed to put key %s: %s', key_name, require('util').inspect(err));

console.log('Wrote key %s', key_name);
console.log('Wrote key %s', key_name);
});
});
});

}
};
}
};

git_manager.init(function(err) {
git_manager.init(function(err) {

if (err) return console.error("Failed to load git_manager:\n%s", err);
if (err) return console.error("Failed to load git_manager:\n%s", err);

var app = express();
var app = express();

// Stash's webhook sends a totally bogus content-encoding. Smack it before the body parser runs.
app.use(function() {return function(req, res, next) {delete req.headers['content-encoding'];next();}}());
// Stash's webhook sends a totally bogus content-encoding. Smack it before the body parser runs.
app.use(function() {return function(req, res, next) {delete req.headers['content-encoding'];next();}}());

// parse application/json
app.use(bodyParser.json());
// parse application/json
app.use(bodyParser.json());

var handle_ref_change = function(refChange) {
var handle_ref_change = function(refChange) {

console.log('Handling reference change %s', util.inspect(refChange));
console.log('Handling reference change %s', util.inspect(refChange));

// Only update if the head of a branch changed
if (refChange.refId && (refChange.refId.indexOf('refs/heads/') === 0) && refChange.toHash) {
// Strip leading 'refs/heads/' from branch name
var branch_name = refChange.refId.substring(11);
// Only update if the head of a branch changed
if (refChange.refId && (refChange.refId.indexOf('refs/heads/') === 0) && refChange.toHash) {
// Strip leading 'refs/heads/' from branch name
var branch_name = refChange.refId.substring(11);

// Update consul git branch
var bm = git_manager.getBranchManager(branch_name);
bm.currentRef(function(err, ref) {
if (err) return console.error('Failed to check current state of branch %s: %s', branch_name, err);
// Update consul git branch
var bm = git_manager.getBranchManager(branch_name);
bm.currentRef(function(err, ref) {
if (err) return console.error('Failed to check current state of branch %s: %s', branch_name, err);

if (ref === refChange.toHash) {
return console.log('Branch %s already at most recent version', branch_name);
}
if (ref === refChange.toHash) {
return console.log('Branch %s already at most recent version', branch_name);
}

bm.pull(function(err) {
if (err) return console.error('Failed to update %s due to %s', branch_name, err);
bm.pull(function(err) {
if (err) return console.error('Failed to update %s due to %s', branch_name, err);

bm.currentRef(function(err, ref) {
if (err) return console.error('Failed to check current state of branch %s', branch_name);
bm.currentRef(function(err, ref) {
if (err) return console.error('Failed to check current state of branch %s', branch_name);

if (ref !== refChange.toHash) {
// TODO: Remove this once testing is complete.
//return console.error('Branch update failed to reach toHash value %s', refChange.toHash);
}
if (ref !== refChange.toHash) {
// TODO: Remove this once testing is complete.
//return console.error('Branch update failed to reach toHash value %s', refChange.toHash);
}

console.log('Branch %s updated. Syncing with Consul', branch_name);
console.log('Branch %s updated. Syncing with Consul', branch_name);

// Add changed branch to Consul
process_file(bm.getPath(), branch_name);
// Add changed branch to Consul
process_file(bm.getPath(), branch_name);
});
});
});
});
}
};
}
};

// TESTING PURPOSES
handle_ref_change({'refId':'refs/heads/development', 'toHash':'0'});
// TESTING PURPOSES
// TESTING PURPOSES
//handle_ref_change({'refId':'refs/heads/development', 'toHash':'0'});
// TESTING PURPOSES

['get','post','put','delete'].forEach(function(verb) {
app[verb]('/gitpoke', function(req, res){
//console.log(util.inspect(req.body));
console.log('Got pinged by git hook, checking results');
['get','post','put','delete'].forEach(function(verb) {
app[verb]('/gitpoke', function(req, res){
//console.log(util.inspect(req.body));
console.log('Got pinged by git hook, checking results');

if (req.body && req.body.refChanges) {
// Only pull changed branches
req.body.refChanges.forEach(handle_ref_change);
}
if (req.body && req.body.refChanges) {
// Only pull changed branches
req.body.refChanges.forEach(handle_ref_change);
}

res.send('ok');
res.send('ok');
});
});
});

app.listen(5050);
app.listen(5050);
});

});
2 changes: 2 additions & 0 deletions lib/git_manager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ module.exports = function(root_directory, git_url, branches) {

mkdirp(branch_directory, function(err) {
if (err) return cb(err);

console.log("Running command in branch %s, dir %s", branch, branch_directory);

var child = exec('git clone -b ' + branch + ' ' + git_url, {cwd: branch_directory}, function(err, stdout, stderr) {
if (stdout) stdout.trim();
Expand Down
19 changes: 19 additions & 0 deletions utils/read_config.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

var Consul = require('consul-node');
var consul = new Consul();

/**
* Grab the meta config to bootstrap git2consul.
*/
module.exports = function(cb) {
consul.kv.get('/git2consul/?recurse', function(err, items) {
if (err) return cb(err);

var config = {};
items.forEach(function(item) {
config[item.key.substring(11)] = item.value.indexOf(',') === -1 ? item.value : item.value.split(',');
});

cb(null, config);
});
};
40 changes: 40 additions & 0 deletions utils/seed_config.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
var fs = require('fs');

var Consul = require('consul-node');

var consul = new Consul();

/**
* This utility adds keys from the top level of a .json config file to the /git2consul/ path of
* Consul. We use this to seed Consul's KV with the bootstrapping information used by git2consul.
*/

if (process.argv.length !== 3) {
console.error('usage: node utils/seed.js config.js');
process.exit(1);
}

var add_entry = function(key, value, cb) {
console.log('Adding config %s : %s', key, value);
consul.kv.put(key, value, cb);
};

var config_file = process.argv[2];

console.log('Adding keys from config file %s to local consul cluster', config_file);

var file = fs.readFileSync(config_file, {'encoding':'utf8'});
var obj = JSON.parse(file);
for (var key in obj) {
var value = obj[key];
if (typeof value === 'string') {
add_entry('git2consul/' + key, value, function(err) {
if (err) return console.error(err);
});
} else if (value.hasOwnProperty('length')) {
value = value.join();
add_entry('git2consul/' + key, value, function(err) {
if (err) return console.error(err);
});
}
}

0 comments on commit 681957a

Please sign in to comment.