Merge pull request kubernetes#88 from Random-Liu/add-arbitray-log-sup…

…port Add arbitray system log support
linxiulei · Feb 15, 2017 · 6e35bcf · 6e35bcf
2 parents 5e56393 + 01334e7
commit 6e35bcf
Show file tree

Hide file tree

Showing 20 changed files with 1,207 additions and 202 deletions.
diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json
diff --git a/config/docker-monitor-filelog.json b/config/docker-monitor-filelog.json
@@ -0,0 +1,14 @@
+{
+	"plugin": "syslog",
+	"pluginConfig": {
+		"timestamp": "^time=\"(\\S*)\"",
+		"message": "msg=\"([^\n]*)\"",
+		"timestampFormat": "2006-01-02T15:04:05.999999999-07:00"
+	},
+	"logPath": "/var/log/docker.log",
+	"lookback": "5m",
+	"bufferSize": 10,
+	"source": "docker-monitor",
+	"conditions": [],
+	"rules": []
+}
diff --git a/config/docker-monitor.json b/config/docker-monitor.json
@@ -0,0 +1,12 @@
+{
+	"plugin": "journald",
+	"pluginConfig": {
+		"source": "docker"
+	},
+	"logPath": "/var/log/journal",
+	"lookback": "5m",
+	"bufferSize": 10,
+	"source": "docker-monitor",
+	"conditions": [],
+	"rules": []
+}
diff --git a/config/kernel-monitor-filelog.json b/config/kernel-monitor-filelog.json
@@ -0,0 +1,58 @@
+{
+	"plugin": "syslog",
+	"pluginConfig": {
+		"timestamp": "^.{15}",
+		"message": "kernel: \\[.*\\] (.*)",
+		"timestampFormat": "Jan _2 15:04:05"
+	},
+	"logPath": "/var/log/kern.log",
+	"lookback": "5m",
+	"bufferSize": 10,
+	"source": "kernel-monitor",
+	"conditions": [
+		{
+			"type": "KernelDeadlock",
+			"reason": "KernelHasNoDeadlock",
+			"message": "kernel has no deadlock"
+		}
+	],
+	"rules": [
+		{
+			"type": "temporary",
+			"reason": "OOMKilling",
+			"pattern": "Kill process \\d+ (.+) score \\d+ or sacrifice child\\nKilled process \\d+ (.+) total-vm:\\d+kB, anon-rss:\\d+kB, file-rss:\\d+kB"
+		},
+		{
+			"type": "temporary",
+			"reason": "TaskHung",
+			"pattern": "task \\S+:\\w+ blocked for more than \\w+ seconds\\."
+		},
+		{
+			"type": "temporary",
+			"reason": "UnregisterNetDevice",
+			"pattern": "unregister_netdevice: waiting for \\w+ to become free. Usage count = \\d+"
+		},
+		{
+			"type": "temporary",
+			"reason": "KernelOops",
+			"pattern": "BUG: unable to handle kernel NULL pointer dereference at .*"
+		},
+		{
+			"type": "temporary",
+			"reason": "KernelOops",
+			"pattern": "divide error: 0000 \\[#\\d+\\] SMP"
+		},
+		{
+			"type": "permanent",
+			"condition": "KernelDeadlock",
+			"reason": "AUFSUmountHung",
+			"pattern": "task umount\\.aufs:\\w+ blocked for more than \\w+ seconds\\."
+		},
+		{
+			"type": "permanent",
+			"condition": "KernelDeadlock",
+			"reason": "DockerHung",
+			"pattern": "task docker:\\w+ blocked for more than \\w+ seconds\\."
+		}
+	]
+}
diff --git a/config/kernel-monitor.json b/config/kernel-monitor.json
@@ -1,5 +1,8 @@
 {
 	"plugin": "journald",
+	"pluginConfig": {
+		"source": "kernel"
+	},
 	"logPath": "/var/log/journal",
 	"lookback": "5m",
 	"bufferSize": 10,

diff --git a/pkg/kernelmonitor/logwatchers/journald/log_watcher.go b/pkg/kernelmonitor/logwatchers/journald/log_watcher.go
@@ -114,8 +114,13 @@ func (j *journaldWatcher) watchLoop() {
 	}
 }
 
-// defaultJournalLogPath is the default path of journal log.
-const defaultJournalLogPath = "/var/log/journal"
+const (
+	// defaultJournalLogPath is the default path of journal log.
+	defaultJournalLogPath = "/var/log/journal"
+
+	// configSourceKey is the key of source configuration in the plugin configuration.
+	configSourceKey = "source"
+)
 
 // getJournal returns a journal client.
 func getJournal(cfg types.WatcherConfig) (*sdjournal.Journal, error) {
@@ -140,14 +145,18 @@ func getJournal(cfg types.WatcherConfig) (*sdjournal.Journal, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to lookback %q: %v", since, err)
 	}
-	// TODO(random-liu): Make this configurable to support parsing other logs.
-	kernelMatch := sdjournal.Match{
-		Field: sdjournal.SD_JOURNAL_FIELD_TRANSPORT,
-		Value: "kernel",
+	// Empty source is not allowed and treated as an error.
+	source := cfg.PluginConfig[configSourceKey]
+	if source == "" {
+		return nil, fmt.Errorf("failed to filter journal log, empty source is not allowed")
+	}
+	match := sdjournal.Match{
+		Field: sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER,
+		Value: source,
 	}
-	err = journal.AddMatch(kernelMatch.String())
+	err = journal.AddMatch(match.String())
 	if err != nil {
-		return nil, fmt.Errorf("failed to add log filter %#v: %v", kernelMatch, err)
+		return nil, fmt.Errorf("failed to add log filter %#v: %v", match, err)
 	}
 	return journal, nil
 }

diff --git a/pkg/kernelmonitor/logwatchers/syslog/helpers.go b/pkg/kernelmonitor/logwatchers/syslog/helpers.go
diff --git a/pkg/kernelmonitor/logwatchers/syslog/helpers_test.go b/pkg/kernelmonitor/logwatchers/syslog/helpers_test.go