From a5421270e5eab7d178c0b22af3c47562d1ca5050 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 15 Oct 2015 20:58:51 +0100 Subject: [PATCH 1/2] cadvisor bump --- Godeps/Godeps.json | 72 +++++----- .../cadvisor/container/docker/factory.go | 23 ++-- .../cadvisor/container/docker/fsHandler.go | 109 +++++++++++++++ .../cadvisor/container/docker/handler.go | 57 +++----- .../container/libcontainer/helpers.go | 87 ++++++++++++ .../src/github.com/google/cadvisor/fs/fs.go | 2 +- .../google/cadvisor/http/handlers.go | 12 +- .../google/cadvisor/info/v1/container.go | 31 +++++ .../google/cadvisor/info/v1/machine.go | 2 + .../google/cadvisor/info/v2/container.go | 18 +++ .../google/cadvisor/manager/container.go | 2 +- .../google/cadvisor/manager/machine.go | 3 +- .../google/cadvisor/metrics/prometheus.go | 119 ++++++++++++++-- .../google/cadvisor/pages/docker.go | 18 +-- .../cadvisor/storage/bigquery/bigquery.go | 2 +- .../storage/bigquery/client/client.go | 2 +- .../storage/elasticsearch/elasticsearch.go | 127 ++++++++++++++++++ .../cadvisor/utils/oomparser/oomparser.go | 2 +- .../google/cadvisor/version/VERSION | 1 + .../google/cadvisor/version/version.go | 21 ++- pkg/kubelet/cadvisor/cadvisor_linux.go | 16 ++- 21 files changed, 606 insertions(+), 120 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/google/cadvisor/container/docker/fsHandler.go create mode 100644 Godeps/_workspace/src/github.com/google/cadvisor/storage/elasticsearch/elasticsearch.go create mode 100644 Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 7c7eb3f125ea3..4d2b66d06e15b 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -281,93 +281,93 @@ }, { "ImportPath": "github.com/google/cadvisor/api", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/cache/memory", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/collector", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/container", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/events", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/fs", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/healthz", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/http", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/info/v1", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/info/v2", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/manager", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/metrics", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/pages", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/storage", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/summary", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/utils", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/validate", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/cadvisor/version", - "Comment": "0.16.0.2", - "Rev": "cefada41b87c35294533638733c563a349b95f05" + "Comment": "0.19.0", + "Rev": "0d6015c74114de9503d68c3c91efceee4dc41bd2" }, { "ImportPath": "github.com/google/gofuzz", diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go index e449586edf269..c07f444597448 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go @@ -42,6 +42,10 @@ var DockerNamespace = "docker" var dockerRootDir = flag.String("docker_root", "/var/lib/docker", "Absolute path to the Docker state root directory (default: /var/lib/docker)") var dockerRunDir = flag.String("docker_run", "/var/run/docker", "Absolute path to the Docker run directory (default: /var/run/docker)") +// Regexp that identifies docker cgroups, containers started with +// --cgroup-parent have another prefix than 'docker' +var dockerCgroupRegexp = regexp.MustCompile(`.+-([a-z0-9]{64})\.scope$`) + // TODO(vmarmol): Export run dir too for newer Dockers. // Directory holding Docker container state information. func DockerStateDir() string { @@ -119,27 +123,30 @@ func ContainerNameToDockerId(name string) string { // Turn systemd cgroup name into Docker ID. if UseSystemd() { - id = strings.TrimPrefix(id, "docker-") - id = strings.TrimSuffix(id, ".scope") + if matches := dockerCgroupRegexp.FindStringSubmatch(id); matches != nil { + id = matches[1] + } } return id } -// Returns a full container name for the specified Docker ID. -func FullContainerName(dockerId string) string { - // Add the full container name. +func isContainerName(name string) bool { if UseSystemd() { - return path.Join("/system.slice", fmt.Sprintf("docker-%s.scope", dockerId)) - } else { - return path.Join("/docker", dockerId) + return dockerCgroupRegexp.MatchString(path.Base(name)) } + return true } // Docker handles all containers under /docker func (self *dockerFactory) CanHandleAndAccept(name string) (bool, bool, error) { // docker factory accepts all containers it can handle. canAccept := true + + if !isContainerName(name) { + return false, canAccept, fmt.Errorf("invalid container name") + } + // Check if the container is known to docker and it is active. id := ContainerNameToDockerId(name) diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/fsHandler.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/fsHandler.go new file mode 100644 index 0000000000000..b55634a70b9b1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/fsHandler.go @@ -0,0 +1,109 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Handler for Docker containers. +package docker + +import ( + "sync" + "time" + + "github.com/golang/glog" + "github.com/google/cadvisor/fs" +) + +type fsHandler interface { + start() + usage() uint64 + stop() +} + +type realFsHandler struct { + sync.RWMutex + lastUpdate time.Time + usageBytes uint64 + period time.Duration + storageDirs []string + fsInfo fs.FsInfo + // Tells the container to stop. + stopChan chan struct{} +} + +const longDu = time.Second + +var _ fsHandler = &realFsHandler{} + +func newFsHandler(period time.Duration, storageDirs []string, fsInfo fs.FsInfo) fsHandler { + return &realFsHandler{ + lastUpdate: time.Time{}, + usageBytes: 0, + period: period, + storageDirs: storageDirs, + fsInfo: fsInfo, + stopChan: make(chan struct{}, 1), + } +} + +func (fh *realFsHandler) needsUpdate() bool { + return time.Now().After(fh.lastUpdate.Add(fh.period)) +} + +func (fh *realFsHandler) update() error { + var usage uint64 + for _, dir := range fh.storageDirs { + // TODO(Vishh): Add support for external mounts. + dirUsage, err := fh.fsInfo.GetDirUsage(dir) + if err != nil { + return err + } + usage += dirUsage + } + fh.Lock() + defer fh.Unlock() + fh.lastUpdate = time.Now() + fh.usageBytes = usage + return nil +} + +func (fh *realFsHandler) trackUsage() { + for { + start := time.Now() + if _, ok := <-fh.stopChan; !ok { + return + } + if err := fh.update(); err != nil { + glog.V(2).Infof("failed to collect filesystem stats - %v", err) + } + duration := time.Since(start) + if duration > longDu { + glog.V(3).Infof("`du` on following dirs took %v: %v", duration, fh.storageDirs) + } + next := start.Add(fh.period) + time.Sleep(next.Sub(time.Now())) + } +} + +func (fh *realFsHandler) start() { + go fh.trackUsage() +} + +func (fh *realFsHandler) stop() { + close(fh.stopChan) +} + +func (fh *realFsHandler) usage() uint64 { + fh.RLock() + defer fh.RUnlock() + return fh.usageBytes +} diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go index c2ff1af697dfc..6ae205a4d8820 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go @@ -25,7 +25,7 @@ import ( "github.com/docker/libcontainer/cgroups" cgroup_fs "github.com/docker/libcontainer/cgroups/fs" libcontainerConfigs "github.com/docker/libcontainer/configs" - "github.com/fsouza/go-dockerclient" + docker "github.com/fsouza/go-dockerclient" "github.com/google/cadvisor/container" containerLibcontainer "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/fs" @@ -83,6 +83,9 @@ type dockerContainerHandler struct { // The network mode of the container networkMode string + + // Filesystem handler. + fsHandler fsHandler } func newDockerContainerHandler( @@ -114,6 +117,9 @@ func newDockerContainerHandler( } id := ContainerNameToDockerId(name) + + storageDirs := []string{path.Join(*dockerRootDir, pathToAufsDir, id)} + handler := &dockerContainerHandler{ id: id, client: client, @@ -124,8 +130,13 @@ func newDockerContainerHandler( usesAufsDriver: usesAufsDriver, fsInfo: fsInfo, rootFs: rootFs, + storageDirs: storageDirs, + fsHandler: newFsHandler(time.Minute, storageDirs, fsInfo), + } + + if usesAufsDriver { + handler.fsHandler.start() } - handler.storageDirs = append(handler.storageDirs, path.Join(*dockerRootDir, pathToAufsDir, id)) // We assume that if Inspect fails then the container is not known to docker. ctnr, err := client.InspectContainer(id) @@ -136,8 +147,7 @@ func newDockerContainerHandler( handler.pid = ctnr.State.Pid // Add the name and bare ID as aliases of the container. - handler.aliases = append(handler.aliases, strings.TrimPrefix(ctnr.Name, "/")) - handler.aliases = append(handler.aliases, id) + handler.aliases = append(handler.aliases, strings.TrimPrefix(ctnr.Name, "/"), id) handler.labels = ctnr.Config.Labels handler.image = ctnr.Config.Image handler.networkMode = ctnr.HostConfig.NetworkMode @@ -256,16 +266,7 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit} - var usage uint64 = 0 - for _, dir := range self.storageDirs { - // TODO(Vishh): Add support for external mounts. - dirUsage, err := self.fsInfo.GetDirUsage(dir) - if err != nil { - return err - } - usage += dirUsage - } - fsStat.Usage = usage + fsStat.Usage = self.fsHandler.usage() stats.Filesystem = append(stats.Filesystem, fsStat) return nil @@ -295,32 +296,8 @@ func (self *dockerContainerHandler) GetStats() (*info.ContainerStats, error) { } func (self *dockerContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) { - if self.name != "/docker" { - return []info.ContainerReference{}, nil - } - opt := docker.ListContainersOptions{ - All: true, - } - containers, err := self.client.ListContainers(opt) - if err != nil { - return nil, err - } - - ret := make([]info.ContainerReference, 0, len(containers)+1) - for _, c := range containers { - if !strings.HasPrefix(c.Status, "Up ") { - continue - } - - ref := info.ContainerReference{ - Name: FullContainerName(c.ID), - Aliases: append(c.Names, c.ID), - Namespace: DockerNamespace, - } - ret = append(ret, ref) - } - - return ret, nil + // No-op for Docker driver. + return []info.ContainerReference{}, nil } func (self *dockerContainerHandler) GetCgroupPath(resource string) (string, error) { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go index a3b2e50a1f6d3..0fa4315b02e39 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/libcontainer/helpers.go @@ -97,6 +97,20 @@ func GetStats(cgroupManager cgroups.Manager, rootFs string, pid int) (*info.Cont } else { stats.Network.Interfaces = append(stats.Network.Interfaces, netStats...) } + + t, err := tcpStatsFromProc(rootFs, pid, "net/tcp") + if err != nil { + glog.V(2).Infof("Unable to get tcp stats from pid %d: %v", pid, err) + } else { + stats.Network.Tcp = t + } + + t6, err := tcpStatsFromProc(rootFs, pid, "net/tcp6") + if err != nil { + glog.V(2).Infof("Unable to get tcp6 stats from pid %d: %v", pid, err) + } else { + stats.Network.Tcp6 = t6 + } } // For backwards compatibility. @@ -173,6 +187,78 @@ func scanInterfaceStats(netStatsFile string) ([]info.InterfaceStats, error) { return stats, nil } +func tcpStatsFromProc(rootFs string, pid int, file string) (info.TcpStat, error) { + tcpStatsFile := path.Join(rootFs, "proc", strconv.Itoa(pid), file) + + tcpStats, err := scanTcpStats(tcpStatsFile) + if err != nil { + return tcpStats, fmt.Errorf("couldn't read tcp stats: %v", err) + } + + return tcpStats, nil +} + +func scanTcpStats(tcpStatsFile string) (info.TcpStat, error) { + + var stats info.TcpStat + + data, err := ioutil.ReadFile(tcpStatsFile) + if err != nil { + return stats, fmt.Errorf("failure opening %s: %v", tcpStatsFile, err) + } + + tcpStatLineRE, _ := regexp.Compile("[0-9:].*") + + tcpStateMap := map[string]uint64{ + "01": 0, //ESTABLISHED + "02": 0, //SYN_SENT + "03": 0, //SYN_RECV + "04": 0, //FIN_WAIT1 + "05": 0, //FIN_WAIT2 + "06": 0, //TIME_WAIT + "07": 0, //CLOSE + "08": 0, //CLOSE_WAIT + "09": 0, //LAST_ACK + "0A": 0, //LISTEN + "0B": 0, //CLOSING + } + + reader := strings.NewReader(string(data)) + scanner := bufio.NewScanner(reader) + + scanner.Split(bufio.ScanLines) + + for scanner.Scan() { + + line := scanner.Text() + //skip header + matched := tcpStatLineRE.MatchString(line) + + if matched { + state := strings.Fields(line) + //#file header tcp state is the 4 filed: + //sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + tcpStateMap[state[3]]++ + } + } + + stats = info.TcpStat{ + Established: tcpStateMap["01"], + SynSent: tcpStateMap["02"], + SynRecv: tcpStateMap["03"], + FinWait1: tcpStateMap["04"], + FinWait2: tcpStateMap["05"], + TimeWait: tcpStateMap["06"], + Close: tcpStateMap["07"], + CloseWait: tcpStateMap["08"], + LastAck: tcpStateMap["09"], + Listen: tcpStateMap["0A"], + Closing: tcpStateMap["0B"], + } + + return stats, nil +} + func GetProcesses(cgroupManager cgroups.Manager) ([]int, error) { pids, err := cgroupManager.GetPids() if err != nil { @@ -262,6 +348,7 @@ func toContainerStats1(s *cgroups.Stats, ret *info.ContainerStats) { func toContainerStats2(s *cgroups.Stats, ret *info.ContainerStats) { ret.Memory.Usage = s.MemoryStats.Usage + ret.Memory.Failcnt = s.MemoryStats.Failcnt if v, ok := s.MemoryStats.Stats["pgfault"]; ok { ret.Memory.ContainerData.Pgfault = v ret.Memory.HierarchicalData.Pgfault = v diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/fs/fs.go b/Godeps/_workspace/src/github.com/google/cadvisor/fs/fs.go index 00c0645661f24..ebf8273b03a36 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/fs/fs.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/fs/fs.go @@ -282,7 +282,7 @@ func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) { } func (self *RealFsInfo) GetDirUsage(dir string) (uint64, error) { - out, err := exec.Command("du", "-s", dir).CombinedOutput() + out, err := exec.Command("nice", "-n", "19", "du", "-s", dir).CombinedOutput() if err != nil { return 0, fmt.Errorf("du command failed on %s with output %s - %s", dir, out, err) } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/http/handlers.go b/Godeps/_workspace/src/github.com/google/cadvisor/http/handlers.go index 8b87bedbb5f1b..c03b24aa90ad9 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/http/handlers.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/http/handlers.go @@ -31,7 +31,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func RegisterHandlers(mux httpMux.Mux, containerManager manager.Manager, httpAuthFile, httpAuthRealm, httpDigestFile, httpDigestRealm, prometheusEndpoint string) error { +func RegisterHandlers(mux httpMux.Mux, containerManager manager.Manager, httpAuthFile, httpAuthRealm, httpDigestFile, httpDigestRealm string) error { // Basic health handler. if err := healthz.RegisterHandler(mux); err != nil { return fmt.Errorf("failed to register healthz handler: %s", err) @@ -85,13 +85,15 @@ func RegisterHandlers(mux httpMux.Mux, containerManager manager.Manager, httpAut } } - collector := metrics.NewPrometheusCollector(containerManager) - prometheus.MustRegister(collector) - http.Handle(prometheusEndpoint, prometheus.Handler()) - return nil } +func RegisterPrometheusHandler(mux httpMux.Mux, containerManager manager.Manager, prometheusEndpoint string, containerNameToLabelsFunc metrics.ContainerNameToLabelsFunc) { + collector := metrics.NewPrometheusCollector(containerManager, containerNameToLabelsFunc) + prometheus.MustRegister(collector) + mux.Handle(prometheusEndpoint, prometheus.Handler()) +} + func staticHandlerNoAuth(w http.ResponseWriter, r *http.Request) { err := static.HandleRequest(w, r.URL) if err != nil { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go index 3ca17c461f65d..0994ad9d7433f 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go @@ -312,6 +312,8 @@ type MemoryStats struct { // Units: Bytes. WorkingSet uint64 `json:"working_set"` + Failcnt uint64 `json:"failcnt"` + ContainerData MemoryStatsMemoryData `json:"container_data,omitempty"` HierarchicalData MemoryStatsMemoryData `json:"hierarchical_data,omitempty"` } @@ -345,6 +347,35 @@ type InterfaceStats struct { type NetworkStats struct { InterfaceStats `json:",inline"` Interfaces []InterfaceStats `json:"interfaces,omitempty"` + // TCP connection stats (Established, Listen...) + Tcp TcpStat `json:"tcp"` + // TCP6 connection stats (Established, Listen...) + Tcp6 TcpStat `json:"tcp6"` +} + +type TcpStat struct { + //Count of TCP connections in state "Established" + Established uint64 + //Count of TCP connections in state "Syn_Sent" + SynSent uint64 + //Count of TCP connections in state "Syn_Recv" + SynRecv uint64 + //Count of TCP connections in state "Fin_Wait1" + FinWait1 uint64 + //Count of TCP connections in state "Fin_Wait2" + FinWait2 uint64 + //Count of TCP connections in state "Time_Wait + TimeWait uint64 + //Count of TCP connections in state "Close" + Close uint64 + //Count of TCP connections in state "Close_Wait" + CloseWait uint64 + //Count of TCP connections in state "Listen_Ack" + LastAck uint64 + //Count of TCP connections in state "Listen" + Listen uint64 + //Count of TCP connections in state "Closing" + Closing uint64 } type FsStats struct { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/machine.go b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/machine.go index dc07ffa67a241..7f50ce3b244fb 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/machine.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/machine.go @@ -179,6 +179,8 @@ type VersionInfo struct { // cAdvisor version. CadvisorVersion string `json:"cadvisor_version"` + // cAdvisor git revision. + CadvisorRevision string `json:"cadvisor_revision"` } type MachineInfoFactory interface { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go b/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go index cbf8c2525faaa..3c854260bd2c2 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/container.go @@ -206,9 +206,27 @@ type ProcessInfo struct { Cmd string `json:"cmd"` } +type TcpStat struct { + Established uint64 + SynSent uint64 + SynRecv uint64 + FinWait1 uint64 + FinWait2 uint64 + TimeWait uint64 + Close uint64 + CloseWait uint64 + LastAck uint64 + Listen uint64 + Closing uint64 +} + type NetworkStats struct { // Network stats by interface. Interfaces []v1.InterfaceStats `json:"interfaces,omitempty"` + // TCP connection stats (Established, Listen...) + Tcp TcpStat `json:"tcp"` + // TCP6 connection stats (Established, Listen...) + Tcp6 TcpStat `json:"tcp6"` } // Instantaneous CPU stats diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/manager/container.go b/Godeps/_workspace/src/github.com/google/cadvisor/manager/container.go index 0dca1b952736c..6f530da13db08 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/manager/container.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/manager/container.go @@ -42,7 +42,7 @@ import ( // Housekeeping interval. var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings") -var cgroupPathRegExp = regexp.MustCompile(".*devices:(.*?)[,;$].*") +var cgroupPathRegExp = regexp.MustCompile(".*devices.*:(.*?)[,;$].*") // Decay value used for load average smoothing. Interval length of 10 seconds is used. var loadDecay = math.Exp(float64(-1 * (*HousekeepingInterval).Seconds() / 10)) diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/manager/machine.go b/Godeps/_workspace/src/github.com/google/cadvisor/manager/machine.go index 5ff60b65ba125..5e288bc6a4513 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/manager/machine.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/manager/machine.go @@ -122,7 +122,8 @@ func getVersionInfo() (*info.VersionInfo, error) { KernelVersion: kernel_version, ContainerOsVersion: container_os, DockerVersion: docker_version, - CadvisorVersion: version.VERSION, + CadvisorVersion: version.Info["version"], + CadvisorRevision: version.Info["revision"], }, nil } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go b/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go index a0c05779b45f5..450b8cd97a193 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/metrics/prometheus.go @@ -24,9 +24,13 @@ import ( ) // This will usually be manager.Manager, but can be swapped out for testing. -type subcontainersInfoProvider interface { +type infoProvider interface { // Get information about all subcontainers of the specified container (includes self). SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error) + // Get information about the version. + GetVersionInfo() (*info.VersionInfo, error) + // Get information about the machine. + GetMachineInfo() (*info.MachineInfo, error) } // metricValue describes a single metric value for a given set of label values @@ -60,21 +64,25 @@ type containerMetric struct { getValues func(s *info.ContainerStats) metricValues } -func (cm *containerMetric) desc() *prometheus.Desc { - return prometheus.NewDesc(cm.name, cm.help, append([]string{"name", "id", "image"}, cm.extraLabels...), nil) +func (cm *containerMetric) desc(baseLabels []string) *prometheus.Desc { + return prometheus.NewDesc(cm.name, cm.help, append(baseLabels, cm.extraLabels...), nil) } +type ContainerNameToLabelsFunc func(containerName string) map[string]string + // PrometheusCollector implements prometheus.Collector. type PrometheusCollector struct { - infoProvider subcontainersInfoProvider - errors prometheus.Gauge - containerMetrics []containerMetric + infoProvider infoProvider + errors prometheus.Gauge + containerMetrics []containerMetric + containerNameToLabels ContainerNameToLabelsFunc } // NewPrometheusCollector returns a new PrometheusCollector. -func NewPrometheusCollector(infoProvider subcontainersInfoProvider) *PrometheusCollector { +func NewPrometheusCollector(infoProvider infoProvider, f ContainerNameToLabelsFunc) *PrometheusCollector { c := &PrometheusCollector{ - infoProvider: infoProvider, + infoProvider: infoProvider, + containerNameToLabels: f, errors: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "container", Name: "scrape_error", @@ -117,6 +125,13 @@ func NewPrometheusCollector(infoProvider subcontainersInfoProvider) *PrometheusC } return values }, + }, { + name: "container_memory_failcnt", + help: "Number of memory usage hits limits", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Failcnt)}} + }, }, { name: "container_memory_usage_bytes", help: "Current memory usage in bytes.", @@ -441,18 +456,34 @@ func NewPrometheusCollector(infoProvider subcontainersInfoProvider) *PrometheusC return c } +var ( + versionInfoDesc = prometheus.NewDesc("cadvisor_version_info", "A metric with a constant '1' value labeled by kernel version, OS version, docker version, cadvisor version & cadvisor revision.", []string{"kernelVersion", "osVersion", "dockerVersion", "cadvisorVersion", "cadvisorRevision"}, nil) + machineInfoCoresDesc = prometheus.NewDesc("machine_cpu_cores", "Number of CPU cores on the machine.", nil, nil) + machineInfoMemoryDesc = prometheus.NewDesc("machine_memory_bytes", "Amount of memory installed on the machine.", nil, nil) +) + // Describe describes all the metrics ever exported by cadvisor. It // implements prometheus.PrometheusCollector. func (c *PrometheusCollector) Describe(ch chan<- *prometheus.Desc) { c.errors.Describe(ch) for _, cm := range c.containerMetrics { - ch <- cm.desc() + ch <- cm.desc([]string{}) } + ch <- versionInfoDesc + ch <- machineInfoCoresDesc + ch <- machineInfoMemoryDesc } // Collect fetches the stats from all containers and delivers them as // Prometheus metrics. It implements prometheus.PrometheusCollector. func (c *PrometheusCollector) Collect(ch chan<- prometheus.Metric) { + c.collectMachineInfo(ch) + c.collectVersionInfo(ch) + c.collectContainersInfo(ch) + c.errors.Collect(ch) +} + +func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) { containers, err := c.infoProvider.SubcontainersInfo("/", &info.ContainerInfoRequest{NumStats: 1}) if err != nil { c.errors.Set(1) @@ -460,20 +491,82 @@ func (c *PrometheusCollector) Collect(ch chan<- prometheus.Metric) { return } for _, container := range containers { + baseLabels := []string{"id"} id := container.Name name := id if len(container.Aliases) > 0 { name = container.Aliases[0] + baseLabels = append(baseLabels, "name") } image := container.Spec.Image - stats := container.Stats[0] + if len(image) > 0 { + baseLabels = append(baseLabels, "image") + } + baseLabelValues := []string{id, name, image}[:len(baseLabels)] + + if c.containerNameToLabels != nil { + newLabels := c.containerNameToLabels(name) + for k, v := range newLabels { + baseLabels = append(baseLabels, k) + baseLabelValues = append(baseLabelValues, v) + } + } + + // Container spec + desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...) + + if container.Spec.HasCpu { + desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", baseLabels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), baseLabelValues...) + } + + if container.Spec.HasMemory { + desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", baseLabels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), baseLabelValues...) + desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", baseLabels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), baseLabelValues...) + } + // Now for the actual metrics + stats := container.Stats[0] for _, cm := range c.containerMetrics { - desc := cm.desc() + desc := cm.desc(baseLabels) for _, metricValue := range cm.getValues(stats) { - ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append([]string{name, id, image}, metricValue.labels...)...) + ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(baseLabelValues, metricValue.labels...)...) } } } - c.errors.Collect(ch) +} + +func (c *PrometheusCollector) collectVersionInfo(ch chan<- prometheus.Metric) { + versionInfo, err := c.infoProvider.GetVersionInfo() + if err != nil { + c.errors.Set(1) + glog.Warningf("Couldn't get version info: %s", err) + return + } + ch <- prometheus.MustNewConstMetric(versionInfoDesc, prometheus.GaugeValue, 1, []string{versionInfo.KernelVersion, versionInfo.ContainerOsVersion, versionInfo.DockerVersion, versionInfo.CadvisorVersion, versionInfo.CadvisorRevision}...) +} + +func (c *PrometheusCollector) collectMachineInfo(ch chan<- prometheus.Metric) { + machineInfo, err := c.infoProvider.GetMachineInfo() + if err != nil { + c.errors.Set(1) + glog.Warningf("Couldn't get machine info: %s", err) + return + } + ch <- prometheus.MustNewConstMetric(machineInfoCoresDesc, prometheus.GaugeValue, float64(machineInfo.NumCores)) + ch <- prometheus.MustNewConstMetric(machineInfoMemoryDesc, prometheus.GaugeValue, float64(machineInfo.MemoryCapacity)) +} + +// Size after which we consider memory to be "unlimited". This is not +// MaxInt64 due to rounding by the kernel. +const maxMemorySize = uint64(1 << 62) + +func specMemoryValue(v uint64) float64 { + if v > maxMemorySize { + return 0 + } + return float64(v) } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/pages/docker.go b/Godeps/_workspace/src/github.com/google/cadvisor/pages/docker.go index 7e822c382ccce..5712ef439bc5c 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/pages/docker.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/pages/docker.go @@ -53,11 +53,11 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error start := time.Now() // The container name is the path after the handler - containerName := u.Path[len(DockerPage):] - rootDir := getRootDir(u.Path) + containerName := u.Path[len(DockerPage)-1:] + rootDir := getRootDir(containerName) var data *pageData - if containerName == "" { + if containerName == "/" { // Get the containers. reqParams := info.ContainerInfoRequest{ NumStats: 0, @@ -70,7 +70,7 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error for _, cont := range conts { subcontainers = append(subcontainers, link{ Text: getContainerDisplayName(cont.ContainerReference), - Link: path.Join("/docker", docker.ContainerNameToDockerId(cont.ContainerReference.Name)), + Link: path.Join(rootDir, DockerPage, docker.ContainerNameToDockerId(cont.ContainerReference.Name)), }) } @@ -93,7 +93,7 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error ParentContainers: []link{ { Text: dockerContainersText, - Link: DockerPage, + Link: path.Join(rootDir, DockerPage), }}, Subcontainers: subcontainers, Root: rootDir, @@ -106,7 +106,7 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error reqParams := info.ContainerInfoRequest{ NumStats: 60, } - cont, err := m.DockerContainer(containerName, &reqParams) + cont, err := m.DockerContainer(containerName[1:], &reqParams) if err != nil { return fmt.Errorf("failed to get container %q with error: %v", containerName, err) } @@ -115,12 +115,12 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error // Make a list of the parent containers and their links var parentContainers []link parentContainers = append(parentContainers, link{ - Text: "Docker containers", - Link: DockerPage, + Text: "Docker Containers", + Link: path.Join(rootDir, DockerPage), }) parentContainers = append(parentContainers, link{ Text: displayName, - Link: path.Join(DockerPage, docker.ContainerNameToDockerId(cont.Name)), + Link: path.Join(rootDir, DockerPage, docker.ContainerNameToDockerId(cont.Name)), }) // Get the MachineInfo diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/bigquery.go b/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/bigquery.go index 9f65f99ecf04a..2d884f5f491a9 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/bigquery.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/bigquery.go @@ -15,10 +15,10 @@ package bigquery import ( - bigquery "code.google.com/p/google-api-go-client/bigquery/v2" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/storage" "github.com/google/cadvisor/storage/bigquery/client" + bigquery "google.golang.org/api/bigquery/v2" ) type bigqueryStorage struct { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/client/client.go b/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/client/client.go index af7736b565467..db01528c47b7a 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/client/client.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/storage/bigquery/client/client.go @@ -20,9 +20,9 @@ import ( "io/ioutil" "strings" - bigquery "code.google.com/p/google-api-go-client/bigquery/v2" "golang.org/x/oauth2" "golang.org/x/oauth2/jwt" + bigquery "google.golang.org/api/bigquery/v2" ) var ( diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/storage/elasticsearch/elasticsearch.go b/Godeps/_workspace/src/github.com/google/cadvisor/storage/elasticsearch/elasticsearch.go new file mode 100644 index 0000000000000..96c62f262baca --- /dev/null +++ b/Godeps/_workspace/src/github.com/google/cadvisor/storage/elasticsearch/elasticsearch.go @@ -0,0 +1,127 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package elasticsearch + +import ( + "fmt" + "sync" + "time" + + info "github.com/google/cadvisor/info/v1" + storage "github.com/google/cadvisor/storage" + "gopkg.in/olivere/elastic.v2" +) + +type elasticStorage struct { + client *elastic.Client + machineName string + indexName string + typeName string + lock sync.Mutex +} + +type detailSpec struct { + Timestamp int64 `json:"timestamp"` + MachineName string `json:"machine_name,omitempty"` + ContainerName string `json:"container_Name,omitempty"` + ContainerStats *info.ContainerStats `json:"container_stats,omitempty"` +} + +func (self *elasticStorage) containerStatsAndDefaultValues( + ref info.ContainerReference, stats *info.ContainerStats) *detailSpec { + timestamp := stats.Timestamp.UnixNano() / 1E3 + var containerName string + if len(ref.Aliases) > 0 { + containerName = ref.Aliases[0] + } else { + containerName = ref.Name + } + detail := &detailSpec{ + Timestamp: timestamp, + MachineName: self.machineName, + ContainerName: containerName, + ContainerStats: stats, + } + return detail +} + +func (self *elasticStorage) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error { + if stats == nil { + return nil + } + func() { + // AddStats will be invoked simultaneously from multiple threads and only one of them will perform a write. + self.lock.Lock() + defer self.lock.Unlock() + // Add some default params based on ContainerStats + detail := self.containerStatsAndDefaultValues(ref, stats) + // Index a cadvisor (using JSON serialization) + _, err := self.client.Index(). + Index(self.indexName). + Type(self.typeName). + BodyJson(detail). + Do() + if err != nil { + // Handle error + panic(fmt.Errorf("failed to write stats to ElasticSearch- %s", err)) + } + }() + return nil +} + +func (self *elasticStorage) Close() error { + self.client = nil + return nil +} + +// machineName: A unique identifier to identify the host that current cAdvisor +// instance is running on. +// ElasticHost: The host which runs ElasticSearch. +func New(machineName, + indexName, + typeName, + elasticHost string, + enableSniffer bool, +) (storage.StorageDriver, error) { + // Obtain a client and connect to the default Elasticsearch installation + // on 127.0.0.1:9200. Of course you can configure your client to connect + // to other hosts and configure it in various other ways. + client, err := elastic.NewClient( + elastic.SetHealthcheck(true), + elastic.SetSniff(enableSniffer), + elastic.SetHealthcheckInterval(30*time.Second), + elastic.SetURL(elasticHost), + ) + if err != nil { + // Handle error + panic(err) + } + + // Ping the Elasticsearch server to get e.g. the version number + info, code, err := client.Ping().URL(elasticHost).Do() + if err != nil { + // Handle error + panic(err) + } + fmt.Printf("Elasticsearch returned with code %d and version %s", code, info.Version.Number) + + ret := &elasticStorage{ + client: client, + machineName: machineName, + indexName: indexName, + typeName: typeName, + } + return ret, nil +} diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/utils/oomparser/oomparser.go b/Godeps/_workspace/src/github.com/google/cadvisor/utils/oomparser/oomparser.go index 4851e2b06eaa9..a1c9bd1c02455 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/utils/oomparser/oomparser.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/utils/oomparser/oomparser.go @@ -164,7 +164,7 @@ func (self *OomParser) StreamOoms(outStream chan *OomInstance) { } func callJournalctl() (io.ReadCloser, error) { - cmd := exec.Command("journalctl", "-f") + cmd := exec.Command("journalctl", "-k", "-f") readcloser, err := cmd.StdoutPipe() if err != nil { return nil, err diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION b/Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION new file mode 100644 index 0000000000000..66333910a4bee --- /dev/null +++ b/Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION @@ -0,0 +1 @@ +0.18.0 diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/version/version.go b/Godeps/_workspace/src/github.com/google/cadvisor/version/version.go index 97ea3e9157ff5..965c2931f0ccc 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/version/version.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/version/version.go @@ -14,5 +14,22 @@ package version -// Version of cAdvisor. -const VERSION = "0.16.0" +// Build information. Populated at build-time. +var ( + Version string + Revision string + Branch string + BuildUser string + BuildDate string + GoVersion string +) + +// Info provides the iterable version information. +var Info = map[string]string{ + "version": Version, + "revision": Revision, + "branch": Branch, + "buildUser": BuildUser, + "buildDate": BuildDate, + "goVersion": GoVersion, +} diff --git a/pkg/kubelet/cadvisor/cadvisor_linux.go b/pkg/kubelet/cadvisor/cadvisor_linux.go index ea8d50f1d9981..5b4a43a5b86d7 100644 --- a/pkg/kubelet/cadvisor/cadvisor_linux.go +++ b/pkg/kubelet/cadvisor/cadvisor_linux.go @@ -21,6 +21,7 @@ package cadvisor import ( "fmt" "net/http" + "regexp" "time" "github.com/golang/glog" @@ -79,11 +80,24 @@ func (cc *cadvisorClient) exportHTTP(port uint) error { // Register the handlers regardless as this registers the prometheus // collector properly. mux := http.NewServeMux() - err := cadvisorHttp.RegisterHandlers(mux, cc, "", "", "", "", "/metrics") + err := cadvisorHttp.RegisterHandlers(mux, cc, "", "", "", "") if err != nil { return err } + re := regexp.MustCompile(`^k8s_(?P[^_\.]+)[^_]+_(?P[^_]+)_(?P[^_]+)`) + reCaptureNames := re.SubexpNames() + cadvisorHttp.RegisterPrometheusHandler(mux, cc, "/metrics", func(name string) map[string]string { + extraLabels := map[string]string{} + matches := re.FindStringSubmatch(name) + for i, match := range matches { + if len(reCaptureNames[i]) > 0 { + extraLabels[re.SubexpNames()[i]] = match + } + } + return extraLabels + }) + // Only start the http server if port > 0 if port > 0 { serv := &http.Server{ From 3276307c7e23f921eaa84672a2e155f251ed908d Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 15 Oct 2015 20:59:33 +0100 Subject: [PATCH 2/2] libcontainer bump --- Godeps/Godeps.json | 4 +- .../docker/libcontainer/apparmor/apparmor.go | 6 +- .../libcontainer/cgroups/fs/apply_raw.go | 49 ++++++--- .../docker/libcontainer/cgroups/fs/blkio.go | 20 ++++ .../docker/libcontainer/cgroups/fs/devices.go | 11 ++ .../docker/libcontainer/cgroups/fs/hugetlb.go | 29 +++++ .../cgroups/systemd/apply_nosystemd.go | 4 - .../cgroups/systemd/apply_systemd.go | 54 +++++++--- .../docker/libcontainer/configs/cgroup.go | 14 +++ .../docker/libcontainer/configs/config.go | 7 ++ .../docker/libcontainer/configs/mount.go | 13 +++ .../docker/libcontainer/configs/namespaces.go | 31 +----- .../configs/namespaces_syscall.go | 31 ++++++ .../configs/namespaces_syscall_unsupported.go | 15 +++ .../docker/libcontainer/configs/network.go | 4 +- .../docker/libcontainer/console_linux.go | 2 +- .../docker/libcontainer/container.go | 2 +- .../docker/libcontainer/devices/devices.go | 2 +- .../github.com/docker/libcontainer/factory.go | 2 +- .../libcontainer/label/label_selinux.go | 6 ++ .../docker/libcontainer/nsenter/README.md | 27 ++++- .../docker/libcontainer/nsenter/nsexec.c | 6 ++ .../docker/libcontainer/nsinit/config.go | 7 ++ .../docker/libcontainer/nsinit/exec.go | 1 + .../docker/libcontainer/nsinit/oom.go | 3 +- .../docker/libcontainer/nsinit/pause.go | 3 +- .../docker/libcontainer/nsinit/utils.go | 12 ++- .../github.com/docker/libcontainer/process.go | 4 +- .../docker/libcontainer/process_linux.go | 3 + .../docker/libcontainer/rootfs_linux.go | 70 +++++++++++- .../libcontainer/standard_init_linux.go | 7 ++ .../docker/libcontainer/update-vendor.sh | 2 +- .../github.com/Sirupsen/logrus/CHANGELOG.md | 7 ++ .../src/github.com/Sirupsen/logrus/README.md | 102 +++++++----------- .../Sirupsen/logrus/examples/hook/hook.go | 7 +- .../github.com/Sirupsen/logrus/formatter.go | 4 + .../logrus/formatters/logstash/logstash.go | 56 ++++++++++ .../logrus/hooks/airbrake/airbrake.go | 60 +++++------ .../Sirupsen/logrus/hooks/bugsnag/bugsnag.go | 68 ++++++++++++ .../Sirupsen/logrus/json_formatter.go | 24 +++-- .../src/github.com/Sirupsen/logrus/logger.go | 84 +++++++++++---- .../Sirupsen/logrus/terminal_openbsd.go | 1 - .../Sirupsen/logrus/text_formatter.go | 12 ++- .../src/github.com/Sirupsen/logrus/writer.go | 2 +- 44 files changed, 656 insertions(+), 222 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/hugetlb.go create mode 100644 Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall.go create mode 100644 Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall_unsupported.go create mode 100644 Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/CHANGELOG.md create mode 100644 Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go create mode 100644 Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 4d2b66d06e15b..8a7ca71907082 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -216,8 +216,8 @@ }, { "ImportPath": "github.com/docker/libcontainer", - "Comment": "v1.4.0-446-gae812bd", - "Rev": "ae812bdca78084dc322037225d170e1883521d87" + "Comment": "v1.4.0-501-ga1fe3f1", + "Rev": "a1fe3f1c7ad2e8eebe6d59e573f04d2b10961cf6" }, { "ImportPath": "github.com/docker/spdystream", diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/apparmor/apparmor.go b/Godeps/_workspace/src/github.com/docker/libcontainer/apparmor/apparmor.go index 3be3294d85dad..18cedf6a191a2 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/apparmor/apparmor.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/apparmor/apparmor.go @@ -14,8 +14,10 @@ import ( func IsEnabled() bool { if _, err := os.Stat("/sys/kernel/security/apparmor"); err == nil && os.Getenv("container") == "" { - buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled") - return err == nil && len(buf) > 1 && buf[0] == 'Y' + if _, err = os.Stat("/sbin/apparmor_parser"); err == nil { + buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled") + return err == nil && len(buf) > 1 && buf[0] == 'Y' + } } return false } diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/apply_raw.go b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/apply_raw.go index 0a2d76bcd4e8e..fa6478b5f32ba 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/apply_raw.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/apply_raw.go @@ -1,6 +1,8 @@ package fs import ( + "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -19,6 +21,7 @@ var ( "cpuset": &CpusetGroup{}, "cpuacct": &CpuacctGroup{}, "blkio": &BlkioGroup{}, + "hugetlb": &HugetlbGroup{}, "perf_event": &PerfEventGroup{}, "freezer": &FreezerGroup{}, } @@ -75,10 +78,13 @@ type data struct { } func (m *Manager) Apply(pid int) error { + if m.Cgroups == nil { return nil } + var c = m.Cgroups + d, err := getCgroupData(m.Cgroups, pid) if err != nil { return err @@ -108,6 +114,12 @@ func (m *Manager) Apply(pid int) error { } m.Paths = paths + if paths["cpu"] != "" { + if err := CheckCpushares(paths["cpu"], c.CpuShares); err != nil { + return err + } + } + return nil } @@ -119,19 +131,6 @@ func (m *Manager) GetPaths() map[string]string { return m.Paths } -// Symmetrical public function to update device based cgroups. Also available -// in the systemd implementation. -func ApplyDevices(c *configs.Cgroup, pid int) error { - d, err := getCgroupData(c, pid) - if err != nil { - return err - } - - devices := subsystems["devices"] - - return devices.Apply(d) -} - func (m *Manager) GetStats() (*cgroups.Stats, error) { stats := cgroups.NewStats() for name, path := range m.Paths { @@ -280,3 +279,27 @@ func removePath(p string, err error) error { } return nil } + +func CheckCpushares(path string, c int64) error { + var cpuShares int64 + + fd, err := os.Open(filepath.Join(path, "cpu.shares")) + if err != nil { + return err + } + defer fd.Close() + + _, err = fmt.Fscanf(fd, "%d", &cpuShares) + if err != nil && err != io.EOF { + return err + } + if c != 0 { + if c > cpuShares { + return fmt.Errorf("The maximum allowed cpu-shares is %d", cpuShares) + } else if c < cpuShares { + return fmt.Errorf("The minimum allowed cpu-shares is %d", cpuShares) + } + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/blkio.go b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/blkio.go index 238fe8044d877..06f0a3b2cd9a2 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/blkio.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/blkio.go @@ -40,6 +40,26 @@ func (s *BlkioGroup) Set(path string, cgroup *configs.Cgroup) error { return err } } + if cgroup.BlkioThrottleReadBpsDevice != "" { + if err := writeFile(path, "blkio.throttle.read_bps_device", cgroup.BlkioThrottleReadBpsDevice); err != nil { + return err + } + } + if cgroup.BlkioThrottleWriteBpsDevice != "" { + if err := writeFile(path, "blkio.throttle.write_bps_device", cgroup.BlkioThrottleWriteBpsDevice); err != nil { + return err + } + } + if cgroup.BlkioThrottleReadIOpsDevice != "" { + if err := writeFile(path, "blkio.throttle.read_iops_device", cgroup.BlkioThrottleReadIOpsDevice); err != nil { + return err + } + } + if cgroup.BlkioThrottleWriteIOpsDevice != "" { + if err := writeFile(path, "blkio.throttle.write_iops_device", cgroup.BlkioThrottleWriteIOpsDevice); err != nil { + return err + } + } return nil } diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/devices.go b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/devices.go index 16e00b1c73b2b..be588d67a148c 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/devices.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/devices.go @@ -32,6 +32,17 @@ func (s *DevicesGroup) Set(path string, cgroup *configs.Cgroup) error { return err } } + return nil + } + + if err := writeFile(path, "devices.allow", "a"); err != nil { + return err + } + + for _, dev := range cgroup.DeniedDevices { + if err := writeFile(path, "devices.deny", dev.CgroupString()); err != nil { + return err + } } return nil diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/hugetlb.go b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/hugetlb.go new file mode 100644 index 0000000000000..8defdd1b914aa --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/fs/hugetlb.go @@ -0,0 +1,29 @@ +package fs + +import ( + "github.com/docker/libcontainer/cgroups" + "github.com/docker/libcontainer/configs" +) + +type HugetlbGroup struct { +} + +func (s *HugetlbGroup) Apply(d *data) error { + // we just want to join this group even though we don't set anything + if _, err := d.join("hugetlb"); err != nil && !cgroups.IsNotFound(err) { + return err + } + return nil +} + +func (s *HugetlbGroup) Set(path string, cgroup *configs.Cgroup) error { + return nil +} + +func (s *HugetlbGroup) Remove(d *data) error { + return removePath(d.path("hugetlb")) +} + +func (s *HugetlbGroup) GetStats(path string, stats *cgroups.Stats) error { + return nil +} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_nosystemd.go b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_nosystemd.go index 95ed4ea7eb280..9b605b3c05dfa 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_nosystemd.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_nosystemd.go @@ -46,10 +46,6 @@ func (m *Manager) Freeze(state configs.FreezerState) error { return fmt.Errorf("Systemd not supported") } -func ApplyDevices(c *configs.Cgroup, pid int) error { - return fmt.Errorf("Systemd not supported") -} - func Freeze(c *configs.Cgroup, state configs.FreezerState) error { return fmt.Errorf("Systemd not supported") } diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_systemd.go b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_systemd.go index 77e39b058f726..2ba10cbb3430d 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_systemd.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/cgroups/systemd/apply_systemd.go @@ -38,6 +38,7 @@ var subsystems = map[string]subsystem{ "cpuset": &fs.CpusetGroup{}, "cpuacct": &fs.CpuacctGroup{}, "blkio": &fs.BlkioGroup{}, + "hugetlb": &fs.HugetlbGroup{}, "perf_event": &fs.PerfEventGroup{}, "freezer": &fs.FreezerGroup{}, } @@ -235,9 +236,14 @@ func (m *Manager) Apply(pid int) error { } paths[sysname] = subsystemPath } - m.Paths = paths + if paths["cpu"] != "" { + if err := fs.CheckCpushares(paths["cpu"], c.CpuShares); err != nil { + return err + } + } + return nil } @@ -357,7 +363,17 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) { } func (m *Manager) Set(container *configs.Config) error { - panic("not implemented") + for name, path := range m.Paths { + sys, ok := subsystems[name] + if !ok || !cgroups.PathExists(path) { + continue + } + if err := sys.Set(path, container.Cgroups); err != nil { + return err + } + } + + return nil } func getUnitName(c *configs.Cgroup) string { @@ -369,7 +385,7 @@ func getUnitName(c *configs.Cgroup) string { // * Support for wildcards to allow /dev/pts support // // The second is available in more recent systemd as "char-pts", but not in e.g. v208 which is -// in wide use. When both these are availalable we will be able to switch, but need to keep the old +// in wide use. When both these are available we will be able to switch, but need to keep the old // implementation for backwards compat. // // Note: we can't use systemd to set up the initial limits, and then change the cgroup @@ -382,17 +398,7 @@ func joinDevices(c *configs.Cgroup, pid int) error { } devices := subsystems["devices"] - if err := devices.Set(path, c); err != nil { - return err - } - - return nil -} - -// Symmetrical public function to update device based cgroups. Also available -// in the fs implementation. -func ApplyDevices(c *configs.Cgroup, pid int) error { - return joinDevices(c, pid) + return devices.Set(path, c) } func joinMemory(c *configs.Cgroup, pid int) error { @@ -438,6 +444,26 @@ func joinBlkio(c *configs.Cgroup, pid int) error { return err } } + if c.BlkioThrottleReadBpsDevice != "" { + if err := writeFile(path, "blkio.throttle.read_bps_device", c.BlkioThrottleReadBpsDevice); err != nil { + return err + } + } + if c.BlkioThrottleWriteBpsDevice != "" { + if err := writeFile(path, "blkio.throttle.write_bps_device", c.BlkioThrottleWriteBpsDevice); err != nil { + return err + } + } + if c.BlkioThrottleReadIOpsDevice != "" { + if err := writeFile(path, "blkio.throttle.read_iops_device", c.BlkioThrottleReadIOpsDevice); err != nil { + return err + } + } + if c.BlkioThrottleWriteIOpsDevice != "" { + if err := writeFile(path, "blkio.throttle.write_iops_device", c.BlkioThrottleWriteIOpsDevice); err != nil { + return err + } + } return nil } diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/cgroup.go b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/cgroup.go index 8a699ac107520..8a161fcff66f9 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/cgroup.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/cgroup.go @@ -19,6 +19,8 @@ type Cgroup struct { AllowedDevices []*Device `json:"allowed_devices"` + DeniedDevices []*Device `json:"denied_devices"` + // Memory limit (in bytes) Memory int64 `json:"memory"` @@ -43,6 +45,18 @@ type Cgroup struct { // MEM to use CpusetMems string `json:"cpuset_mems"` + // IO read rate limit per cgroup per device, bytes per second. + BlkioThrottleReadBpsDevice string `json:"blkio_throttle_read_bps_device"` + + // IO write rate limit per cgroup per divice, bytes per second. + BlkioThrottleWriteBpsDevice string `json:"blkio_throttle_write_bps_device"` + + // IO read rate limit per cgroup per device, IO per second. + BlkioThrottleReadIOpsDevice string `json:"blkio_throttle_read_iops_device"` + + // IO write rate limit per cgroup per device, IO per second. + BlkioThrottleWriteIOpsDevice string `json:"blkio_throttle_write_iops_device"` + // Specifies per cgroup weight, range is from 10 to 1000. BlkioWeight int64 `json:"blkio_weight"` diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/config.go b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/config.go index b07f252b5e620..2c311a0cdf1ff 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/config.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/config.go @@ -37,6 +37,9 @@ type Config struct { // bind mounts are writtable. Readonlyfs bool `json:"readonlyfs"` + // Privatefs will mount the container's rootfs as private where mount points from the parent will not propogate + Privatefs bool `json:"privatefs"` + // Mounts specify additional source and destination paths that will be mounted inside the container's // rootfs and mount namespace if specified Mounts []*Mount `json:"mounts"` @@ -96,6 +99,10 @@ type Config struct { // ReadonlyPaths specifies paths within the container's rootfs to remount as read-only // so that these files prevent any writes. ReadonlyPaths []string `json:"readonly_paths"` + + // SystemProperties is a map of properties and their values. It is the equivalent of using + // sysctl -w my.property.name value in Linux. + SystemProperties map[string]string `json:"system_properties"` } // Gets the root uid for the process on host which could be non-zero diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/mount.go b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/mount.go index 7b3dea3312a54..5a69f815e4e3a 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/mount.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/mount.go @@ -18,4 +18,17 @@ type Mount struct { // Relabel source if set, "z" indicates shared, "Z" indicates unshared. Relabel string `json:"relabel"` + + // Optional Command to be run before Source is mounted. + PremountCmds []Command `json:"premount_cmds"` + + // Optional Command to be run after Source is mounted. + PostmountCmds []Command `json:"postmount_cmds"` +} + +type Command struct { + Path string `json:"path"` + Args []string `json:"args"` + Env []string `json:"env"` + Dir string `json:"dir"` } diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces.go b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces.go index ac6a7fa2cdb6f..2c2a9fd20a9d2 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces.go @@ -1,9 +1,6 @@ package configs -import ( - "fmt" - "syscall" -) +import "fmt" type NamespaceType string @@ -34,10 +31,6 @@ type Namespace struct { Path string `json:"path"` } -func (n *Namespace) Syscall() int { - return namespaceInfo[n.Type] -} - func (n *Namespace) GetPath(pid int) string { if n.Path != "" { return n.Path @@ -96,25 +89,3 @@ func (n *Namespaces) index(t NamespaceType) int { func (n *Namespaces) Contains(t NamespaceType) bool { return n.index(t) != -1 } - -var namespaceInfo = map[NamespaceType]int{ - NEWNET: syscall.CLONE_NEWNET, - NEWNS: syscall.CLONE_NEWNS, - NEWUSER: syscall.CLONE_NEWUSER, - NEWIPC: syscall.CLONE_NEWIPC, - NEWUTS: syscall.CLONE_NEWUTS, - NEWPID: syscall.CLONE_NEWPID, -} - -// CloneFlags parses the container's Namespaces options to set the correct -// flags on clone, unshare. This functions returns flags only for new namespaces. -func (n *Namespaces) CloneFlags() uintptr { - var flag int - for _, v := range *n { - if v.Path != "" { - continue - } - flag |= namespaceInfo[v.Type] - } - return uintptr(flag) -} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall.go b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall.go new file mode 100644 index 0000000000000..c962999efd4bc --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall.go @@ -0,0 +1,31 @@ +// +build linux + +package configs + +import "syscall" + +func (n *Namespace) Syscall() int { + return namespaceInfo[n.Type] +} + +var namespaceInfo = map[NamespaceType]int{ + NEWNET: syscall.CLONE_NEWNET, + NEWNS: syscall.CLONE_NEWNS, + NEWUSER: syscall.CLONE_NEWUSER, + NEWIPC: syscall.CLONE_NEWIPC, + NEWUTS: syscall.CLONE_NEWUTS, + NEWPID: syscall.CLONE_NEWPID, +} + +// CloneFlags parses the container's Namespaces options to set the correct +// flags on clone, unshare. This functions returns flags only for new namespaces. +func (n *Namespaces) CloneFlags() uintptr { + var flag int + for _, v := range *n { + if v.Path != "" { + continue + } + flag |= namespaceInfo[v.Type] + } + return uintptr(flag) +} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall_unsupported.go b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall_unsupported.go new file mode 100644 index 0000000000000..1bd26bd6e63eb --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/namespaces_syscall_unsupported.go @@ -0,0 +1,15 @@ +// +build !linux + +package configs + +func (n *Namespace) Syscall() int { + panic("No namespace syscall support") + return 0 +} + +// CloneFlags parses the container's Namespaces options to set the correct +// flags on clone, unshare. This functions returns flags only for new namespaces. +func (n *Namespaces) CloneFlags() uintptr { + panic("No namespace syscall support") + return uintptr(0) +} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/network.go b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/network.go index 9d5ed7a65fa09..ccdb228e14c85 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/configs/network.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/configs/network.go @@ -2,7 +2,7 @@ package configs // Network defines configuration for a container's networking stack // -// The network configuration can be omited from a container causing the +// The network configuration can be omitted from a container causing the // container to be setup with the host's networking stack type Network struct { // Type sets the networks type, commonly veth and loopback @@ -53,7 +53,7 @@ type Network struct { // Routes can be specified to create entries in the route table as the container is started // // All of destination, source, and gateway should be either IPv4 or IPv6. -// One of the three options must be present, and ommitted entries will use their +// One of the three options must be present, and omitted entries will use their // IP family default for the route table. For IPv4 for example, setting the // gateway to 1.2.3.4 and the interface to eth0 will set up a standard // destination of 0.0.0.0(or *) when viewed in the route table. diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/console_linux.go b/Godeps/_workspace/src/github.com/docker/libcontainer/console_linux.go index afdc2976c4454..a3a0551cf688c 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/console_linux.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/console_linux.go @@ -38,7 +38,7 @@ func newConsole(uid, gid int) (Console, error) { }, nil } -// newConsoleFromPath is an internal fucntion returning an initialzied console for use inside +// newConsoleFromPath is an internal function returning an initialized console for use inside // a container's MNT namespace. func newConsoleFromPath(slavePath string) *linuxConsole { return &linuxConsole{ diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/container.go b/Godeps/_workspace/src/github.com/docker/libcontainer/container.go index 35bdfd781f3f1..a38df8269d820 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/container.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/container.go @@ -67,7 +67,7 @@ type Container interface { // State returns the current container's state information. // // errors: - // Systemerror - System erroor. + // Systemerror - System error. State() (*State, error) // Returns the current config of the container. diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/devices/devices.go b/Godeps/_workspace/src/github.com/docker/libcontainer/devices/devices.go index 537f71aff1f05..7a11eaf11bb01 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/devices/devices.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/devices/devices.go @@ -21,7 +21,7 @@ var ( ioutilReadDir = ioutil.ReadDir ) -// Given the path to a device and it's cgroup_permissions(which cannot be easilly queried) look up the information about a linux device and return that information as a Device struct. +// Given the path to a device and it's cgroup_permissions(which cannot be easily queried) look up the information about a linux device and return that information as a Device struct. func DeviceFromPath(path, permissions string) (*configs.Device, error) { fileInfo, err := osLstat(path) if err != nil { diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/factory.go b/Godeps/_workspace/src/github.com/docker/libcontainer/factory.go index 6a3706fd971fc..2b3ff85d8f773 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/factory.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/factory.go @@ -32,7 +32,7 @@ type Factory interface { // System error Load(id string) (Container, error) - // StartInitialization is an internal API to libcontainer used during the rexec of the + // StartInitialization is an internal API to libcontainer used during the reexec of the // container. // // Errors: diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/label/label_selinux.go b/Godeps/_workspace/src/github.com/docker/libcontainer/label/label_selinux.go index efe4b46967225..7bc40ddde2542 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/label/label_selinux.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/label/label_selinux.go @@ -101,9 +101,15 @@ func SetFileCreateLabel(fileLabel string) error { // the MCS label should continue to be used. SELinux will use this field // to make sure the content can not be shared by other containes. func Relabel(path string, fileLabel string, relabel string) error { + exclude_path := []string{"/", "/usr", "/etc"} if fileLabel == "" { return nil } + for _, p := range exclude_path { + if path == p { + return fmt.Errorf("Relabeling of %s is not allowed", path) + } + } if !strings.ContainsAny(relabel, "zZ") { return nil } diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/README.md b/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/README.md index ac94cba05919f..d1a60ef9850c1 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/README.md +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/README.md @@ -1,6 +1,25 @@ ## nsenter -The `nsenter` package registers a special init constructor that is called before the Go runtime has -a chance to boot. This provides us the ability to `setns` on existing namespaces and avoid the issues -that the Go runtime has with multiple threads. This constructor is only called if this package is -registered, imported, in your go application and the argv 0 is `nsenter`. +The `nsenter` package registers a special init constructor that is called before +the Go runtime has a chance to boot. This provides us the ability to `setns` on +existing namespaces and avoid the issues that the Go runtime has with multiple +threads. This constructor will be called if this package is registered, +imported, in your go application. + +The `nsenter` package will `import "C"` and it uses [cgo](https://golang.org/cmd/cgo/) +package. In cgo, if the import of "C" is immediately preceded by a comment, that comment, +called the preamble, is used as a header when compiling the C parts of the package. +So every time we import package `nsenter`, the C code function `nsexec()` would be +called. And package `nsenter` is now only imported in Docker execdriver, so every time +before we call `execdriver.Exec()`, that C code would run. + +`nsexec()` will first check the environment variable `_LIBCONTAINER_INITPID` +which will give the process of the container that should be joined. Namespaces fd will +be found from `/proc/[pid]/ns` and set by `setns` syscall. + +And then get the pipe number from `_LIBCONTAINER_INITPIPE`, error message could +be transfered through it. If tty is added, `_LIBCONTAINER_CONSOLE_PATH` will +have value and start a console for output. + +Finally, `nsexec()` will clone a child process , exit the parent process and let +the Go runtime take over. diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/nsexec.c b/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/nsexec.c index 2db051d147c17..d8e45f3cda2dc 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/nsexec.c +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/nsenter/nsexec.c @@ -137,6 +137,8 @@ void nsexec() } if (setjmp(env) == 1) { + // Child + if (setsid() == -1) { pr_perror("setsid failed"); exit(1); @@ -162,7 +164,11 @@ void nsexec() // Finish executing, let the Go runtime take over. return; } + // Parent + // We must fork to actually enter the PID namespace, use CLONE_PARENT + // so the child can have the right parent, and we don't need to forward + // the child's exit code or resend its death signal. child = clone_parent(&env); if (child < 0) { pr_perror("Unable to fork"); diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/config.go b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/config.go index e50bb3c11dbbf..1eee9dd92998d 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/config.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/config.go @@ -43,6 +43,7 @@ var createFlags = []cli.Flag{ cli.StringFlag{Name: "veth-address", Usage: "veth ip address"}, cli.StringFlag{Name: "veth-gateway", Usage: "veth gateway address"}, cli.IntFlag{Name: "veth-mtu", Usage: "veth mtu"}, + cli.BoolFlag{Name: "cgroup", Usage: "mount the cgroup data for the container"}, } var configCommand = cli.Command{ @@ -187,6 +188,12 @@ func modify(config *configs.Config, context *cli.Context) { } config.Networks = append(config.Networks, network) } + if context.Bool("cgroup") { + config.Mounts = append(config.Mounts, &configs.Mount{ + Destination: "/sys/fs/cgroup", + Device: "cgroup", + }) + } } func getTemplate() *configs.Config { diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/exec.go b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/exec.go index 9d302aa31e7d7..cf40a5951a1fc 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/exec.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/exec.go @@ -23,6 +23,7 @@ var execCommand = cli.Command{ Action: execAction, Flags: append([]cli.Flag{ cli.BoolFlag{Name: "tty,t", Usage: "allocate a TTY to the container"}, + cli.BoolFlag{Name: "systemd", Usage: "Use systemd for managing cgroups, if available"}, cli.StringFlag{Name: "id", Value: "nsinit", Usage: "specify the ID for a container"}, cli.StringFlag{Name: "config", Value: "", Usage: "path to the configuration file"}, cli.StringFlag{Name: "user,u", Value: "root", Usage: "set the user, uid, and/or gid for the process"}, diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/oom.go b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/oom.go index a59b753336f93..412534bcd67ec 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/oom.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/oom.go @@ -1,8 +1,7 @@ package main import ( - "log" - + log "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" ) diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/pause.go b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/pause.go index 89af0b6f73f53..40aace444b3fb 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/pause.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/pause.go @@ -1,8 +1,7 @@ package main import ( - "log" - + log "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" ) diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/utils.go b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/utils.go index 4deca76640bb4..92f0a9d9ef260 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/utils.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/nsinit/utils.go @@ -3,10 +3,12 @@ package main import ( "encoding/json" "fmt" + log "github.com/Sirupsen/logrus" "os" "github.com/codegangsta/cli" "github.com/docker/libcontainer" + "github.com/docker/libcontainer/cgroups/systemd" "github.com/docker/libcontainer/configs" ) @@ -29,7 +31,15 @@ func loadConfig(context *cli.Context) (*configs.Config, error) { } func loadFactory(context *cli.Context) (libcontainer.Factory, error) { - return libcontainer.New(context.GlobalString("root"), libcontainer.Cgroupfs) + cgm := libcontainer.Cgroupfs + if context.Bool("systemd") { + if systemd.UseSystemd() { + cgm = libcontainer.SystemdCgroups + } else { + log.Warn("systemd cgroup flag passed, but systemd support for managing cgroups is not available.") + } + } + return libcontainer.New(context.GlobalString("root"), cgm) } func getContainer(context *cli.Context) (libcontainer.Container, error) { diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/process.go b/Godeps/_workspace/src/github.com/docker/libcontainer/process.go index caabbea9b72cb..7902d08ce4c42 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/process.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/process.go @@ -23,7 +23,7 @@ type Process struct { Env []string // User will set the uid and gid of the executing process running inside the container - // local to the contaienr's user and group configuration. + // local to the container's user and group configuration. User string // Cwd will change the processes current working directory inside the container's rootfs. @@ -45,7 +45,7 @@ type Process struct { consolePath string // Capabilities specify the capabilities to keep when executing the process inside the container - // All capbilities not specified will be dropped from the processes capability mask + // All capabilities not specified will be dropped from the processes capability mask Capabilities []string ops processOperations diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/process_linux.go b/Godeps/_workspace/src/github.com/docker/libcontainer/process_linux.go index 1c74b65490763..66411a8a9d256 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/process_linux.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/process_linux.go @@ -119,6 +119,9 @@ func (p *setnsProcess) execSetns() error { // terminate sends a SIGKILL to the forked process for the setns routine then waits to // avoid the process becomming a zombie. func (p *setnsProcess) terminate() error { + if p.cmd.Process == nil { + return nil + } err := p.cmd.Process.Kill() if _, werr := p.wait(); err == nil { err = werr diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/rootfs_linux.go b/Godeps/_workspace/src/github.com/docker/libcontainer/rootfs_linux.go index 38ed032dd47f9..d8c61e97a0bff 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/rootfs_linux.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/rootfs_linux.go @@ -6,11 +6,14 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" + "path" "path/filepath" "strings" "syscall" "time" + "github.com/docker/libcontainer/cgroups" "github.com/docker/libcontainer/configs" "github.com/docker/libcontainer/label" ) @@ -24,9 +27,20 @@ func setupRootfs(config *configs.Config, console *linuxConsole) (err error) { return newSystemError(err) } for _, m := range config.Mounts { + for _, precmd := range m.PremountCmds { + if err := mountCmd(precmd); err != nil { + return newSystemError(err) + } + } if err := mountToRootfs(m, config.Rootfs, config.MountLabel); err != nil { return newSystemError(err) } + + for _, postcmd := range m.PostmountCmds { + if err := mountCmd(postcmd); err != nil { + return newSystemError(err) + } + } } if err := createDevices(config); err != nil { return newSystemError(err) @@ -62,6 +76,18 @@ func setupRootfs(config *configs.Config, console *linuxConsole) (err error) { return nil } +func mountCmd(cmd configs.Command) error { + + command := exec.Command(cmd.Path, cmd.Args[:]...) + command.Env = cmd.Env + command.Dir = cmd.Dir + if out, err := command.CombinedOutput(); err != nil { + return fmt.Errorf("%#v failed: %s: %v", cmd, string(out), err) + } + + return nil +} + func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error { var ( dest = m.Destination @@ -134,6 +160,37 @@ func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error { return err } } + case "cgroup": + mounts, err := cgroups.GetCgroupMounts() + if err != nil { + return err + } + var binds []*configs.Mount + for _, mm := range mounts { + dir, err := mm.GetThisCgroupDir() + if err != nil { + return err + } + binds = append(binds, &configs.Mount{ + Device: "bind", + Source: filepath.Join(mm.Mountpoint, dir), + Destination: filepath.Join(m.Destination, strings.Join(mm.Subsystems, ",")), + Flags: syscall.MS_BIND | syscall.MS_REC | syscall.MS_RDONLY, + }) + } + tmpfs := &configs.Mount{ + Device: "tmpfs", + Destination: m.Destination, + Flags: syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV, + } + if err := mountToRootfs(tmpfs, rootfs, mountLabel); err != nil { + return err + } + for _, b := range binds { + if err := mountToRootfs(b, rootfs, mountLabel); err != nil { + return err + } + } default: return fmt.Errorf("unknown mount device %q to %q", m.Device, m.Destination) } @@ -248,9 +305,9 @@ func mknodDevice(dest string, node *configs.Device) error { } func prepareRoot(config *configs.Config) error { - flag := syscall.MS_PRIVATE | syscall.MS_REC - if config.NoPivotRoot { - flag = syscall.MS_SLAVE | syscall.MS_REC + flag := syscall.MS_SLAVE | syscall.MS_REC + if config.Privatefs { + flag = syscall.MS_PRIVATE | syscall.MS_REC } if err := syscall.Mount("", "/", "", uintptr(flag), ""); err != nil { return err @@ -363,3 +420,10 @@ func maskFile(path string) error { } return nil } + +// writeSystemProperty writes the value to a path under /proc/sys as determined from the key. +// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward. +func writeSystemProperty(key, value string) error { + keyPath := strings.Replace(key, ".", "/", -1) + return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644) +} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/standard_init_linux.go b/Godeps/_workspace/src/github.com/docker/libcontainer/standard_init_linux.go index 282832b568ff2..251c09f696e64 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/standard_init_linux.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/standard_init_linux.go @@ -64,6 +64,13 @@ func (l *linuxStandardInit) Init() error { if err := label.SetProcessLabel(l.config.Config.ProcessLabel); err != nil { return err } + + for key, value := range l.config.Config.SystemProperties { + if err := writeSystemProperty(key, value); err != nil { + return err + } + } + for _, path := range l.config.Config.ReadonlyPaths { if err := remountReadonly(path); err != nil { return err diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/update-vendor.sh b/Godeps/_workspace/src/github.com/docker/libcontainer/update-vendor.sh index b68f5d46107f3..ab471872b75a2 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/update-vendor.sh +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/update-vendor.sh @@ -43,7 +43,7 @@ clone() { clone git github.com/codegangsta/cli 1.1.0 clone git github.com/coreos/go-systemd v2 clone git github.com/godbus/dbus v2 -clone git github.com/Sirupsen/logrus v0.6.6 +clone git github.com/Sirupsen/logrus v0.7.3 clone git github.com/syndtr/gocapability 8e4cdcb # intentionally not vendoring Docker itself... that'd be a circle :) diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/CHANGELOG.md b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/CHANGELOG.md new file mode 100644 index 0000000000000..eb72bff93b78b --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/CHANGELOG.md @@ -0,0 +1,7 @@ +# 0.7.3 + +formatter/\*: allow configuration of timestamp layout + +# 0.7.2 + +formatter/text: Add configuration option for time format (#158) diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/README.md b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/README.md index e755e7c18028a..d55f90924792b 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/README.md +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/README.md @@ -37,11 +37,13 @@ attached, the output is compatible with the [logfmt](http://godoc.org/github.com/kr/logfmt) format: ```text -time="2014-04-20 15:36:23.830442383 -0400 EDT" level="info" msg="A group of walrus emerges from the ocean" animal="walrus" size=10 -time="2014-04-20 15:36:23.830584199 -0400 EDT" level="warning" msg="The group's number increased tremendously!" omg=true number=122 -time="2014-04-20 15:36:23.830596521 -0400 EDT" level="info" msg="A giant walrus appears!" animal="walrus" size=10 -time="2014-04-20 15:36:23.830611837 -0400 EDT" level="info" msg="Tremendously sized cow enters the ocean." animal="walrus" size=9 -time="2014-04-20 15:36:23.830626464 -0400 EDT" level="fatal" msg="The ice breaks!" omg=true number=100 +time="2015-03-26T01:27:38-04:00" level=debug msg="Started observing beach" animal=walrus number=8 +time="2015-03-26T01:27:38-04:00" level=info msg="A group of walrus emerges from the ocean" animal=walrus size=10 +time="2015-03-26T01:27:38-04:00" level=warning msg="The group's number increased tremendously!" number=122 omg=true +time="2015-03-26T01:27:38-04:00" level=debug msg="Temperature changes" temperature=-4 +time="2015-03-26T01:27:38-04:00" level=panic msg="It's over 9000!" animal=orca size=9009 +time="2015-03-26T01:27:38-04:00" level=fatal msg="The ice breaks!" err=&{0x2082280c0 map[animal:orca size:9009] 2015-03-26 01:27:38.441574009 -0400 EDT panic It's over 9000!} number=100 omg=true +exit status 1 ``` #### Example @@ -82,7 +84,7 @@ func init() { // Use the Airbrake hook to report errors that have Error severity or above to // an exception tracker. You can create custom hooks, see the Hooks section. - log.AddHook(&logrus_airbrake.AirbrakeHook{}) + log.AddHook(airbrake.NewHook("https://example.com", "xyz", "development")) // Output to stderr instead of stdout, could also be a file. log.SetOutput(os.Stderr) @@ -106,6 +108,16 @@ func main() { "omg": true, "number": 100, }).Fatal("The ice breaks!") + + // A common pattern is to re-use fields between logging statements by re-using + // the logrus.Entry returned from WithFields() + contextLogger := log.WithFields(log.Fields{ + "common": "this is a common field", + "other": "I also should be logged always", + }) + + contextLogger.Info("I'll be logged with common and other field") + contextLogger.Info("Me too") } ``` @@ -164,43 +176,8 @@ You can add hooks for logging levels. For example to send errors to an exception tracking service on `Error`, `Fatal` and `Panic`, info to StatsD or log to multiple places simultaneously, e.g. syslog. -```go -// Not the real implementation of the Airbrake hook. Just a simple sample. -import ( - log "github.com/Sirupsen/logrus" -) - -func init() { - log.AddHook(new(AirbrakeHook)) -} - -type AirbrakeHook struct{} - -// `Fire()` takes the entry that the hook is fired for. `entry.Data[]` contains -// the fields for the entry. See the Fields section of the README. -func (hook *AirbrakeHook) Fire(entry *logrus.Entry) error { - err := airbrake.Notify(entry.Data["error"].(error)) - if err != nil { - log.WithFields(log.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - }).Info("Failed to send error to Airbrake") - } - - return nil -} - -// `Levels()` returns a slice of `Levels` the hook is fired for. -func (hook *AirbrakeHook) Levels() []log.Level { - return []log.Level{ - log.ErrorLevel, - log.FatalLevel, - log.PanicLevel, - } -} -``` - -Logrus comes with built-in hooks. Add those, or your custom hook, in `init`: +Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in +`init`: ```go import ( @@ -211,7 +188,7 @@ import ( ) func init() { - log.AddHook(new(logrus_airbrake.AirbrakeHook)) + log.AddHook(airbrake.NewHook("https://example.com", "xyz", "development")) hook, err := logrus_syslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_INFO, "") if err != nil { @@ -222,28 +199,18 @@ func init() { } ``` -* [`github.com/Sirupsen/logrus/hooks/airbrake`](https://github.com/Sirupsen/logrus/blob/master/hooks/airbrake/airbrake.go) - Send errors to an exception tracking service compatible with the Airbrake API. - Uses [`airbrake-go`](https://github.com/tobi/airbrake-go) behind the scenes. -* [`github.com/Sirupsen/logrus/hooks/papertrail`](https://github.com/Sirupsen/logrus/blob/master/hooks/papertrail/papertrail.go) - Send errors to the Papertrail hosted logging service via UDP. - -* [`github.com/Sirupsen/logrus/hooks/syslog`](https://github.com/Sirupsen/logrus/blob/master/hooks/syslog/syslog.go) - Send errors to remote syslog server. - Uses standard library `log/syslog` behind the scenes. - -* [`github.com/nubo/hiprus`](https://github.com/nubo/hiprus) - Send errors to a channel in hipchat. - -* [`github.com/sebest/logrusly`](https://github.com/sebest/logrusly) - Send logs to Loggly (https://www.loggly.com/) - -* [`github.com/johntdyer/slackrus`](https://github.com/johntdyer/slackrus) - Hook for Slack chat. - -* [`github.com/wercker/journalhook`](https://github.com/wercker/journalhook). - Hook for logging to `systemd-journald`. +| Hook | Description | +| ----- | ----------- | +| [Airbrake](https://github.com/Sirupsen/logrus/blob/master/hooks/airbrake/airbrake.go) | Send errors to an exception tracking service compatible with the Airbrake API. Uses [`airbrake-go`](https://github.com/tobi/airbrake-go) behind the scenes. | +| [Papertrail](https://github.com/Sirupsen/logrus/blob/master/hooks/papertrail/papertrail.go) | Send errors to the Papertrail hosted logging service via UDP. | +| [Syslog](https://github.com/Sirupsen/logrus/blob/master/hooks/syslog/syslog.go) | Send errors to remote syslog server. Uses standard library `log/syslog` behind the scenes. | +| [BugSnag](https://github.com/Sirupsen/logrus/blob/master/hooks/bugsnag/bugsnag.go) | Send errors to the Bugsnag exception tracking service. | +| [Hiprus](https://github.com/nubo/hiprus) | Send errors to a channel in hipchat. | +| [Logrusly](https://github.com/sebest/logrusly) | Send logs to [Loggly](https://www.loggly.com/) | +| [Slackrus](https://github.com/johntdyer/slackrus) | Hook for Slack chat. | +| [Journalhook](https://github.com/wercker/journalhook) | Hook for logging to `systemd-journald` | +| [Graylog](https://github.com/gemnasium/logrus-hooks/tree/master/graylog) | Hook for logging to [Graylog](http://graylog2.org/) | #### Level logging @@ -321,6 +288,11 @@ The built-in logging formatters are: field to `true`. To force no colored output even if there is a TTY set the `DisableColors` field to `true` * `logrus.JSONFormatter`. Logs fields as JSON. +* `logrus_logstash.LogstashFormatter`. Logs fields as Logstash Events (http://logstash.net). + + ```go + logrus.SetFormatter(&logrus_logstash.LogstashFormatter{Type: “application_name"}) + ``` Third party logging formatters: diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/examples/hook/hook.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/examples/hook/hook.go index 42e7a4c98252e..cb5759a35cf52 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/examples/hook/hook.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/examples/hook/hook.go @@ -3,21 +3,16 @@ package main import ( "github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus/hooks/airbrake" - "github.com/tobi/airbrake-go" ) var log = logrus.New() func init() { log.Formatter = new(logrus.TextFormatter) // default - log.Hooks.Add(new(logrus_airbrake.AirbrakeHook)) + log.Hooks.Add(airbrake.NewHook("https://example.com", "xyz", "development")) } func main() { - airbrake.Endpoint = "https://exceptions.whatever.com/notifier_api/v2/notices.xml" - airbrake.ApiKey = "whatever" - airbrake.Environment = "production" - log.WithFields(logrus.Fields{ "animal": "walrus", "size": 10, diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatter.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatter.go index 038ce9fd2970a..104d689f187eb 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatter.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatter.go @@ -1,5 +1,9 @@ package logrus +import "time" + +const DefaultTimestampFormat = time.RFC3339 + // The Formatter interface is used to implement a custom Formatter. It takes an // `Entry`. It exposes all the fields, including the default ones: // diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go new file mode 100644 index 0000000000000..8ea93ddf20a03 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/formatters/logstash/logstash.go @@ -0,0 +1,56 @@ +package logstash + +import ( + "encoding/json" + "fmt" + + "github.com/Sirupsen/logrus" +) + +// Formatter generates json in logstash format. +// Logstash site: http://logstash.net/ +type LogstashFormatter struct { + Type string // if not empty use for logstash type field. + + // TimestampFormat sets the format used for timestamps. + TimestampFormat string +} + +func (f *LogstashFormatter) Format(entry *logrus.Entry) ([]byte, error) { + entry.Data["@version"] = 1 + + if f.TimestampFormat == "" { + f.TimestampFormat = logrus.DefaultTimestampFormat + } + + entry.Data["@timestamp"] = entry.Time.Format(f.TimestampFormat) + + // set message field + v, ok := entry.Data["message"] + if ok { + entry.Data["fields.message"] = v + } + entry.Data["message"] = entry.Message + + // set level field + v, ok = entry.Data["level"] + if ok { + entry.Data["fields.level"] = v + } + entry.Data["level"] = entry.Level.String() + + // set type field + if f.Type != "" { + v, ok = entry.Data["type"] + if ok { + entry.Data["fields.type"] = v + } + entry.Data["type"] = f.Type + } + + serialized, err := json.Marshal(entry.Data) + if err != nil { + return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) + } + return append(serialized, '\n'), nil +} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go index 75f4db1513e4f..b0502c335a96d 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/airbrake/airbrake.go @@ -1,51 +1,51 @@ -package logrus_airbrake +package airbrake import ( + "errors" + "fmt" + "github.com/Sirupsen/logrus" "github.com/tobi/airbrake-go" ) // AirbrakeHook to send exceptions to an exception-tracking service compatible -// with the Airbrake API. You must set: -// * airbrake.Endpoint -// * airbrake.ApiKey -// * airbrake.Environment -// -// Before using this hook, to send an error. Entries that trigger an Error, -// Fatal or Panic should now include an "error" field to send to Airbrake. -type AirbrakeHook struct{} - -func (hook *AirbrakeHook) Fire(entry *logrus.Entry) error { - if entry.Data["error"] == nil { - entry.Logger.WithFields(logrus.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - }).Warn("Exceptions sent to Airbrake must have an 'error' key with the error") - return nil +// with the Airbrake API. +type airbrakeHook struct { + APIKey string + Endpoint string + Environment string +} + +func NewHook(endpoint, apiKey, env string) *airbrakeHook { + return &airbrakeHook{ + APIKey: apiKey, + Endpoint: endpoint, + Environment: env, } +} + +func (hook *airbrakeHook) Fire(entry *logrus.Entry) error { + airbrake.ApiKey = hook.APIKey + airbrake.Endpoint = hook.Endpoint + airbrake.Environment = hook.Environment + var notifyErr error err, ok := entry.Data["error"].(error) - if !ok { - entry.Logger.WithFields(logrus.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - }).Warn("Exceptions sent to Airbrake must have an `error` key of type `error`") - return nil + if ok { + notifyErr = err + } else { + notifyErr = errors.New(entry.Message) } - airErr := airbrake.Notify(err) + airErr := airbrake.Notify(notifyErr) if airErr != nil { - entry.Logger.WithFields(logrus.Fields{ - "source": "airbrake", - "endpoint": airbrake.Endpoint, - "error": airErr, - }).Warn("Failed to send error to Airbrake") + return fmt.Errorf("Failed to send error to Airbrake: %s", airErr) } return nil } -func (hook *AirbrakeHook) Levels() []logrus.Level { +func (hook *airbrakeHook) Levels() []logrus.Level { return []logrus.Level{ logrus.ErrorLevel, logrus.FatalLevel, diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go new file mode 100644 index 0000000000000..d20a0f54ab770 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/hooks/bugsnag/bugsnag.go @@ -0,0 +1,68 @@ +package logrus_bugsnag + +import ( + "errors" + + "github.com/Sirupsen/logrus" + "github.com/bugsnag/bugsnag-go" +) + +type bugsnagHook struct{} + +// ErrBugsnagUnconfigured is returned if NewBugsnagHook is called before +// bugsnag.Configure. Bugsnag must be configured before the hook. +var ErrBugsnagUnconfigured = errors.New("bugsnag must be configured before installing this logrus hook") + +// ErrBugsnagSendFailed indicates that the hook failed to submit an error to +// bugsnag. The error was successfully generated, but `bugsnag.Notify()` +// failed. +type ErrBugsnagSendFailed struct { + err error +} + +func (e ErrBugsnagSendFailed) Error() string { + return "failed to send error to Bugsnag: " + e.err.Error() +} + +// NewBugsnagHook initializes a logrus hook which sends exceptions to an +// exception-tracking service compatible with the Bugsnag API. Before using +// this hook, you must call bugsnag.Configure(). The returned object should be +// registered with a log via `AddHook()` +// +// Entries that trigger an Error, Fatal or Panic should now include an "error" +// field to send to Bugsnag. +func NewBugsnagHook() (*bugsnagHook, error) { + if bugsnag.Config.APIKey == "" { + return nil, ErrBugsnagUnconfigured + } + return &bugsnagHook{}, nil +} + +// Fire forwards an error to Bugsnag. Given a logrus.Entry, it extracts the +// "error" field (or the Message if the error isn't present) and sends it off. +func (hook *bugsnagHook) Fire(entry *logrus.Entry) error { + var notifyErr error + err, ok := entry.Data["error"].(error) + if ok { + notifyErr = err + } else { + notifyErr = errors.New(entry.Message) + } + + bugsnagErr := bugsnag.Notify(notifyErr) + if bugsnagErr != nil { + return ErrBugsnagSendFailed{bugsnagErr} + } + + return nil +} + +// Levels enumerates the log levels on which the error should be forwarded to +// bugsnag: everything at or above the "Error" level. +func (hook *bugsnagHook) Levels() []logrus.Level { + return []logrus.Level{ + logrus.ErrorLevel, + logrus.FatalLevel, + logrus.PanicLevel, + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/json_formatter.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/json_formatter.go index 0e38a61919326..dcc4f1d9fd7e3 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/json_formatter.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/json_formatter.go @@ -3,24 +3,32 @@ package logrus import ( "encoding/json" "fmt" - "time" ) -type JSONFormatter struct{} +type JSONFormatter struct { + // TimestampFormat sets the format used for marshaling timestamps. + TimestampFormat string +} func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { data := make(Fields, len(entry.Data)+3) for k, v := range entry.Data { - // Otherwise errors are ignored by `encoding/json` - // https://github.com/Sirupsen/logrus/issues/137 - if err, ok := v.(error); ok { - data[k] = err.Error() - } else { + switch v := v.(type) { + case error: + // Otherwise errors are ignored by `encoding/json` + // https://github.com/Sirupsen/logrus/issues/137 + data[k] = v.Error() + default: data[k] = v } } prefixFieldClashes(data) - data["time"] = entry.Time.Format(time.RFC3339) + + if f.TimestampFormat == "" { + f.TimestampFormat = DefaultTimestampFormat + } + + data["time"] = entry.Time.Format(f.TimestampFormat) data["msg"] = entry.Message data["level"] = entry.Level.String() diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/logger.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/logger.go index b392e547a7ba2..da928a3750990 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/logger.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/logger.go @@ -65,11 +65,15 @@ func (logger *Logger) WithFields(fields Fields) *Entry { } func (logger *Logger) Debugf(format string, args ...interface{}) { - NewEntry(logger).Debugf(format, args...) + if logger.Level >= DebugLevel { + NewEntry(logger).Debugf(format, args...) + } } func (logger *Logger) Infof(format string, args ...interface{}) { - NewEntry(logger).Infof(format, args...) + if logger.Level >= InfoLevel { + NewEntry(logger).Infof(format, args...) + } } func (logger *Logger) Printf(format string, args ...interface{}) { @@ -77,31 +81,45 @@ func (logger *Logger) Printf(format string, args ...interface{}) { } func (logger *Logger) Warnf(format string, args ...interface{}) { - NewEntry(logger).Warnf(format, args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnf(format, args...) + } } func (logger *Logger) Warningf(format string, args ...interface{}) { - NewEntry(logger).Warnf(format, args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnf(format, args...) + } } func (logger *Logger) Errorf(format string, args ...interface{}) { - NewEntry(logger).Errorf(format, args...) + if logger.Level >= ErrorLevel { + NewEntry(logger).Errorf(format, args...) + } } func (logger *Logger) Fatalf(format string, args ...interface{}) { - NewEntry(logger).Fatalf(format, args...) + if logger.Level >= FatalLevel { + NewEntry(logger).Fatalf(format, args...) + } } func (logger *Logger) Panicf(format string, args ...interface{}) { - NewEntry(logger).Panicf(format, args...) + if logger.Level >= PanicLevel { + NewEntry(logger).Panicf(format, args...) + } } func (logger *Logger) Debug(args ...interface{}) { - NewEntry(logger).Debug(args...) + if logger.Level >= DebugLevel { + NewEntry(logger).Debug(args...) + } } func (logger *Logger) Info(args ...interface{}) { - NewEntry(logger).Info(args...) + if logger.Level >= InfoLevel { + NewEntry(logger).Info(args...) + } } func (logger *Logger) Print(args ...interface{}) { @@ -109,31 +127,45 @@ func (logger *Logger) Print(args ...interface{}) { } func (logger *Logger) Warn(args ...interface{}) { - NewEntry(logger).Warn(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warn(args...) + } } func (logger *Logger) Warning(args ...interface{}) { - NewEntry(logger).Warn(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warn(args...) + } } func (logger *Logger) Error(args ...interface{}) { - NewEntry(logger).Error(args...) + if logger.Level >= ErrorLevel { + NewEntry(logger).Error(args...) + } } func (logger *Logger) Fatal(args ...interface{}) { - NewEntry(logger).Fatal(args...) + if logger.Level >= FatalLevel { + NewEntry(logger).Fatal(args...) + } } func (logger *Logger) Panic(args ...interface{}) { - NewEntry(logger).Panic(args...) + if logger.Level >= PanicLevel { + NewEntry(logger).Panic(args...) + } } func (logger *Logger) Debugln(args ...interface{}) { - NewEntry(logger).Debugln(args...) + if logger.Level >= DebugLevel { + NewEntry(logger).Debugln(args...) + } } func (logger *Logger) Infoln(args ...interface{}) { - NewEntry(logger).Infoln(args...) + if logger.Level >= InfoLevel { + NewEntry(logger).Infoln(args...) + } } func (logger *Logger) Println(args ...interface{}) { @@ -141,21 +173,31 @@ func (logger *Logger) Println(args ...interface{}) { } func (logger *Logger) Warnln(args ...interface{}) { - NewEntry(logger).Warnln(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnln(args...) + } } func (logger *Logger) Warningln(args ...interface{}) { - NewEntry(logger).Warnln(args...) + if logger.Level >= WarnLevel { + NewEntry(logger).Warnln(args...) + } } func (logger *Logger) Errorln(args ...interface{}) { - NewEntry(logger).Errorln(args...) + if logger.Level >= ErrorLevel { + NewEntry(logger).Errorln(args...) + } } func (logger *Logger) Fatalln(args ...interface{}) { - NewEntry(logger).Fatalln(args...) + if logger.Level >= FatalLevel { + NewEntry(logger).Fatalln(args...) + } } func (logger *Logger) Panicln(args ...interface{}) { - NewEntry(logger).Panicln(args...) + if logger.Level >= PanicLevel { + NewEntry(logger).Panicln(args...) + } } diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/terminal_openbsd.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/terminal_openbsd.go index d238bfa0b48f5..af609a53d6492 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/terminal_openbsd.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/terminal_openbsd.go @@ -1,4 +1,3 @@ - package logrus import "syscall" diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/text_formatter.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/text_formatter.go index 71dcb6617a449..612417ff9c846 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/text_formatter.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/text_formatter.go @@ -3,7 +3,6 @@ package logrus import ( "bytes" "fmt" - "regexp" "sort" "strings" "time" @@ -21,7 +20,6 @@ const ( var ( baseTimestamp time.Time isTerminal bool - noQuoteNeeded *regexp.Regexp ) func init() { @@ -48,6 +46,9 @@ type TextFormatter struct { // the time passed since beginning of execution. FullTimestamp bool + // TimestampFormat to use for display when a full timestamp is printed + TimestampFormat string + // The fields are sorted by default for a consistent output. For applications // that log extremely frequently and don't use the JSON formatter this may not // be desired. @@ -70,11 +71,14 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { isColored := (f.ForceColors || isTerminal) && !f.DisableColors + if f.TimestampFormat == "" { + f.TimestampFormat = DefaultTimestampFormat + } if isColored { f.printColored(b, entry, keys) } else { if !f.DisableTimestamp { - f.appendKeyValue(b, "time", entry.Time.Format(time.RFC3339)) + f.appendKeyValue(b, "time", entry.Time.Format(f.TimestampFormat)) } f.appendKeyValue(b, "level", entry.Level.String()) f.appendKeyValue(b, "msg", entry.Message) @@ -105,7 +109,7 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin if !f.FullTimestamp { fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, miniTS(), entry.Message) } else { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %-44s ", levelColor, levelText, entry.Time.Format(time.RFC3339), entry.Message) + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %-44s ", levelColor, levelText, entry.Time.Format(f.TimestampFormat), entry.Message) } for _, k := range keys { v := entry.Data[k] diff --git a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/writer.go b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/writer.go index 90d3e01b45987..1e30b1c753a74 100644 --- a/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/writer.go +++ b/Godeps/_workspace/src/github.com/docker/libcontainer/vendor/src/github.com/Sirupsen/logrus/writer.go @@ -6,7 +6,7 @@ import ( "runtime" ) -func (logger *Logger) Writer() (*io.PipeWriter) { +func (logger *Logger) Writer() *io.PipeWriter { reader, writer := io.Pipe() go logger.writerScanner(reader)