Skip to content

Commit

Permalink
tolerate cross-platform building of crio/cgroup build logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@bparees
bparees committed Sep 29, 2017
1 parent 621b7e3 commit 1c1f653
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 140 deletions.
195 changes: 59 additions & 136 deletions pkg/build/builder/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,16 @@ package builder

import (
"bufio"
"errors"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"regexp"
"strconv"
"strings"

"github.com/google/cadvisor/container/crio"
crioclient "github.com/kubernetes-incubator/cri-o/client"
"github.com/kubernetes-incubator/cri-o/pkg/annotations"

docker "github.com/fsouza/go-dockerclient"

s2iapi "github.com/openshift/source-to-image/pkg/api"

buildapi "github.com/openshift/origin/pkg/build/apis/build"
)

Expand All @@ -28,134 +20,6 @@ var (
procCGroupPattern = regexp.MustCompile(`\d+:([a-z_,]+):/.*/(\w+-|)([a-z0-9]+).*`)
)

// readNetClsCGroup parses /proc/self/cgroup in order to determine the container id that can be used
// the network namespace that this process is running on, it returns the cgroup and container type
// (docker vs crio).
func readNetClsCGroup(reader io.Reader) (string, string) {

containerType := "docker"

cgroups := make(map[string]string)
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
if match := procCGroupPattern.FindStringSubmatch(scanner.Text()); match != nil {
containerType = strings.TrimSuffix(match[2], "-")

list := strings.Split(match[1], ",")
containerId := match[3]
if len(list) > 0 {
for _, key := range list {
cgroups[key] = containerId
}
} else {
cgroups[match[1]] = containerId
}
}
}

names := []string{"net_cls", "cpu"}
for _, group := range names {
if value, ok := cgroups[group]; ok {
return value, containerType
}
}

return "", containerType
}

// getContainerNetworkConfig determines whether the builder is running as a container
// by examining /proc/self/cgroup. This context is then passed to source-to-image.
// It returns a suitable argument for NetworkMode. If the container platform is
// CRI-O, it also returns a path for /etc/resolv.conf, suitable for bindmounting.
func getContainerNetworkConfig() (string, string, error) {
file, err := os.Open("/proc/self/cgroup")
if err != nil {
return "", "", err
}
defer file.Close()

if id, containerType := readNetClsCGroup(file); id != "" {
glog.V(5).Infof("container type=%s", containerType)
if containerType != "crio" {
return s2iapi.DockerNetworkModeContainerPrefix + id, "", nil
}

crioClient, err := crioclient.New(crio.CrioSocket)
if err != nil {
return "", "", err
}
info, err := crioClient.ContainerInfo(id)
if err != nil {
return "", "", err
}
pid := strconv.Itoa(info.Pid)
resolvConfHostPath := info.CrioAnnotations[annotations.ResolvPath]
if len(resolvConfHostPath) == 0 {
return "", "", errors.New("/etc/resolv.conf hostpath is empty")
}

return fmt.Sprintf("netns:/proc/%s/ns/net", pid), resolvConfHostPath, nil
}
return "", "", nil
}

// GetCGroupLimits returns a struct populated with cgroup limit values gathered
// from the local /sys/fs/cgroup filesystem. Overflow values are set to
// math.MaxInt64.
func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
byteLimit, err := readInt64("/sys/fs/cgroup/memory/memory.limit_in_bytes")
if err != nil {
// for systems without cgroups builds should succeed
if _, err := os.Stat("/sys/fs/cgroup"); os.IsNotExist(err) {
return &s2iapi.CGroupLimits{}, nil
}
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
}
// math.MaxInt64 seems to give cgroups trouble, this value is
// still 92 terabytes, so it ought to be sufficiently large for
// our purposes.
if byteLimit > 92233720368547 {
byteLimit = 92233720368547
}

parent, err := getCgroupParent()
if err != nil {
return nil, fmt.Errorf("read cgroup parent: %v", err)
}

return &s2iapi.CGroupLimits{
// Though we are capped on memory and cpu at the cgroup parent level,
// some build containers care what their memory limit is so they can
// adapt, thus we need to set the memory limit at the container level
// too, so that information is available to them.
MemoryLimitBytes: byteLimit,
// Set memoryswap==memorylimit, this ensures no swapping occurs.
// see: https://docs.docker.com/engine/reference/run/#runtime-constraints-on-cpu-and-memory
MemorySwap: byteLimit,
Parent: parent,
}, nil
}

func readInt64(filePath string) (int64, error) {
data, err := ioutil.ReadFile(filePath)
if err != nil {
return -1, err
}
s := strings.TrimSpace(string(data))
val, err := strconv.ParseInt(s, 10, 64)
// overflow errors are ok, we'll get return a math.MaxInt64 value which is more
// than enough anyway. For underflow we'll return MinInt64 and the error.
if err != nil && err.(*strconv.NumError).Err == strconv.ErrRange {
if s[0] == '-' {
return math.MinInt64, err
}
return math.MaxInt64, nil
} else if err != nil {
return -1, err
}
return val, nil
}

// MergeEnv will take an existing environment and merge it with a new set of
// variables. For variables with the same name in both, only the one in the
// new environment will be kept.
Expand Down Expand Up @@ -204,6 +68,65 @@ func addBuildLabels(labels map[string]string, build *buildapi.Build) {
labels[buildapi.DefaultDockerLabelNamespace+"build.namespace"] = build.Namespace
}

// readInt64 reads a file containing a 64 bit integer value
// and returns the value as an int64. If the file contains
// a value larger than an int64, it returns MaxInt64,
// if the value is smaller than an int64, it returns MinInt64.
func readInt64(filePath string) (int64, error) {
data, err := ioutil.ReadFile(filePath)
if err != nil {
return -1, err
}
s := strings.TrimSpace(string(data))
val, err := strconv.ParseInt(s, 10, 64)
// overflow errors are ok, we'll get return a math.MaxInt64 value which is more
// than enough anyway. For underflow we'll return MinInt64 and the error.
if err != nil && err.(*strconv.NumError).Err == strconv.ErrRange {
if s[0] == '-' {
return math.MinInt64, err
}
return math.MaxInt64, nil
} else if err != nil {
return -1, err
}
return val, nil
}

// readNetClsCGroup parses /proc/self/cgroup in order to determine the container id that can be used
// the network namespace that this process is running on, it returns the cgroup and container type
// (docker vs crio).
func readNetClsCGroup(reader io.Reader) (string, string) {

containerType := "docker"

cgroups := make(map[string]string)
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
if match := procCGroupPattern.FindStringSubmatch(scanner.Text()); match != nil {
containerType = strings.TrimSuffix(match[2], "-")

list := strings.Split(match[1], ",")
containerId := match[3]
if len(list) > 0 {
for _, key := range list {
cgroups[key] = containerId
}
} else {
cgroups[match[1]] = containerId
}
}
}

names := []string{"net_cls", "cpu"}
for _, group := range names {
if value, ok := cgroups[group]; ok {
return value, containerType
}
}

return "", containerType
}

// extractParentFromCgroupMap finds the cgroup parent in the cgroup map
func extractParentFromCgroupMap(cgMap map[string]string) (string, error) {
memory, ok := cgMap["memory"]
Expand Down
82 changes: 82 additions & 0 deletions pkg/build/builder/util_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,91 @@
package builder

import (
"errors"
"fmt"
"os"
"strconv"

"github.com/google/cadvisor/container/crio"
crioclient "github.com/kubernetes-incubator/cri-o/client"
"github.com/kubernetes-incubator/cri-o/pkg/annotations"
"github.com/opencontainers/runc/libcontainer/cgroups"
s2iapi "github.com/openshift/source-to-image/pkg/api"
)

// getContainerNetworkConfig determines whether the builder is running as a container
// by examining /proc/self/cgroup. This context is then passed to source-to-image.
// It returns a suitable argument for NetworkMode. If the container platform is
// CRI-O, it also returns a path for /etc/resolv.conf, suitable for bindmounting.
func getContainerNetworkConfig() (string, string, error) {
file, err := os.Open("/proc/self/cgroup")
if err != nil {
return "", "", err
}
defer file.Close()

if id, containerType := readNetClsCGroup(file); id != "" {
glog.V(5).Infof("container type=%s", containerType)
if containerType != "crio" {
return s2iapi.DockerNetworkModeContainerPrefix + id, "", nil
}

crioClient, err := crioclient.New(crio.CrioSocket)
if err != nil {
return "", "", err
}
info, err := crioClient.ContainerInfo(id)
if err != nil {
return "", "", err
}
pid := strconv.Itoa(info.Pid)
resolvConfHostPath := info.CrioAnnotations[annotations.ResolvPath]
if len(resolvConfHostPath) == 0 {
return "", "", errors.New("/etc/resolv.conf hostpath is empty")
}

return fmt.Sprintf("netns:/proc/%s/ns/net", pid), resolvConfHostPath, nil
}
return "", "", nil
}

// GetCGroupLimits returns a struct populated with cgroup limit values gathered
// from the local /sys/fs/cgroup filesystem. Overflow values are set to
// math.MaxInt64.
func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
byteLimit, err := readInt64("/sys/fs/cgroup/memory/memory.limit_in_bytes")
if err != nil {
// for systems without cgroups builds should succeed
if _, err := os.Stat("/sys/fs/cgroup"); os.IsNotExist(err) {
return &s2iapi.CGroupLimits{}, nil
}
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
}
// math.MaxInt64 seems to give cgroups trouble, this value is
// still 92 terabytes, so it ought to be sufficiently large for
// our purposes.
if byteLimit > 92233720368547 {
byteLimit = 92233720368547
}

parent, err := getCgroupParent()
if err != nil {
return nil, fmt.Errorf("read cgroup parent: %v", err)
}

return &s2iapi.CGroupLimits{
// Though we are capped on memory and cpu at the cgroup parent level,
// some build containers care what their memory limit is so they can
// adapt, thus we need to set the memory limit at the container level
// too, so that information is available to them.
MemoryLimitBytes: byteLimit,
// Set memoryswap==memorylimit, this ensures no swapping occurs.
// see: https://docs.docker.com/engine/reference/run/#runtime-constraints-on-cpu-and-memory
MemorySwap: byteLimit,
Parent: parent,
}, nil
}

// getCgroupParent determines the parent cgroup for a container from
// within that container.
func getCgroupParent() (string, error) {
Expand Down
21 changes: 20 additions & 1 deletion pkg/build/builder/util_unsupported.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,26 @@

package builder

import "errors"
import (
"errors"

s2iapi "github.com/openshift/source-to-image/pkg/api"
)

// getContainerNetworkConfig determines whether the builder is running as a container
// by examining /proc/self/cgroup. This context is then passed to source-to-image.
// It returns a suitable argument for NetworkMode. If the container platform is
// CRI-O, it also returns a path for /etc/resolv.conf, suitable for bindmounting.
func getContainerNetworkConfig() (string, string, error) {
return "", "", errors.New("getContainerNetworkConfig is unsupported on this platform")
}

// GetCGroupLimits returns a struct populated with cgroup limit values gathered
// from the local /sys/fs/cgroup filesystem. Overflow values are set to
// math.MaxInt64.
func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
return nil, errors.New("GetCGroupLimits is unsupported on this platform")
}

// getCgroupParent determines the parent cgroup for a container from
// within that container.
Expand Down
6 changes: 3 additions & 3 deletions pkg/build/controller/strategy/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"strings"

"github.com/golang/glog"
"github.com/google/cadvisor/container/crio"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kvalidation "k8s.io/apimachinery/pkg/util/validation"
Expand All @@ -25,6 +24,7 @@ import (
const (
// dockerSocketPath is the default path for the Docker socket inside the builder container
dockerSocketPath = "/var/run/docker.sock"
crioSocketPath = "/var/run/crio.sock"
sourceSecretMountPath = "/var/run/secrets/openshift.io/source"

DockerPushSecretMountPath = "/var/run/secrets/openshift.io/push"
Expand Down Expand Up @@ -99,14 +99,14 @@ func setupCrioSocket(pod *v1.Pod) {
Name: "crio-socket",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: crio.CrioSocket,
Path: crioSocketPath,
},
},
}

crioSocketVolumeMount := v1.VolumeMount{
Name: "crio-socket",
MountPath: crio.CrioSocket,
MountPath: crioSocketPath,
}

pod.Spec.Volumes = append(pod.Spec.Volumes,
Expand Down

0 comments on commit 1c1f653

Please sign in to comment.