Skip to content

Commit

Permalink
Ceph op: Add test package, container tests
Browse files Browse the repository at this point in the history 
Add a 'test' package to the Ceph operator and define a test to verify
that containers produced by Rook-Ceph match what is expected. Because
this is for 'containers' and not strictly for 'ceph containers', this
could be moved a level up to the operator test package; however, if
other backends wish to use the container tests, they will likely need to
make modifications, and there is concern that this might make the Ceph
tests brittle.

Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>
  • Loading branch information
@BlaineEXE
BlaineEXE committed Sep 21, 2018
1 parent e3d2410 commit 7a5fe84
Show file tree
Hide file tree
Showing 8 changed files with 327 additions and 113 deletions.
2 changes: 1 addition & 1 deletion pkg/operator/ceph/cluster/mon/mon_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ func TestSaveMonEndpoints(t *testing.T) {
assert.Equal(t, "2", cm.Data[MaxMonIDKey])
}

func TestMonInQuourm(t *testing.T) {
func TestMonInQuorum(t *testing.T) {
entry := client.MonMapEntry{Name: "foo", Rank: 23}
quorum := []int{}
// Nothing in quorum
Expand Down
8 changes: 0 additions & 8 deletions pkg/operator/ceph/cluster/mon/spec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"os"
"path"

cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
mondaemon "github.com/rook/rook/pkg/daemon/ceph/mon"
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
"github.com/rook/rook/pkg/operator/k8sutil"
Expand Down Expand Up @@ -198,14 +197,7 @@ func (c *Cluster) makeMonmapInitContainer(monConfig *monConfig, containerName st
func (c *Cluster) cephMonCommonArgs(monConfig *monConfig) []string {
return []string{
"--name", fmt.Sprintf("mon.%s", monConfig.DaemonName),
"--cluster", c.clusterInfo.Name,
"--mon-data", mondaemon.GetMonDataDirPath(c.context.ConfigDir, monConfig.DaemonName),
// It's safe to use the default path for the config b/c 'GenerateConfigFile'
// also writes a copy of the config to the default path
"--conf", cephconfig.DefaultConfigFilePath(),
// It's similarly safe to use the default path for the keyring b/c
// 'CreateKeyring' also writes a copy of the keyring to the default path
"--keyring", cephconfig.DefaultKeyringFilePath(),
}
}

Expand Down
191 changes: 106 additions & 85 deletions pkg/operator/ceph/cluster/mon/spec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ package mon

import (
"fmt"
"strings"
"testing"

cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
mondaemon "github.com/rook/rook/pkg/daemon/ceph/mon"
test_opceph "github.com/rook/rook/pkg/operator/ceph/test"
"github.com/rook/rook/pkg/operator/k8sutil"
testop "github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
Expand All @@ -39,17 +39,11 @@ func TestPodSpecs(t *testing.T) {
testPodSpec(t, "/var/lib/mydatadir")
}

func testCephMonCommonArgs(t *testing.T, c *Cluster, name string, cont v1.Container, startIndex int) {
assert.Equal(t, "--name", cont.Args[startIndex])
assert.Equal(t, fmt.Sprintf("mon.%s", name), cont.Args[startIndex+1])
assert.Equal(t, "--cluster", cont.Args[startIndex+2])
assert.Equal(t, c.clusterInfo.Name, cont.Args[startIndex+3])
assert.Equal(t, "--mon-data", cont.Args[startIndex+4])
assert.Equal(t, mondaemon.GetMonDataDirPath(c.context.ConfigDir, name), cont.Args[startIndex+5])
assert.Equal(t, "--conf", cont.Args[startIndex+6])
assert.Equal(t, cephconfig.DefaultConfigFilePath(), cont.Args[startIndex+7])
assert.Equal(t, "--keyring", cont.Args[startIndex+8])
assert.Equal(t, cephconfig.DefaultKeyringFilePath(), cont.Args[startIndex+9])
func monCommonExpectedArgs(name string, c *Cluster) [][]string {
return [][]string{
{"--name", fmt.Sprintf("mon.%s", name)},
{"--mon-data", mondaemon.GetMonDataDirPath(c.context.ConfigDir, name)},
}
}

func testPodSpec(t *testing.T, dataDir string) {
Expand Down Expand Up @@ -93,76 +87,107 @@ func testPodSpec(t *testing.T, dataDir string) {
assert.Equal(t, appName, pod.ObjectMeta.Labels["app"])
assert.Equal(t, c.Namespace, pod.ObjectMeta.Labels["mon_cluster"])

assert.Equal(t, 3, len(pod.Spec.InitContainers))
assert.Equal(t, 1, len(pod.Spec.Containers))

// All containers have the same privilege
isPrivileged := false

// config w/ rook binary init container
cont := pod.Spec.InitContainers[0]
assert.Equal(t, "rook/rook:myversion", cont.Image)
assert.Equal(t, 0, len(cont.Ports))
assert.Equal(t, 3, len(cont.VolumeMounts))
assert.Nil(t, testop.VolumeMountExists(cephconfig.DefaultConfigMountName, cont.VolumeMounts))
assert.Nil(t, testop.VolumeMountExists(k8sutil.ConfigOverrideName, cont.VolumeMounts))
assert.Equal(t, 7, len(cont.Env))
assert.False(t, *cont.SecurityContext.Privileged)
logCommandWithArgs("confg init", cont.Command, cont.Args)
assert.Equal(t, 0, len(cont.Command))
assert.Equal(t, "ceph", cont.Args[0])
assert.Equal(t, mondaemon.InitCommand, cont.Args[1])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2])
assert.Equal(t, fmt.Sprintf("--name=%s", name), cont.Args[3])
assert.Equal(t, "--port=6790", cont.Args[4])
assert.Equal(t, fmt.Sprintf("--fsid=%s", c.clusterInfo.FSID), cont.Args[5])

// monmap init
cont = pod.Spec.InitContainers[1]
assert.Equal(t, "rook/rook:myversion", cont.Image)
assert.Equal(t, 0, len(cont.Ports))
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Nil(t, testop.VolumeMountExists(cephconfig.DefaultConfigMountName, cont.VolumeMounts))
assert.Equal(t, 0, len(cont.Env))
assert.False(t, *cont.SecurityContext.Privileged)
logCommandWithArgs("monmap init", cont.Command, cont.Args)
assert.Equal(t, 1, len(cont.Command))
assert.Equal(t, "/usr/bin/monmaptool", cont.Command[0])
assert.Equal(t, "/var/lib/rook/mon-a/monmap", cont.Args[0])
assert.Equal(t, "--create", cont.Args[1])
assert.Equal(t, "--clobber", cont.Args[2])
assert.Equal(t, "--fsid", cont.Args[3])
assert.Equal(t, c.clusterInfo.FSID, cont.Args[4])

// mon fs init
cont = pod.Spec.InitContainers[2]
assert.Equal(t, "rook/rook:myversion", cont.Image)
assert.Equal(t, 0, len(cont.Ports))
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Nil(t, testop.VolumeMountExists(cephconfig.DefaultConfigMountName, cont.VolumeMounts))
assert.Equal(t, 0, len(cont.Env))
assert.False(t, *cont.SecurityContext.Privileged)
logCommandWithArgs("mon fs init", cont.Command, cont.Args)
assert.Equal(t, 1, len(cont.Command))
assert.Equal(t, "/usr/bin/ceph-mon", cont.Command[0])
assert.Equal(t, "--mkfs", cont.Args[0])
assert.Equal(t, "--monmap", cont.Args[1])
assert.Equal(t, "/var/lib/rook/mon-a/monmap", cont.Args[2])
testCephMonCommonArgs(t, c, name, cont, 3)

// main mon daemon
cont = pod.Spec.Containers[0]
assert.Equal(t, "rook/rook:myversion", cont.Image)
assert.Equal(t, 1, len(cont.Ports))
// 6790/TCP
assert.Equal(t, cont.Ports[0].ContainerPort, int32(6790))
assert.Equal(t, cont.Ports[0].Protocol, v1.ProtocolTCP)
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Nil(t, testop.VolumeMountExists(cephconfig.DefaultConfigMountName, cont.VolumeMounts))
assert.Equal(t, 0, len(cont.Env))
assert.False(t, *cont.SecurityContext.Privileged)
logCommandWithArgs("main mon daemon", cont.Command, cont.Args)

assert.Equal(t, 1, len(cont.Command))
assert.Equal(t, "/usr/bin/ceph-mon", cont.Command[0])
assert.Equal(t, "--foreground", cont.Args[0])
assert.Equal(t, "--public-addr", cont.Args[1])
assert.Equal(t, "2.4.6.1:6790", cont.Args[2])
testCephMonCommonArgs(t, c, name, cont, 3)
configImage := "rook/rook:myversion"
configEnvs := 7
configContDev := test_opceph.ContainerTestDefinition{
Image: &configImage,
Command: []string{}, // no command
Args: [][]string{
{"ceph"},
{mondaemon.InitCommand},
{"--config-dir=/var/lib/rook"},
{fmt.Sprintf("--name=%s", name)},
{"--port=6790"},
{fmt.Sprintf("--fsid=%s", c.clusterInfo.FSID)}},
InOrderArgs: map[int]string{
0: "ceph", // ceph must be first arg
1: mondaemon.InitCommand}, // mgr init command must be second arg
VolumeMountNames: []string{
"rook-data",
cephconfig.DefaultConfigMountName,
k8sutil.ConfigOverrideName},
EnvCount: &configEnvs,
Ports: []v1.ContainerPort{},
IsPrivileged: &isPrivileged,
}
cont := &pod.Spec.InitContainers[0]
configContDev.TestContainer(t, "config init", cont, logger)
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())

// All ceph images have the same image, no envs, and the same volume mounts
cephImage := "rook/rook:myversion"
cephEnvs := 0
cephVolumeMountNames := []string{
"rook-data",
cephconfig.DefaultConfigMountName}

// monmap init container
monmapContDev := test_opceph.ContainerTestDefinition{
Image: &cephImage,
Command: []string{
"/usr/bin/monmaptool"},
Args: [][]string{
{"/var/lib/rook/mon-a/monmap"},
{"--create"},
{"--clobber"},
{"--fsid", c.clusterInfo.FSID}},
VolumeMountNames: cephVolumeMountNames,
EnvCount: &cephEnvs,
Ports: []v1.ContainerPort{},
IsPrivileged: &isPrivileged,
}
cont = &pod.Spec.InitContainers[1]
monmapContDev.TestContainer(t, "monmap init", cont, logger)
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())

// mon fs init container
monFsContDev := test_opceph.ContainerTestDefinition{
Image: &cephImage,
Command: []string{
"/usr/bin/ceph-mon"},
Args: append(
monCommonExpectedArgs(name, c),
[]string{"--mkfs"},
[]string{"--monmap", "/var/lib/rook/mon-a/monmap"}),
VolumeMountNames: cephVolumeMountNames,
EnvCount: &cephEnvs,
Ports: []v1.ContainerPort{},
IsPrivileged: &isPrivileged,
}
cont = &pod.Spec.InitContainers[2]
monFsContDev.TestContainer(t, "monmap init", cont, logger)
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())

// main mon daemon container
monDaemonContDev := test_opceph.ContainerTestDefinition{
Image: &cephImage,
Command: []string{
"/usr/bin/ceph-mon"},
Args: append(
monCommonExpectedArgs(name, c),
[]string{"--foreground"},
[]string{"--public-addr", "2.4.6.1:6790"}),
VolumeMountNames: cephVolumeMountNames,
EnvCount: &cephEnvs,
Ports: []v1.ContainerPort{
{ContainerPort: config.Port,
Protocol: v1.ProtocolTCP}},
IsPrivileged: &isPrivileged,
}
cont = &pod.Spec.Containers[0]
monDaemonContDev.TestContainer(t, "monmap init", cont, logger)
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())

// Verify that all the mounts have volumes and that there are no extraneous volumes
volsMountsTestDef := testop.VolumesAndMountsTestDefinition{
Expand All @@ -178,7 +203,3 @@ func testPodSpec(t *testing.T, dataDir string) {
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
}

func logCommandWithArgs(moniker string, command, args []string) {
logger.Infof("%s command : %s %s", moniker, strings.Join(command, " "), strings.Join(args, " "))
}
3 changes: 1 addition & 2 deletions pkg/operator/ceph/spec/spec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const (
ConfigInitContainerName = "config-init"
)

// PodVolumes returns the common list of Kubernetes volumes for use in Ceph pods.
// PodVolumes fills in the volumes parameter with the common list of Kubernetes volumes for use in Ceph pods.
func PodVolumes(dataDirHostPath string) []v1.Volume {
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
if dataDirHostPath != "" {
Expand All @@ -48,7 +48,6 @@ func CephVolumeMounts() []v1.VolumeMount {
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
cephconfig.DefaultConfigMount(),
// Rook doesn't run in ceph containers, so it doesn't need the config override mounted

}
}

Expand Down
27 changes: 10 additions & 17 deletions pkg/operator/ceph/spec/spec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,41 +20,34 @@ import (
"testing"

"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/testlib"
"github.com/rook/rook/pkg/operator/test"
)

func TestPodVolumes(t *testing.T) {
type args struct {
dataDirHostPath string
}
t.Run("Empty string dataDirHostPath is EmptyDir volume", func(t *testing.T) {
err := testlib.VolumeIsEmptyDir(k8sutil.DataDirVolume, PodVolumes(""))
err := test.VolumeIsEmptyDir(k8sutil.DataDirVolume, PodVolumes(""))
if err != nil {
t.Errorf("PodVolumes(\"\") - dataDirHostPath is not EmptyDir: %s", err.Error())
}
})
t.Run("Specified dataDirHostPath is HostPath volume", func(t *testing.T) {
err := testlib.VolumeIsHostPath(k8sutil.DataDirVolume, "/dev/sdb", PodVolumes("/dev/sdb"))
err := test.VolumeIsHostPath(k8sutil.DataDirVolume, "/dev/sdb", PodVolumes("/dev/sdb"))
if err != nil {
t.Errorf("PodVolumes(\"/dev/sdb\") - dataDirHostPath is not HostPath: %s", err.Error())
}
})
}

func TestMountsMatchVolumes(t *testing.T) {
volsSpec := testlib.VolumesSpec{
Moniker: "PodVolumes(\"/dev/sdc\")",
Volumes: PodVolumes("/dev/sdc"),
volsMountsTestDef := test.VolumesAndMountsTestDefinition{
VolumesSpec: &test.VolumesSpec{
Moniker: "PodVolumes(\"/dev/sdc\")", Volumes: PodVolumes("/dev/sdc")},
MountsSpecItems: []*test.MountsSpec{
{Moniker: "CephVolumeMounts()", Mounts: CephVolumeMounts()},
{Moniker: "RookVolumeMounts()", Mounts: RookVolumeMounts()}},
}
mountsSpecItems := []testlib.MountsSpec{
{
Moniker: "CephVolumeMounts()",
Mounts: CephVolumeMounts(),
},
{
Moniker: "RookVolumeMounts()",
Mounts: RookVolumeMounts(),
},
}
testlib.TestMountsMatchVolumes(t, volsSpec, mountsSpecItems...)
volsMountsTestDef.TestMountsMatchVolumes(t)
}
Loading

0 comments on commit 7a5fe84

Please sign in to comment.