diff --git a/.changelog/unreleased/improvements/1559-e2e-latency-emulation.md b/.changelog/unreleased/improvements/1559-e2e-latency-emulation.md new file mode 100644 index 0000000000..02e8d0a035 --- /dev/null +++ b/.changelog/unreleased/improvements/1559-e2e-latency-emulation.md @@ -0,0 +1,2 @@ +- `[e2e]` Allow latency emulation between nodes. + ([\#1559](https://github.com/cometbft/cometbft/pull/1559)) \ No newline at end of file diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go index 8061db4012..4295876054 100644 --- a/test/e2e/pkg/infra/digitalocean/digitalocean.go +++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go @@ -20,8 +20,13 @@ type Provider struct { infra.ProviderData } -// Noop currently. Setup is performed externally to the e2e test tool. +// Setup generates the file mapping IPs to zones, used for emulating latencies. func (p *Provider) Setup() error { + err := infra.GenerateIPZonesTable(p.Testnet.Nodes, p.IPZonesFilePath(), false) + if err != nil { + return err + } + return nil } @@ -47,9 +52,32 @@ func (p Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error { return execAnsible(ctx, p.Testnet.Dir, playbookFile, nodeIPs) } -// Currently unsupported. -func (p Provider) SetLatency(_ context.Context, _ *e2e.Node) error { - return fmt.Errorf("SetLatency() currently unsupported for Digital Ocean") +// SetLatency prepares and executes the latency-setter script in the given node. +func (p Provider) SetLatency(ctx context.Context, node *e2e.Node) error { + // Directory in the DigitalOcean node that contains all latency files. + remoteDir := "/root/cometbft/test/e2e/latency/" + + playbook := "- name: e2e custom playbook\n" + + " hosts: all\n" + + " tasks:\n" + + // Add task to copy the necessary files to the node. + playbook = ansibleAddCopyTask(playbook, "copy zones file to node", filepath.Base(p.IPZonesFilePath()), remoteDir) + + // Add task to execute latency-setter script in the node. + cmd := fmt.Sprintf("%s set %s %s eth0", + filepath.Join(remoteDir, "latency-setter.py"), + filepath.Join(remoteDir, filepath.Base(p.IPZonesFilePath())), + filepath.Join(remoteDir, "aws-latencies.csv"), + ) + playbook = ansibleAddShellTasks(playbook, "execute latency setter script", cmd) + + // Execute playbook + playbookFile := getNextPlaybookFilename() + if err := p.writePlaybook(playbookFile, playbook); err != nil { + return err + } + return execAnsible(ctx, p.Testnet.Dir, playbookFile, []string{node.ExternalIP.String()}) } func (p Provider) StopTestnet(ctx context.Context) error { @@ -109,7 +137,15 @@ const basePlaybook = `- name: e2e custom playbook ` func ansibleAddTask(playbook, name, contents string) string { - return playbook + " - name: " + name + "\n" + contents + return playbook + " - name: " + name + "\n" + contents + "\n" +} + +func ansibleAddCopyTask(playbook, name, src, dest string) string { + copyTask := fmt.Sprintf(" ansible.builtin.copy:\n"+ + " src: %s\n"+ + " dest: %s\n", + src, dest) + return ansibleAddTask(playbook, name, copyTask) } func ansibleAddSystemdTask(playbook string, starting bool) string { @@ -117,6 +153,7 @@ func ansibleAddSystemdTask(playbook string, starting bool) string { if starting { startStop = "started" } + // testappd is the name of the deamon running the node in the ansible scripts in the qa-infra repo. contents := fmt.Sprintf(` ansible.builtin.systemd: name: testappd state: %s diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go index b13dff527b..a1685054a3 100644 --- a/test/e2e/pkg/infra/docker/docker.go +++ b/test/e2e/pkg/infra/docker/docker.go @@ -32,13 +32,7 @@ func (p *Provider) Setup() error { return err } - // Generate file with table mapping IP addresses to geographical zone for latencies. - zonesTable, err := zonesTableBytes(p.Testnet.Nodes) - if err != nil { - return err - } - //nolint: gosec // G306: Expect WriteFile permissions to be 0600 or less - err = os.WriteFile(filepath.Join(p.Testnet.Dir, "zones.csv"), zonesTable, 0o644) + err = infra.GenerateIPZonesTable(p.Testnet.Nodes, p.IPZonesFilePath(), true) if err != nil { return err } @@ -85,15 +79,14 @@ func (p Provider) SetLatency(ctx context.Context, node *e2e.Node) error { containerDir := "/scripts/" // Copy zone file used by the script that sets latency. - zonesFile := filepath.Join(p.Testnet.Dir, "zones.csv") - if err := Exec(ctx, "cp", zonesFile, node.Name+":"+containerDir); err != nil { + if err := Exec(ctx, "cp", p.IPZonesFilePath(), node.Name+":"+containerDir); err != nil { return err } // Execute the latency setter script in the container. if err := ExecVerbose(ctx, "exec", "--privileged", node.Name, filepath.Join(containerDir, "latency-setter.py"), "set", - filepath.Join(containerDir, "zones.csv"), + filepath.Join(containerDir, filepath.Base(p.IPZonesFilePath())), filepath.Join(containerDir, "aws-latencies.csv"), "eth0"); err != nil { return err } @@ -188,24 +181,6 @@ services: return buf.Bytes(), nil } -func zonesTableBytes(nodes []*e2e.Node) ([]byte, error) { - tmpl, err := template.New("zones").Parse(`Node,IP,Zone -{{- range . }} -{{- if .Zone }} -{{ .Name }},{{ .InternalIP }},{{ .Zone }} -{{- end }} -{{- end }}`) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, nodes) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - // ExecCompose runs a Docker Compose command for a testnet. func ExecCompose(ctx context.Context, dir string, args ...string) error { return exec.Command(ctx, append( diff --git a/test/e2e/pkg/infra/latencies.go b/test/e2e/pkg/infra/latencies.go new file mode 100644 index 0000000000..0d2223b5ee --- /dev/null +++ b/test/e2e/pkg/infra/latencies.go @@ -0,0 +1,48 @@ +package infra + +import ( + "bytes" + "os" + "text/template" + + e2e "github.com/cometbft/cometbft/test/e2e/pkg" +) + +// GenerateIPZonesTable generates a file with a table mapping IP addresses to geographical zone for latencies. +func GenerateIPZonesTable(nodes []*e2e.Node, zonesPath string, useInternalIP bool) error { + // Generate file with table mapping IP addresses to geographical zone for latencies. + zonesTable, err := zonesTableBytes(nodes, useInternalIP) + if err != nil { + return err + } + //nolint: gosec // G306: Expect WriteFile permissions to be 0600 or less + err = os.WriteFile(zonesPath, zonesTable, 0o644) + if err != nil { + return err + } + return nil +} + +func zonesTableBytes(nodes []*e2e.Node, useInternalIP bool) ([]byte, error) { + tmpl, err := template.New("zones").Parse(`Node,IP,Zone +{{- range .Nodes }} +{{- if .Zone }} +{{ .Name }},{{ if $.UseInternalIP }}{{ .InternalIP }}{{ else }}{{ .ExternalIP }}{{ end }},{{ .Zone }} +{{- end }} +{{- end }}`) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, struct { + Nodes []*e2e.Node + UseInternalIP bool + }{ + Nodes: nodes, + UseInternalIP: useInternalIP, + }) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go index 0a9fa28408..9151a09bc7 100644 --- a/test/e2e/pkg/infra/provider.go +++ b/test/e2e/pkg/infra/provider.go @@ -2,6 +2,7 @@ package infra import ( "context" + "path/filepath" e2e "github.com/cometbft/cometbft/test/e2e/pkg" ) @@ -9,7 +10,6 @@ import ( // Provider defines an API for manipulating the infrastructure of a // specific set of testnet infrastructure. type Provider interface { - // Setup generates any necessary configuration for the infrastructure // provider during testnet setup. Setup() error @@ -44,7 +44,12 @@ type ProviderData struct { InfrastructureData e2e.InfrastructureData } -// Returns the the provider's infrastructure data +// GetInfrastructureData returns the provider's infrastructure data. func (pd ProviderData) GetInfrastructureData() *e2e.InfrastructureData { return &pd.InfrastructureData } + +// IPZonesFilePath returns the path to the file with the mapping from IP addresses to zones. +func (pd ProviderData) IPZonesFilePath() string { + return filepath.Join(pd.Testnet.Dir, "zones.csv") +}