Add Custom Scenario for Performance Testing (algorand#3278)

Add Custom Scenario for Performance Testing. Add README on how to run custom scenario and modify create_and_deploy_recipe.sh to accept a network template that will generate a new recipe.
rikokir · Dec 6, 2021 · 70ff3c7 · 70ff3c7
1 parent e466aa1
commit 70ff3c7
Show file tree

Hide file tree

Showing 15 changed files with 339 additions and 5 deletions.
diff --git a/scripts/create_and_deploy_recipe.sh b/scripts/create_and_deploy_recipe.sh
@@ -2,7 +2,7 @@
 
 # create_and_deploy_recipe.sh - Generates deployed network configuration (based on a recipe) and private build and pushes to S3
 #
-# Syntax:   create_and_deploy_recipe.sh -c <channel/network> [-n network] --recipe <recipe file> -r <rootdir> [--nodeploy] [--skip-build] [--force] [-m genesisVersionModifier] [ -b <bucket> ]"
+# Syntax:   create_and_deploy_recipe.sh -c <channel/network> [-n network] --recipe <recipe file> -r <rootdir> [--nodeploy] [--skip-build] [--force] [-m genesisVersionModifier] [ -b <bucket> ] [--template <template_path>]"
 #
 # Outputs:  <errors or warnings>
 #
@@ -15,6 +15,9 @@
 # Examples: create_and_deploy_recipe.sh -c TestCatchup --recipe test/testdata/deployednettemplates/recipes/devnet-like.config -r ~/networks/gen
 #
 # Notes:    If you're running on a Mac, this will attempt to use docker to build for linux.
+#           If you need to generate a recipe, you can pass in the --template flag with the path to the
+#           template json and it will run the generate_recipe.py on the file. Make sure it's in the same
+#           directory as the recipe file path.
 
 set -e
 
@@ -80,6 +83,10 @@ while [ "$1" != "" ]; do
         --skip-build)
             SKIP_BUILD="true"
             ;;
+        --template)
+            shift
+            NETWORK_TEMPLATE="$1"
+            ;;
         *)
             echo "Unknown option" "$1"
             exit 1
@@ -89,7 +96,7 @@ while [ "$1" != "" ]; do
 done
 
 if [[ -z "${CHANNEL}" || -z "${RECIPEFILE}" || -z "${ROOTDIR}" ]]; then
-    echo "Syntax: create_and_deploy_recipe.sh -c <channel/network> [-n network] --recipe <recipe file> -r <rootdir> [--nodeploy] [--force]"
+    echo "Syntax: create_and_deploy_recipe.sh -c <channel/network> [-n network] --recipe <recipe file> -r <rootdir> [--nodeploy] [--force] [--template <template_path>]"
     echo "e.g. create_and_deploy_recipe.sh -c TestCatchup --recipe test/testdata/deployednettemplates/recipes/devnet-like.config -r ~/networks/<channel>/gen"
     exit 1
 fi
@@ -111,6 +118,11 @@ if [[ "${SKIP_BUILD}" != "true" || ! -f ${GOPATH}/bin/netgoal ]]; then
     (cd ${SRCPATH} && make)
 fi
 
+# If template is passed in, a recipe will be generated in the same folder as the template
+if [[ ! -z ${NETWORK_TEMPLATE} ]]; then
+    python3 ${SRCPATH}/test/testdata/deployednettemplates/generate-recipe/generate_network.py -f ${NETWORK_TEMPLATE}
+fi
+
 # Generate the nodecfg package directory
 ${GOPATH}/bin/netgoal build -r "${ROOTDIR}" -n "${NETWORK}" --recipe "${RECIPEFILE}" "${FORCE_OPTION}" -m "${SCHEMA_MODIFIER}" -b=${BOOTSTRAP:-true}
 

diff --git a/test/testdata/deployednettemplates/generate-recipe/generate_network.py b/test/testdata/deployednettemplates/generate-recipe/generate_network.py
@@ -25,7 +25,7 @@ def build_network(template):
 
     netgoal_params = build_netgoal_params(template_dict)
     build_net(template_path, netgoal_params)
-    build_genesis(template_path, netgoal_params)
+    build_genesis(template_path, netgoal_params, template_dict)
 
 def build_netgoal_params(template_dict):
     instances = template_dict['instances']
@@ -38,7 +38,6 @@ def build_netgoal_params(template_dict):
         relay_count += getInstanceCount(instances['relays'], group['percent']['relays'])
         participating_node_count += getInstanceCount(instances['participatingNodes'], group['percent']['participatingNodes'])
         non_participating_node_count += getInstanceCount(instances['nonParticipatingNodes'], group['percent']['nonParticipatingNodes'])
-
 
     relay_config = instances['relays']['config']
     participating_node_config = instances['participatingNodes']['config']
@@ -66,13 +65,22 @@ def build_net(template_path, netgoal_params):
     args.extend(netgoal_params)
     netgoal(args, template_path)
 
-def build_genesis(template_path, netgoal_params):
+def build_genesis(template_path, netgoal_params, template_dict):
     args = [
         '-t', 'genesis',
         '-o', f"{template_path}/generated/genesis.json"
     ]
     args.extend(netgoal_params)
     netgoal(args, template_path)
+    if template_dict['network']['ConsensusProtocol']:
+        updateProtocol(f"{template_path}/generated/genesis.json", template_dict['network']['ConsensusProtocol'])
+
+def updateProtocol(genesis_path, consensus_protocol):
+    with open(genesis_path, 'r') as genfile:
+        genjson = json.load(genfile)
+        genjson["ConsensusProtocol"] = consensus_protocol
+    with open(genesis_path, 'w') as genfile:
+        json.dump(genjson, genfile, indent="\t")
 
 def netgoal(args, template_path='.'):
     cmd = [

diff --git a/test/testdata/deployednettemplates/recipes/custom/README.md b/test/testdata/deployednettemplates/recipes/custom/README.md
@@ -0,0 +1,84 @@
+# Custom Recipe
+This custom recipe serves as a template for performance testing on algonet (new network on AWS EC2 machines). With this recipe, you can modify the number of nodes, the type of machines, introduce new parameters to modify the network's configs and consensus parameters.
+
+N = participating Nodes
+NPN = Non-Participating Nodes
+R = relays
+
+## Running a Small Network (less than 20 total nodes)
+If you are running a network with less than 20 nodes, then you will need to update the default "FractionApply"
+1. Modify `configs/node.json` folder
+    - `"FractionApply"` in configs/node.json represents the number of nodes to report to telemetry. We don't want to overwhelm the telemetry server, so use something small like "0.2" on a large network.
+    - For small networks, update this value to "1.0"
+
+## Quick Start - Jenkins
+Build and create the recipe.
+- (See the first section above for small networks.)
+1. Modify the `network_templates/network-tpl.json` file.
+2. Select "custom" recipe
+3. Specify `network-tpl.json` as the `CUSTOM_NETWORK_TEMPLATE`
+- See Modify consensus values (below) to update consensus
+- See Update config.json (below) to update config.json
+
+## "Quick" Start - Manual recipe generation (not using Jenkins)
+Generate the recipe with the `network-tpl.json` file
+- (See the first section above for small networks.)
+1. Make sure you're in the same directory as this README and `cp network_templates/network-tpl.json network-tpl.json`
+2. Generate the recipe with a python script:
+```
+cd go-algorand
+python3 test/testdata/deployednettemplates/generate-recipe/generate_network.py -f test/testdata/deployednettemplates/recipes/custom/network-tpl.json
+```
+3. This will create a new set of files in the `generated` folder
+
+## Network Templates
+With the custom recipe, you can store multiple network templates in the network_templates directory.
+Variables to modify:
+- `wallets`: Number of wallets used by N
+- `nodes`: Number of N
+- `ConsensusProtocol`: ConsensusProtocol used for the genesis
+- `type`: machine sizes. For `us-east-2`, you can use `m5d.4xl` for most testing. If you need more powerful compute (make sure you get approval because it can get costly) use `c4d.4xl` or `c4d.18xl`
+- `count`: Number of machines per type
+- `percent`: percentage of machines in group to dedicate to certain types of nodes.
+
+## Modify consensus values
+If you add a `consensus.json` file in this folder with the protocol matching the one in `network-tpl.json`, the `consensus.json` will merge with a generated_consensus.json template on Jenkins.
+- see `example/consensus.json`
+
+### How is consensus updated in Jenkins?
+- In Jenkins, this will be generated via `goal protocols > generated_consensus.json`
+- This means that you do not have to provide the whole `consensus.json` in this folder, but only the values you wish to update.
+- If you are spinning up a network manually and wish to update a network with `consensus.json`, you must have all of the existing keys for the particular protocol in your consensus.json.
+
+## Update config.json in the network
+If you look at the files in the "configs" folder, you will see `node.json`, `nonPartNode.json`, and `relay.json`. These jsons already have a `ConfigJSONOverride` parameter which will generate a config.json in the node's data directories. For testing, if you want to update all three types of nodes at once, you can save a `config.json` file here.
+1. copy and paste something like this into a json file and save into `config_jsons`:
+```
+{
+  "ProposalAssemblyTime": 250000000,
+  "TxPoolSize": 20000
+}
+```
+This file will merge with the config.json created by `ConfigJSONOverride` and update the parameters if the keys match. This will be applied to participating nodes, nonParticipating Nodes, and relays.
+
+See `example/config_jsons` for an example of what it should look like.
+
+Most parameters that can be modified by config.json can be found in `go-algorand/config/local_defaults.go`.
+
+## Troubleshooting
+### Can't find netgoal
+- Make sure you have netgoal installed
+- Make sure you export GOBIN and GOPATH in your environment and add it to your path.
+On a mac, update by editing `~/.zshrc`, add
+```
+export GOBIN=/Users/{user}/go/bin
+
+export GOPATH=/Users/{user}/go
+export PATH=$PATH:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/Users/ec2-user/Library/Python/3.8/bin:/usr/local/go/bin:$GOBIN:$GOPATH
+
+```
+### Machine Type doesn't exist
+- Make sure the machine type exists. It uses the regions in the groups and the type to come up with the host template name in `test/testdata/deployednettemplates/hosttemplates/hosttemplates.json`. If it doesn't exist, you will have to add it to that file.
+
+### couldn't initialize the node: unsupported protocol
+- check your consensus.json. It may be missing the keys in the future protocol if you are doing this manually. Compare the consensus.json with `goal protocols > generated_consensus.json`
diff --git a/test/testdata/deployednettemplates/recipes/custom/configs/node.json b/test/testdata/deployednettemplates/recipes/custom/configs/node.json
@@ -0,0 +1,22 @@
+{
+    "APIToken": "{{APIToken}}",
+    "EnableBlockStats": false,
+    "EnableTelemetry": false,
+    "TelemetryURI": "{{TelemetryURI}}",
+    "EnableMetrics": false,
+    "MetricsURI": "{{MetricsURI}}",
+    "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true, \"CadaverSizeTarget\": 0 }",
+    "AltConfigs": [
+        {
+            "APIToken": "{{APIToken}}",
+            "EnableBlockStats": true,
+            "EnableTelemetry": true,
+            "TelemetryURI": "{{TelemetryURI}}",
+            "EnableMetrics": true,
+            "MetricsURI": "{{MetricsURI}}",
+            "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true, \"CadaverSizeTarget\": 0 }",
+            "FractionApply": 1.0
+        }
+    ]
+}
+
diff --git a/test/testdata/deployednettemplates/recipes/custom/configs/nonPartNode.json b/test/testdata/deployednettemplates/recipes/custom/configs/nonPartNode.json
@@ -0,0 +1,5 @@
+{
+    "APIEndpoint": "{{APIEndpoint}}",
+    "APIToken": "{{APIToken}}",
+    "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4, \"CadaverSizeTarget\": 0  }"
+}
diff --git a/test/testdata/deployednettemplates/recipes/custom/configs/relay.json b/test/testdata/deployednettemplates/recipes/custom/configs/relay.json
@@ -0,0 +1,11 @@
+{
+    "NetAddress": "{{NetworkPort}}",
+    "APIEndpoint": "{{APIEndpoint}}",
+    "APIToken": "{{APIToken}}",
+    "EnableBlockStats": true,
+    "EnableTelemetry": true,
+    "TelemetryURI": "{{TelemetryURI}}",
+    "EnableMetrics": true,
+    "MetricsURI": "{{MetricsURI}}",
+    "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }"
+}
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/README.md b/test/testdata/deployednettemplates/recipes/custom/example/README.md
@@ -0,0 +1,12 @@
+# Example for custom recipe
+Feel free to copy over and replace custom recipe with the contents in this folder or read the descriptions below.
+
+## config_jsons
+example of config.json that you can include. The `custom/config_jsons` folder has a README with more information.
+
+## netowrk_templates
+Shows that you can have more than one network template in the custom recipe. You can specify the template in the Jenkins Pipeline.
+
+## consensus.json
+An example of the consensus.json that you can copy into the custom folder to update the "future" protocol.
+
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/config_jsons/config.json b/test/testdata/deployednettemplates/recipes/custom/example/config_jsons/config.json
@@ -0,0 +1,4 @@
+{
+    "ProposalAssemblyTime": 250000000,
+    "TxPoolSize": 20000
+}
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/configs/node.json b/test/testdata/deployednettemplates/recipes/custom/example/configs/node.json
@@ -0,0 +1,22 @@
+{
+    "APIToken": "{{APIToken}}",
+    "EnableBlockStats": false,
+    "EnableTelemetry": false,
+    "TelemetryURI": "{{TelemetryURI}}",
+    "EnableMetrics": false,
+    "MetricsURI": "{{MetricsURI}}",
+    "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true, \"CadaverSizeTarget\": 0 }",
+    "AltConfigs": [
+        {
+            "APIToken": "{{APIToken}}",
+            "EnableBlockStats": true,
+            "EnableTelemetry": true,
+            "TelemetryURI": "{{TelemetryURI}}",
+            "EnableMetrics": true,
+            "MetricsURI": "{{MetricsURI}}",
+            "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true, \"CadaverSizeTarget\": 0 }",
+            "FractionApply": 1.0
+        }
+    ]
+}
+
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/configs/nonPartNode.json b/test/testdata/deployednettemplates/recipes/custom/example/configs/nonPartNode.json
@@ -0,0 +1,5 @@
+{
+    "APIEndpoint": "{{APIEndpoint}}",
+    "APIToken": "{{APIToken}}",
+    "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4, \"CadaverSizeTarget\": 0  }"
+}
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/configs/relay.json b/test/testdata/deployednettemplates/recipes/custom/example/configs/relay.json
@@ -0,0 +1,11 @@
+{
+    "NetAddress": "{{NetworkPort}}",
+    "APIEndpoint": "{{APIEndpoint}}",
+    "APIToken": "{{APIToken}}",
+    "EnableBlockStats": true,
+    "EnableTelemetry": true,
+    "TelemetryURI": "{{TelemetryURI}}",
+    "EnableMetrics": true,
+    "MetricsURI": "{{MetricsURI}}",
+    "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \"<network>.algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }"
+}
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/consensus.json b/test/testdata/deployednettemplates/recipes/custom/example/consensus.json
@@ -0,0 +1,6 @@
+{
+    "future": {
+      "AgreementFilterTimeoutPeriod0": 4000000001,
+      "MaxTxnBytesPerBlock": 1100001
+    }
+}
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/network_templates/c5dmachines.json b/test/testdata/deployednettemplates/recipes/custom/example/network_templates/c5dmachines.json
@@ -0,0 +1,44 @@
+{
+  "network": {
+    "wallets": 6,
+    "nodes": 3,
+    "ConsensusProtocol": "future"
+  },
+  "instances": {
+    "relays": {
+      "config": "./configs/relay.json",
+      "type": "c5d.4xl",
+      "count": 1
+    },
+    "participatingNodes": {
+      "config": "./configs/node.json",
+      "type": "c5d.4xl",
+      "count": 3
+    },
+    "nonParticipatingNodes": {
+      "config": "./configs/nonPartNode.json",
+      "type": "c5d.4xl",
+      "count": 5
+    }
+  },
+  "groups": [
+    {
+      "name": "us-r",
+      "percent": {
+        "relays": 100,
+        "participatingNodes": 0,
+        "nonParticipatingNodes": 0
+      },
+      "region": "us-east-2"
+    },
+    {
+      "name": "us-n",
+      "percent": {
+        "relays": 0,
+        "participatingNodes": 100,
+        "nonParticipatingNodes": 100
+      },
+      "region": "us-east-2"
+    }
+  ]
+}
diff --git a/test/testdata/deployednettemplates/recipes/custom/example/network_templates/network-tpl.json b/test/testdata/deployednettemplates/recipes/custom/example/network_templates/network-tpl.json
@@ -0,0 +1,44 @@
+{
+  "network": {
+    "wallets": 6,
+    "nodes": 3,
+    "ConsensusProtocol": "future"
+  },
+  "instances": {
+    "relays": {
+      "config": "./configs/relay.json",
+      "type": "m5d.4xl",
+      "count": 1
+    },
+    "participatingNodes": {
+      "config": "./configs/node.json",
+      "type": "m5d.4xl",
+      "count": 3
+    },
+    "nonParticipatingNodes": {
+      "config": "./configs/nonPartNode.json",
+      "type": "m5d.4xl",
+      "count": 5
+    }
+  },
+  "groups": [
+    {
+      "name": "us-r",
+      "percent": {
+        "relays": 100,
+        "participatingNodes": 0,
+        "nonParticipatingNodes": 0
+      },
+      "region": "us-east-2"
+    },
+    {
+      "name": "us-n",
+      "percent": {
+        "relays": 0,
+        "participatingNodes": 100,
+        "nonParticipatingNodes": 100
+      },
+      "region": "us-east-2"
+    }
+  ]
+}