Merge "[FAB-3494] Enhance reporting"

hyperledger · May 26, 2017 · 9683d72 · 9683d72
2 parents 30f7191 + ae16208
commit 9683d72
Show file tree

Hide file tree

Showing 12 changed files with 209 additions and 43 deletions.
diff --git a/bddtests/environment.py b/bddtests/environment.py
@@ -43,26 +43,27 @@ def after_scenario(context, scenario):
     contextHelper.after_scenario(scenario)
 
     get_logs = context.config.userdata.get("logs", "N")
-    if get_logs.lower() == "force" or (scenario.status == "failed" and get_logs.lower() == "y" and "compose_containers" in context):
-        print("Scenario {0} failed. Getting container logs".format(scenario.name))
-        file_suffix = "_" + scenario.name.replace(" ", "_") + ".log"
-        # get logs from the peer containers
-        for containerData in context.compose_containers:
-            (fileName, fileExists) = contextHelper.getTmpPathForName(name="{0}_{1}".format(containerData.containerName, scenario.name), extension="log", path_relative_to_tmp="logs")
-            with open(fileName, "w+") as logfile:
-                sys_rc = subprocess.call(["docker", "logs", containerData.containerName], stdout=logfile, stderr=logfile)
-                if sys_rc !=0 :
-                    print("Cannot get logs for {0}. Docker rc = {1}".format(containerData.containerName,sys_rc))
-        # get logs from the chaincode containers
-        cc_output, cc_error, cc_returncode = \
-            cli_call(["docker",  "ps", "-f",  "name=dev-", "--format", "{{.Names}}"], expect_success=True)
-        for containerName in cc_output.splitlines():
-            namePart,sep,junk = containerName.rpartition("-")
-            (fileName, fileExists) = contextHelper.getTmpPathForName(name="{0}_{1}".format(namePart, scenario.name), extension="log", path_relative_to_tmp="logs")
-            with open(fileName, "w+") as logfile:
-                sys_rc = subprocess.call(["docker", "logs", containerName], stdout=logfile, stderr=logfile)
-                if sys_rc !=0 :
-                    print("Cannot get logs for {0}. Docker rc = {1}".format(namepart,sys_rc))
+    if "compose_containers" in context:
+        if get_logs.lower() == "force" or (scenario.status == "failed" and get_logs.lower() == "y"):
+            print("Scenario {0} failed. Getting container logs".format(scenario.name))
+            file_suffix = "_" + scenario.name.replace(" ", "_") + ".log"
+            # get logs from the peer containers
+            for containerData in context.compose_containers:
+                (fileName, fileExists) = contextHelper.getTmpPathForName(name="{0}_{1}".format(containerData.containerName, scenario.name), extension="log", path_relative_to_tmp="logs")
+                with open(fileName, "w+") as logfile:
+                    sys_rc = subprocess.call(["docker", "logs", containerData.containerName], stdout=logfile, stderr=logfile)
+                    if sys_rc !=0 :
+                        print("Cannot get logs for {0}. Docker rc = {1}".format(containerData.containerName,sys_rc))
+            # get logs from the chaincode containers
+            cc_output, cc_error, cc_returncode = \
+                cli_call(["docker",  "ps", "-f",  "name=dev-", "--format", "{{.Names}}"], expect_success=True)
+            for containerName in cc_output.splitlines():
+                namePart,sep,junk = containerName.rpartition("-")
+                (fileName, fileExists) = contextHelper.getTmpPathForName(name="{0}_{1}".format(namePart, scenario.name), extension="log", path_relative_to_tmp="logs")
+                with open(fileName, "w+") as logfile:
+                    sys_rc = subprocess.call(["docker", "logs", containerName], stdout=logfile, stderr=logfile)
+                    if sys_rc !=0 :
+                        print("Cannot get logs for {0}. Docker rc = {1}".format(namepart,sys_rc))
     if 'doNotDecompose' in scenario.tags:
         if 'compose_yaml' in context:
             print("Not going to decompose after scenario {0}, with yaml '{1}'".format(scenario.name, context.compose_yaml))

diff --git a/bddtests/steps/bdd_grpc_util.py b/bddtests/steps/bdd_grpc_util.py
@@ -23,7 +23,7 @@ def getGRPCChannel(ipAddress, port, root_certificates, ssl_target_name_override)
     channel = grpc.secure_channel("{0}:{1}".format(ipAddress, port), creds,
                                   options=(('grpc.ssl_target_name_override', ssl_target_name_override,),('grpc.default_authority', ssl_target_name_override,),('grpc.max_receive_message_length', 100*1024*1024)))
 
-    print("Returning GRPC for address: {0}".format(ipAddress))
+    # print("Returning GRPC for address: {0}".format(ipAddress))
     return channel
 
 def toStringArray(items):

diff --git a/bddtests/steps/bootstrap_impl.py b/bddtests/steps/bootstrap_impl.py
@@ -144,7 +144,7 @@ def step_impl(context, userName, templateName, chainCreatePolicyName):
     ' At the moment, only really defining MSP Config Items (NOT SIGNED)'
     directory = bootstrap_util.getDirectory(context)
     user = directory.getUser(userName)
-    user.setTagValue(templateName, [directory.getOrganization(row['Organization']) for row in context.table.rows])
+    user.setTagValue(templateName, [directory.getOrganization(row['Organization']).name for row in context.table.rows])
 
 @given(u'the user "{userName}" creates a configUpdateEnvelope "{configUpdateEnvelopeName}" using configUpdate "{configUpdateName}"')
 def step_impl(context, userName, configUpdateEnvelopeName, configUpdateName):
@@ -234,9 +234,7 @@ def step_impl(context, userName, transactionAlias, orderer, channelId):
 def step_impl(context, userName, certAlias, composeService):
     directory = bootstrap_util.getDirectory(context)
     user = directory.getUser(userName=userName)
-    nodeAdminTuple = user.tags[certAlias]
-    cert = directory.findCertForNodeAdminTuple(nodeAdminTuple)
-    user.connectToDeliverFunction(context, composeService, certAlias, nodeAdminTuple=nodeAdminTuple)
+    user.connectToDeliverFunction(context, composeService, nodeAdminTuple=user.tags[certAlias])
 
 @when(u'user "{userName}" sends deliver a seek request on orderer "{composeService}" with properties')
 def step_impl(context, userName, composeService):

diff --git a/bddtests/steps/bootstrap_util.py b/bddtests/steps/bootstrap_util.py
@@ -162,12 +162,13 @@ def createCertificate(req, issuerCertKey, serial, validityPeriod, digest="sha256
 # SUBJECT_DEFAULT = {countryName : "US", stateOrProvinceName : "NC", localityName : "RTP", organizationName : "IBM", organizationalUnitName : "Blockchain"}
 
 class Entity:
-    def __init__(self, name):
+    def __init__(self, name, ecdsaSigningKey, rsaSigningKey):
         self.name = name
         # Create a ECDSA key, then a crypto pKey from the DER for usage with cert requests, etc.
-        self.ecdsaSigningKey = createECDSAKey()
-        self.rsaSigningKey = createRSAKey()
-        self.pKey = crypto.load_privatekey(crypto.FILETYPE_ASN1, self.ecdsaSigningKey.to_der())
+        self.ecdsaSigningKey = ecdsaSigningKey
+        self.rsaSigningKey = rsaSigningKey
+        if self.ecdsaSigningKey:
+            self.pKey = crypto.load_privatekey(crypto.FILETYPE_ASN1, self.ecdsaSigningKey.to_der())
         # Signing related ecdsa config
         self.hashfunc = hashlib.sha256
         self.sigencode = ecdsa.util.sigencode_der_canonize
@@ -200,10 +201,16 @@ def verifySignature(self, signature, signersCert, data):
     def getPrivateKeyAsPEM(self):
         return self.ecdsaSigningKey.to_pem()
 
+    def __getstate__(self):
+        state = dict(self.__dict__)
+        del state['ecdsaSigningKey']
+        del state['rsaSigningKey']
+        del state['pKey']
+        return state
 
 class User(Entity, orderer_util.UserRegistration):
-    def __init__(self, name, directory):
-        Entity.__init__(self, name)
+    def __init__(self, name, directory, ecdsaSigningKey, rsaSigningKey):
+        Entity.__init__(self, name, ecdsaSigningKey=ecdsaSigningKey, rsaSigningKey=rsaSigningKey)
         orderer_util.UserRegistration.__init__(self, name, directory)
         self.tags = {}
 
@@ -222,8 +229,8 @@ def cleanup(self):
 
 class Organization(Entity):
 
-    def __init__(self, name):
-        Entity.__init__(self, name)
+    def __init__(self, name, ecdsaSigningKey, rsaSigningKey):
+        Entity.__init__(self, name, ecdsaSigningKey, rsaSigningKey)
         req = createCertRequest(self.pKey, C="US", ST="North Carolina", L="RTP", O="IBM", CN=name)
         numYrs = 1
         self.signedCert = createCertificate(req, (req, self.pKey), 1000, (0, 60 * 60 * 24 * 365 * numYrs), isCA=True)
@@ -265,21 +272,23 @@ def addToNetwork(self, network):
 
 class Directory:
     def __init__(self):
+        import atexit
         self.organizations = {}
         self.users = {}
         self.ordererAdminTuples = {}
+        atexit.register(self.cleanup)
 
     def getNamedCtxTuples(self):
         return self.ordererAdminTuples
 
     def _registerOrg(self, orgName):
         assert orgName not in self.organizations, "Organization already registered {0}".format(orgName)
-        self.organizations[orgName] = Organization(orgName)
+        self.organizations[orgName] = Organization(orgName, ecdsaSigningKey = createECDSAKey(), rsaSigningKey = createRSAKey())
         return self.organizations[orgName]
 
     def _registerUser(self, userName):
         assert userName not in self.users, "User already registered {0}".format(userName)
-        self.users[userName] = User(userName, directory=self)
+        self.users[userName] = User(userName, directory=self, ecdsaSigningKey = createECDSAKey(), rsaSigningKey = createRSAKey())
         return self.users[userName]
 
     def getUser(self, userName, shouldCreate=False):
@@ -353,6 +362,48 @@ def registerOrdererAdminTuple(self, userName, ordererName, organizationName):
         self.ordererAdminTuples[ordererAdminTuple] = userCert
         return ordererAdminTuple
 
+    def dump(self, output):
+        'Will dump the directory to the provided store'
+        import cPickle
+        data = {'users' : {}, 'organizations' : {}, 'nats' : {}}
+        dump_cert = lambda cert: crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+        for userName, user in self.users.iteritems():
+            # for k, v in user.tags.iteritems():
+            #     try:
+            #         cPickle.dumps(v)
+            #     except:
+            #         raise Exception("Failed on key {0}".format(k))
+            data['users'][userName] = (user.ecdsaSigningKey.to_pem(), crypto.dump_privatekey(crypto.FILETYPE_PEM, user.rsaSigningKey), user.tags)
+        for orgName, org in self.organizations.iteritems():
+            networks = [n.name for n in  org.networks]
+            data['organizations'][orgName] = (
+                org.ecdsaSigningKey.to_pem(), crypto.dump_privatekey(crypto.FILETYPE_PEM, org.rsaSigningKey),
+                dump_cert(org.getSelfSignedCert()), networks)
+        for nat, cert in self.ordererAdminTuples.iteritems():
+            data['nats'][nat] = dump_cert(cert)
+        cPickle.dump(data, output)
+
+    def initFromPath(self, path):
+        'Will initialize the directory from the path supplied'
+        import cPickle
+        data = None
+        with open(path,'r') as f:
+            data = cPickle.load(f)
+        assert data != None, "Expected some data, did not load any."
+        priv_key_from_pem = lambda x: crypto.load_privatekey(crypto.FILETYPE_PEM, x)
+        for userName, keyTuple in data['users'].iteritems():
+            self.users[userName] = User(userName, directory=self,
+                                        ecdsaSigningKey=ecdsa.SigningKey.from_pem(keyTuple[0]),
+                                        rsaSigningKey=priv_key_from_pem(keyTuple[1]))
+            self.users[userName].tags = keyTuple[2]
+        for orgName, tuple in data['organizations'].iteritems():
+            org = Organization(orgName, ecdsaSigningKey=ecdsa.SigningKey.from_pem(keyTuple[0]),
+                               rsaSigningKey=priv_key_from_pem(keyTuple[0]))
+            org.signedCert = crypto.load_certificate(crypto.FILETYPE_PEM, tuple[2])
+            org.networks = [Network[name] for name in tuple[3]]
+            self.organizations[orgName] = org
+        for nat, cert_as_pem in data['nats'].iteritems():
+            self.ordererAdminTuples[nat] = crypto.load_certificate(crypto.FILETYPE_PEM, cert_as_pem)
 
 class AuthDSLHelper:
     @classmethod
@@ -1075,7 +1126,7 @@ def get_latest_configuration_block(deliverer_stream_helper, channel_id):
         deliverer_stream_helper.seekToRange(chainID=channel_id, start=last_config.index, end=last_config.index)
         blocks = deliverer_stream_helper.getBlocks()
         assert len(blocks) == 1, "Expected single block, received: {0} blocks".format(len(blocks))
-        assert len(block.data.data) == 1, "Expected single transaction for configuration block, instead found {0} transactions".format(len(block.data.data))
+        assert len(blocks[0].data.data) == 1, "Expected single transaction for configuration block, instead found {0} transactions".format(len(block.data.data))
         latest_config_block = blocks[0]
     return latest_config_block
 

diff --git a/bddtests/steps/compose.py b/bddtests/steps/compose.py
@@ -193,7 +193,7 @@ def issueCommand(self, command, components=[]):
         return output
 
     def rebuildContainerData(self):
-        self.containerDataList = []
+        self.containerDataList[:] = []
         for containerID in self.refreshContainerIDs():
 
             # get container metadata

diff --git a/bddtests/steps/contexthelper.py b/bddtests/steps/contexthelper.py
@@ -8,6 +8,27 @@
 import shutil
 from slugify import slugify
 
+
+class Context(object):
+    def __init__(self):
+        pass
+
+    def __setattr__(self, attr, value):
+        self.__dict__[attr] = value
+
+    def __getattr__(self, attr):
+        return self.__dict__[attr]
+        # raise AttributeError(e)
+
+    def __getstate__(self):
+        return self.__dict__
+
+    def __setstate__(self, value):
+        return self.__dict__.update(value)
+
+    def __contains__(self, attr):
+        return attr in self.__dict__
+
 class ContextHelper:
 
     @classmethod

diff --git a/bddtests/steps/docgen.py b/bddtests/steps/docgen.py
@@ -137,6 +137,10 @@ def registerCompositionAdvice(self, joinpoint):
             #Now get the config for the composition and dump out.
             self.composition = composition
             configAsYaml = composition.getConfig()
+            (dokerComposeYmlFileName, fileExists) = self.contextHelper.getTmpPathForName(name="docker-compose", extension="yml")
+            with open(dokerComposeYmlFileName, 'w') as f:
+                f.write(configAsYaml)
+            self.output.write(env.get_template("html/composition-py.html").render(compose_project_name= self.composition.projectName,docker_compose_yml_file=dokerComposeYmlFileName))
             self.output.write(env.get_template("html/header.html").render(text="Configuration", level=4))
             self.output.write(env.get_template("html/cli.html").render(command=configAsYaml))
             #Inject the graph
@@ -180,8 +184,23 @@ def registerUserAdvice(self, joinpoint):
         self.output.write(env.get_template("html/user.html").render(user=newlyRegisteredUser, private_key_href=self._getLinkInfoForFile(fileName)))
         return newlyRegisteredUser
 
+    def _dump_context(self):
+        (dirPickleFileName, fileExists) = self.contextHelper.getTmpPathForName("dir", extension="pickle")
+        with open(dirPickleFileName, 'w') as f:
+            self.directory.dump(f)
+        #Now the jinja output
+        self.output.write(env.get_template("html/directory.html").render(directory=self.directory, path_to_pickle=dirPickleFileName))
+        if self.composition:
+            (dokerComposeYmlFileName, fileExists) = self.contextHelper.getTmpPathForName(name="docker-compose", extension="yml")
+            self.output.write(env.get_template("html/appendix-py.html").render(directory=self.directory,
+                                                                               path_to_pickle=dirPickleFileName,
+                                                                               compose_project_name=self.composition.projectName,
+                                                                               docker_compose_yml_file=dokerComposeYmlFileName))
+
+
     def afterScenarioAdvice(self, joinpoint):
         scenario = joinpoint.kwargs['scenario']
+        self._dump_context()
         #Render with jinja
         header = env.get_template("html/scenario.html").render(scenario=scenario, steps=scenario.steps)
         main = env.get_template("html/main.html").render(header=header, body=self.output.getvalue())

diff --git a/bddtests/steps/orderer_util.py b/bddtests/steps/orderer_util.py
@@ -103,7 +103,7 @@ def handleNetworkError(self, networkError):
 
 class DeliverStreamHelper(StreamHelper):
 
-    def __init__(self, ordererStub, entity, directory, nodeAdminTuple, timeout = 110):
+    def __init__(self, ordererStub, entity, directory, nodeAdminTuple, timeout = 600):
         StreamHelper.__init__(self)
         self.nodeAdminTuple = nodeAdminTuple
         self.directory = directory
@@ -118,8 +118,6 @@ def createSeekInfo(self, chainID, start = 'Oldest', end = 'Newest',  behavior =
             stop = seekPosition(end),
             behavior = ab_pb2.SeekInfo.SeekBehavior.Value(behavior),
         )
-        print("SeekInfo = {0}".format(seekInfo))
-        print("")
         return seekInfo
 
     def seekToRange(self, chainID = TEST_CHAIN_ID, start = 'Oldest', end = 'Newest'):
@@ -138,7 +136,7 @@ def getBlocks(self):
                 else:
                     if reply.status != common_pb2.SUCCESS:
                         print("Got error: {0}".format(reply.status))
-                    print("Done receiving blocks")
+                    # print("Done receiving blocks")
                     break
         except Exception as e:
             print("getBlocks got error: {0}".format(e) )
@@ -163,7 +161,7 @@ def closeStreams(self):
         for compose_service, deliverStreamHelper in self.abDeliversStreamHelperDict.iteritems():
             deliverStreamHelper.sendQueue.put(None)
 
-    def connectToDeliverFunction(self, context, composeService, certAlias, nodeAdminTuple, timeout=1):
+    def connectToDeliverFunction(self, context, composeService, nodeAdminTuple, timeout=1):
         'Connect to the deliver function and drain messages to associated orderer queue'
         assert not composeService in self.abDeliversStreamHelperDict, "Already connected to deliver stream on {0}".format(composeService)
         streamHelper = DeliverStreamHelper(directory=self.directory,
@@ -200,7 +198,7 @@ def getABStubForComposeService(self, context, composeService):
         # Get the IP address of the server that the user registered on
         root_certificates = self.directory.getTrustedRootsForOrdererNetworkAsPEM()
         ipAddress, port = bdd_test_util.getPortHostMapping(context.compose_containers, composeService, 7050)
-        print("ipAddress in getABStubForComposeService == {0}:{1}".format(ipAddress, port))
+        # print("ipAddress in getABStubForComposeService == {0}:{1}".format(ipAddress, port))
         channel = bdd_grpc_util.getGRPCChannel(ipAddress=ipAddress, port=port, root_certificates=root_certificates, ssl_target_name_override=composeService)
         newABStub = ab_pb2_grpc.AtomicBroadcastStub(channel)
         self.atomicBroadcastStubsDict[composeService] = newABStub