mdf3reader : corrected bug of mask calculation for bits in bytes

Reorder channels according to its firstbit number, CANAPE putting time at the end. Started next release 1.1
sneusse · Apr 28, 2015 · 53a7081 · 53a7081
1 parent 3da3936
commit 53a7081
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 10 deletions.
diff --git a/mdf3reader.py b/mdf3reader.py
@@ -81,7 +81,7 @@ def processDataBlocks(Q, buf, info, dataGroup, channelList, multiProc):
 
                 # Process concatenated bits inside uint8
                 if not chan.bitCount // 8.0 == chan.bitCount / 8.0:  # if channel data do not use complete bytes
-                    mask = int(pow(2, chan.bitCount + 1) - 1)  # masks isBitUnit8
+                    mask = int(pow(2, chan.bitCount) - 1)  # masks isBitUnit8
                     if chan.signalDataType in (0, 1, 9, 10, 13, 14):  # integers
                         temp = right_shift(temp, chan.bitOffset)
                         temp = bitwise_and(temp, mask)
@@ -1255,39 +1255,43 @@ def arrayformat3(signalDataType, numberOfBits):
     if signalDataType in (0, 9, 10, 11):  # unsigned
         if numberOfBits <= 8:
             dataType = 'uint8'
-        elif numberOfBits == 16:
+        elif numberOfBits <= 16:
             dataType = 'uint16'
-        elif numberOfBits == 32:
+        elif numberOfBits <= 32:
             dataType = 'uint32'
+        elif numberOfBits <= 64:
+            dataType = 'uint64'
         elif numberOfBits == 1:
             dataType = 'uint8'  # not directly processed
         elif numberOfBits == 2:
             dataType = 'uint8'  # not directly processed
         else:
-            print(('Unsupported number of bits for unsigned int ' + str(signalDataType)))
+            print('Unsupported number of bits for unsigned int ' + str(signalDataType) + ' nBits ', numberOfBits)
 
     elif signalDataType == 1:  # signed int
         if numberOfBits <= 8:
             dataType = 'int8'
-        elif numberOfBits == 16:
+        elif numberOfBits <= 16:
             dataType = 'int16'
-        elif numberOfBits == 32:
+        elif numberOfBits <= 32:
             dataType = 'int32'
+        elif numberOfBits <= 64:
+            dataType = 'int64'
         else:
-            print(('Unsupported number of bits for signed int ' + str(signalDataType)))
+            print('Unsupported number of bits for signed int ' + str(signalDataType) + ' nBits ', numberOfBits)
 
     elif signalDataType in (2, 3):  # floating point
         if numberOfBits == 32:
             dataType = 'float32'
         elif numberOfBits == 64:
             dataType = 'float64'
         else:
-            print(('Unsupported number of bit for floating point ' + str(signalDataType)))
+            print('Unsupported number of bit for floating point ' + str(signalDataType) + ' nBits ', numberOfBits)
 
     elif signalDataType == 7:  # string
         dataType = 'str'  # not directly processed
     elif signalDataType == 8:  # array of bytes
         dataType = 'buffer'  # not directly processed
     else:
-        print(('Unsupported Signal Data Type ' + str(signalDataType) + ' ', numberOfBits))
+        print('Unsupported Signal Data Type ' + str(signalDataType) + ' nBits ', numberOfBits)
     return dataType
diff --git a/mdfinfo3.py b/mdfinfo3.py
@@ -25,6 +25,7 @@
 from sys import version_info
 PythonVersion = version_info
 PythonVersion = PythonVersion[0]
+from numpy import sort, zeros
 from struct import calcsize, unpack
 
 
@@ -305,6 +306,30 @@ def readinfo3(self, fid):
 
         # CLose the file
         fid.close()
+
+        # reorder channel blocks and related blocks based on first bit position
+        # this reorder is meant to improve performance while parsing records using core.records.fromfile
+        # as it will not use cn_byte_offset
+        # first, calculate new mapping/order
+        for dg in range(self['HDBlock']['numberOfDataGroups']):
+            for cg in range(self['DGBlock'][dataGroup]['numberOfChannelGroups']):
+                nChannel = len(self['CNBlock'][dg][cg])
+                Map = zeros(shape=len(self['CNBlock'][dg][cg]), dtype=[('index', 'u4'), ('first_bit', 'u4')])
+                for cn in range(nChannel):
+                    Map[cn] = (cn, self['CNBlock'][dg][cg][cn]['numberOfTheFirstBits'])
+                orderedMap = sort(Map, order='first_bit')
+
+                toChangeIndex = Map == orderedMap
+                for cn in range(nChannel):
+                    if not toChangeIndex[cn]:
+                        # offset all indexes of indexes to be moved
+                        self['CNBlock'][dg][cg][cn + nChannel] = self['CNBlock'][dg][cg].pop(cn)
+                        self['CCBlock'][dg][cg][cn + nChannel] = self['CCBlock'][dg][cg].pop(cn)
+                for cn in range(nChannel):
+                    if not toChangeIndex[cn]:
+                        # change to ordered index
+                        self['CNBlock'][dg][cg][cn] = self['CNBlock'][dg][cg].pop(orderedMap[cn][0] + nChannel)
+                        self['CCBlock'][dg][cg][cn] = self['CCBlock'][dg][cg].pop(orderedMap[cn][0] + nChannel)
 
     def listChannels3(self, fileName=None):
         """ reads data, channel group and channel blocks to list channel names

diff --git a/setup.py b/setup.py
@@ -14,7 +14,7 @@
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/development.html#single-sourcing-the-version
-    version='0.0.6',
+    version='0.1.1',
 
     description='A Measured Data Format file parser',
     long_description=long_description,