From 58306232ca1359a714cb68759ea5e75708c8849e Mon Sep 17 00:00:00 2001 From: Aymeric Rateau Date: Fri, 7 Aug 2015 23:46:23 +0100 Subject: [PATCH] Introduce dataread cython module to read and unpack channel data --- mdfreader.e4p | 3 +- mdfreader/dataRead.pyx | 399 ++++++++++++++++++++++++++-------------- mdfreader/mdf4reader.py | 117 ++++++------ setup.py | 6 +- 4 files changed, 329 insertions(+), 196 deletions(-) diff --git a/mdfreader.e4p b/mdfreader.e4p index 460fcb3..266bd3d 100644 --- a/mdfreader.e4p +++ b/mdfreader.e4p @@ -1,7 +1,7 @@ - + en @@ -30,6 +30,7 @@ setup.py mdfreader/tests/tests.py mdfreader/__main__.py + mdfreader/dataRead.pyx diff --git a/mdfreader/dataRead.pyx b/mdfreader/dataRead.pyx index e3083ca..440e4ec 100644 --- a/mdfreader/dataRead.pyx +++ b/mdfreader/dataRead.pyx @@ -1,160 +1,277 @@ -from __future__ import division import numpy as np cimport numpy as np cimport cython -from libc.stdlib cimport malloc, free +from struct import unpack, Struct +#cdef extern from "Python.h": +# char* PyByteArray_AsString(object bytearray) except NULL +#cdef extern from "stdint.h": +# ctypedef char int8_t +# ctypedef unsigned char uint8_t +# ctypedef short int16_t +# ctypedef unsigned short uint16_t +# ctypedef long int32_t +# ctypedef unsigned long uint32_t +# ctypedef long long int64_t +# ctypedef unsigned long long uint64_t #@cython.boundscheck(False) #@cython.wraparound(False) -def dataRead(char* bita, np.dtype RecordFormat, \ - char Format, unsigned long long numberOfRecords, \ +def dataRead(unsigned char[:] bita, unsigned short bitCount, \ + unsigned short signalDataType, RecordFormat, unsigned long long numberOfRecords, \ unsigned long record_byte_size, \ unsigned long bitOffset, unsigned long posByteBeg, unsigned long posByteEnd, \ unsigned long posBitEnd): - """ reads stream of record bytes needed for not byte aligned data - - Parameters - ------------ - bita : stream - stream of bytes - channelList : List of str, optional - list of channel to read - - Returns - -------- - rec : numpy ndarray - contains a matrix of raw data in a ndarray (attributes corresponding to channel name) - """ # initialise variables - cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array cdef unsigned long trailBits = posByteEnd * 8 - posBitEnd - cdef unsigned int nbytes = posByteEnd - posByteBeg # size of each values - cdef char *mask=malloc(nbytes) - cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 - cdef unsigned long long i - cdef signed char temp1byte - cdef unsigned char utemp1byte - cdef short temp2bytes - cdef unsigned short utemp2bytes - cdef long temp4bytes - cdef unsigned long utemp4bytes - cdef long long temp8bytes - cdef unsigned long long utemp8bytes + #cdef char* temp = PyByteArray_AsString(bita) # slice stream in array - if Format == 'f': # float - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - temp4bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - temp4bytes = temp4bytes << bitOffset - buf[i] = temp4bytes - elif Format == 'd': # double - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - temp8bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - temp8bytes = temp8bytes << bitOffset - buf[i] = temp8bytes - elif Format == 'H': # unsigned short - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - utemp2bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - utemp2bytes = utemp2bytes << bitOffset - buf[i] = utemp2bytes - elif Format == 'h': # signed short - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - temp2bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - temp2bytes = temp2bytes << bitOffset - buf[i] = temp2bytes - elif Format == 'B': # unsigned char - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - temp1byte = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - temp1byte = temp1byte << bitOffset - buf[i] = temp1byte - elif Format == 'b': # signed char - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - utemp1byte = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - utemp1byte = utemp1byte << bitOffset - buf[i] = utemp1byte - elif Format == 'I': # unsigned int + if signalDataType in (4, 5) and bitCount == 32: # float + return dataReadFloat(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (4, 5) and bitCount == 64: # double + return dataReadDouble(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (0, 1) and bitCount <= 8: # unsigned char + return dataReadUChar(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (2, 3) and bitCount <= 8: # signed char + return dataReadChar(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (0, 1) and bitCount <=16: # unsigned short + return dataReadUShort(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (2, 3) and bitCount <= 16: # signed short + return dataReadShort(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (0, 1) and bitCount <=32: # unsigned int + return dataReadUInt(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (2, 3) and bitCount <= 32: # signed int + return dataReadInt(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (0, 1) and bitCount <=64: # unsigned long long + return dataReadULongLong(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + elif signalDataType in (2, 3) and bitCount <= 64: # signed long long + return dataReadLongLong(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + else: + return dataReadByte(bita, RecordFormat, numberOfRecords, \ + record_byte_size, posByteBeg, posByteEnd, posBitEnd, trailBits, bitOffset) + +cdef dataReadFloat(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef long mask = ((1 << maskBit) - 1) + cdef long temp4bytes = 0 + if trailBits == 0 and bitOffset==0: + CFormat = Struct('f') for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - utemp4bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - utemp4bytes = utemp4bytes << bitOffset - buf[i] = utemp4bytes - elif Format == 'i': # signed int + buf[i] = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + else: + CFormat = Struct('i') for i in range(numberOfRecords): - # mask right part + temp4bytes = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] if trailBits > 0: - mask = ((1 << maskBit) - 1) - temp4bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask + temp4bytes = temp4bytes & mask # left shift if bitOffset > 0: temp4bytes = temp4bytes << bitOffset - buf[i] = temp4bytes - elif Format == 'Q': # unsigned long long - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - utemp8bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - utemp8bytes = utemp8bytes << bitOffset - buf[i] = utemp8bytes - elif Format == 'q': # signed long long - for i in range(numberOfRecords): - # mask right part - if trailBits > 0: - mask = ((1 << maskBit) - 1) - temp8bytes = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] & mask - # left shift - if bitOffset > 0: - temp8bytes = temp8bytes << bitOffset - buf[i] = temp8bytes - else: - for i in range(numberOfRecords): - buf[i] = bita[posByteBeg + record_byte_size * i:\ - posByteEnd + record_byte_size * i] - free(mask) + buf[i] = temp4bytes + return buf + +cdef dataReadDouble(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef long long mask = ((1 << maskBit) - 1) + CFormat = Struct('d') + for i in range(numberOfRecords): + buf[i] = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + return buf + +cdef dataReadUChar(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef unsigned char mask = ((1 << maskBit) - 1) + cdef unsigned char temp1byte = 0 + CFormat = Struct('B') + for i in range(numberOfRecords): + temp1byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp1byte = temp1byte & mask + # left shift + if bitOffset > 0: + temp1byte = temp1byte << bitOffset + buf[i] = temp1byte + return buf + +cdef dataReadChar(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef char mask = ((1 << maskBit) - 1) + cdef char temp1byte = 0 + CFormat = Struct('b') + for i in range(numberOfRecords): + temp1byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp1byte = temp1byte & mask + # left shift + if bitOffset > 0: + temp1byte = temp1byte << bitOffset + buf[i] = temp1byte + return buf + +cdef dataReadUShort(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef unsigned short mask = ((1 << maskBit) - 1) + cdef unsigned short temp2byte = 0 + CFormat = Struct('H') + for i in range(numberOfRecords): + temp2byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp2byte = temp2byte & mask + # left shift + if bitOffset > 0: + temp2byte = temp2byte << bitOffset + buf[i] = temp2byte + return buf + +cdef dataReadShort(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef short mask = ((1 << maskBit) - 1) + cdef short temp2byte = 0 + CFormat = Struct('h') + for i in range(numberOfRecords): + temp2byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp2byte = temp2byte & mask + # left shift + if bitOffset > 0: + temp2byte = temp2byte << bitOffset + buf[i] = temp2byte + return buf + +cdef dataReadUInt(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef unsigned int mask = ((1 << maskBit) - 1) + cdef unsigned int temp4byte = 0 + CFormat = Struct('I') + for i in range(numberOfRecords): + temp4byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp4byte = temp4byte & mask + # left shift + if bitOffset > 0: + temp4byte = temp4byte << bitOffset + buf[i] = temp4byte + return buf + +cdef dataReadInt(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef int mask = ((1 << maskBit) - 1) + cdef int temp4byte = 0 + CFormat = Struct('i') + for i in range(numberOfRecords): + temp4byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp4byte = temp4byte & mask + # left shift + if bitOffset > 0: + temp4byte = temp4byte << bitOffset + buf[i] = temp4byte + return buf + +cdef dataReadULongLong(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef unsigned long long mask = ((1 << maskBit) - 1) + cdef unsigned long long temp8byte = 0 + CFormat = Struct('Q') + for i in range(numberOfRecords): + temp8byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp8byte = temp8byte & mask + # left shift + if bitOffset > 0: + temp8byte = temp8byte << bitOffset + buf[i] = temp8byte + return buf + +cdef dataReadLongLong(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + cdef unsigned long maskBit = posBitEnd - posByteBeg * 8 + cdef long long mask = ((1 << maskBit) - 1) + cdef long long temp4byte = 0 + CFormat = Struct('q') + for i in range(numberOfRecords): + temp8byte = CFormat.unpack(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i])[0] + # mask right part + if trailBits > 0: + temp8byte = temp8byte & mask + # left shift + if bitOffset > 0: + temp8byte = temp8byte << bitOffset + buf[i] = temp8byte + return buf + +cdef dataReadByte(unsigned char[:] bita, RecordFormat, unsigned long long numberOfRecords, \ + unsigned long record_byte_size, unsigned long posByteBeg, unsigned long posByteEnd, \ + unsigned long posBitEnd, unsigned long trailBits, unsigned long bitOffset): + cdef np.ndarray buf = np.zeros(numberOfRecords, dtype=RecordFormat) # return numpy array + cdef unsigned long long i + for i in range(numberOfRecords): + buf[i] = bytes(bita[posByteBeg + record_byte_size * i:\ + posByteEnd + record_byte_size * i]) return buf - diff --git a/mdfreader/mdf4reader.py b/mdfreader/mdf4reader.py index 3c576fc..948387f 100644 --- a/mdfreader/mdf4reader.py +++ b/mdfreader/mdf4reader.py @@ -32,7 +32,7 @@ from struct import Struct from struct import unpack as structunpack from math import pow -from sys import platform, version_info +from sys import platform, version_info, exc_info from io import open # for python 3 and 2 consistency from .mdfinfo4 import info4, MDFBlock, ATBlock from time import gmtime, strftime @@ -981,13 +981,13 @@ def readRecordBuf(self, buf, channelList=None): temp[channel.name] = channel.CFormat.unpack(buf[channel.posByteBeg:channel.posByteEnd])[0] return temp # returns dictionary of channel with its corresponding values - + #@profile def readBitarray(self, bita, channelList=None): """ reads stream of record bytes using bitarray module needed for not byte aligned data Parameters ------------ - bitarray : stream + bita : stream stream of bytes channelList : List of str, optional list of channel to read @@ -997,9 +997,6 @@ def readBitarray(self, bita, channelList=None): rec : numpy recarray contains a matrix of raw data in a recarray (attributes corresponding to channel name) """ - from bitarray import bitarray - B = bitarray(endian="little") # little endian by default - B.frombytes(bytes(bita)) # initialise data structure if channelList is None: channelList = self.channelNames @@ -1008,52 +1005,68 @@ def readBitarray(self, bita, channelList=None): if channel.name in channelList: format.append(channel.RecordFormat) buf = recarray(self.numberOfRecords, format) - def signedInt(temp, extension): - """ extend bits of signed data managing two's complement - """ - extension.setall(False) - extensionInv = bitarray(extension, endian='little') - extensionInv.setall(True) - for i in range(self.numberOfRecords): # extend data of bytes to match numpy requirement - signBit = temp[i][-1] - if not signBit: # positive value, extend with 0 - temp[i].extend(extension) - else: # negative value, extend with 1 - signBit = temp[i].pop(-1) - temp[i].extend(extensionInv) - temp[i].append(signBit) - return temp - # read data - record_bit_size = self.CGrecordLength * 8 - for chan in range(len(self)): - if self[chan].name in channelList: - temp = [B[self[chan].posBitBeg + record_bit_size * i:\ - self[chan].posBitEnd + record_bit_size * i]\ - for i in range(self.numberOfRecords)] - nbytes = len(temp[0].tobytes()) - if not nbytes == self[chan].nBytes and \ - self[chan].signalDataType not in (6, 7, 8, 9, 10, 11, 12): # not Ctype byte length - byte = bitarray(8 * (self[chan].nBytes - nbytes), endian='little') - byte.setall(False) - if self[chan].signalDataType not in (2, 3): # not signed integer - for i in range(self.numberOfRecords): # extend data of bytes to match numpy requirement - temp[i].extend(byte) - else: # signed integer (two's complement), keep sign bit and extend with bytes - temp = signedInt(temp, byte) - nTrailBits = self[chan].nBytes*8 - self[chan].bitCount - if self[chan].signalDataType in (2, 3) and \ - nbytes == self[chan].nBytes and \ - nTrailBits > 0: # Ctype byte length but signed integer - trailBits = bitarray(nTrailBits, endian='little') - temp = signedInt(temp, trailBits) - if 's' not in self[chan].Format: - temp = [self[chan].CFormat.unpack(temp[i].tobytes())[0] \ - for i in range(self.numberOfRecords)] - else: - temp = [temp[i].tobytes() \ - for i in range(self.numberOfRecords)] - buf[self[chan].name] = asarray(temp) - return buf + try: # use rather cython compiled code for performance + from .dataRead import dataRead + for chan in range(len(self)): + if self[chan].name in channelList: + buf[self[chan].name] = dataRead(bita, self[chan].bitCount, \ + self[chan].signalDataType, self[chan].RecordFormat[1], \ + self.numberOfRecords, self.CGrecordLength, \ + self[chan].bitOffset, self[chan].posByteBeg, self[chan].posByteEnd, \ + self[chan].posBitEnd) + return buf + except: + print("Unexpected error:", exc_info()[0]) + print('dataRead crashed, back to python data reading') + from bitarray import bitarray + B = bitarray(endian="little") # little endian by default + B.frombytes(bytes(bita)) + def signedInt(temp, extension): + """ extend bits of signed data managing two's complement + """ + extension.setall(False) + extensionInv = bitarray(extension, endian='little') + extensionInv.setall(True) + for i in range(self.numberOfRecords): # extend data of bytes to match numpy requirement + signBit = temp[i][-1] + if not signBit: # positive value, extend with 0 + temp[i].extend(extension) + else: # negative value, extend with 1 + signBit = temp[i].pop(-1) + temp[i].extend(extensionInv) + temp[i].append(signBit) + return temp + # read data + record_bit_size = self.CGrecordLength * 8 + for chan in range(len(self)): + if self[chan].name in channelList: + temp = [B[self[chan].posBitBeg + record_bit_size * i:\ + self[chan].posBitEnd + record_bit_size * i]\ + for i in range(self.numberOfRecords)] + nbytes = len(temp[0].tobytes()) + if not nbytes == self[chan].nBytes and \ + self[chan].signalDataType not in (6, 7, 8, 9, 10, 11, 12): # not Ctype byte length + byte = bitarray(8 * (self[chan].nBytes - nbytes), endian='little') + byte.setall(False) + if self[chan].signalDataType not in (2, 3): # not signed integer + for i in range(self.numberOfRecords): # extend data of bytes to match numpy requirement + temp[i].extend(byte) + else: # signed integer (two's complement), keep sign bit and extend with bytes + temp = signedInt(temp, byte) + nTrailBits = self[chan].nBytes*8 - self[chan].bitCount + if self[chan].signalDataType in (2, 3) and \ + nbytes == self[chan].nBytes and \ + nTrailBits > 0: # Ctype byte length but signed integer + trailBits = bitarray(nTrailBits, endian='little') + temp = signedInt(temp, trailBits) + if 's' not in self[chan].Format: + temp = [self[chan].CFormat.unpack(temp[i].tobytes())[0] \ + for i in range(self.numberOfRecords)] + else: + temp = [temp[i].tobytes() \ + for i in range(self.numberOfRecords)] + buf[self[chan].name] = asarray(temp) + return buf class mdf4(dict): diff --git a/setup.py b/setup.py index 5674337..8a90e4a 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ from setuptools import setup, find_packages from codecs import open # To use a consistent encoding from os import path +from Cython.Build import cythonize here = path.abspath(path.dirname(__file__)) @@ -66,7 +67,7 @@ # project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files - install_requires=['numpy>=1.6', 'sympy', 'bitarray'], + install_requires=['numpy>=1.6', 'sympy', 'bitarray', 'cython'], # List additional groups of dependencies here (e.g. development dependencies). # You can install these using the following syntax, for example: @@ -96,5 +97,6 @@ 'console_scripts': [ 'mdfconverter=mdfconverter.mdfconverter:main', ], - } + }, + ext_modules=cythonize('mdfreader/dataRead.pyx') )