Skip to content

Commit

Permalink
mdf_skeleton class introduction : __init__ and basic channel addition…
Browse files Browse the repository at this point in the history 
… and deletion

refactoring and commonization mdf3 and mdf4 classes intialisation.
put all file metadata in mdf_skeleton attribute
  • Loading branch information
@ratal
ratal committed Sep 25, 2015
1 parent ab5d9fb commit 8611323
Show file tree
Hide file tree
Showing 6 changed files with 70 additions and 175 deletions.
3 changes: 2 additions & 1 deletion mdfreader.e4p
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE Project SYSTEM "Project-5.1.dtd">
<!-- eric project file for project mdfreader -->
<!-- Saved: 2015-07-26, 04:47:31 -->
<!-- Saved: 2015-09-24, 20:31:04 -->
<!-- Copyright (C) 2015 Aymeric Rateau, aymeric.rateau@gmail.com -->
<Project version="5.1">
<Language>en</Language>
Expand Down Expand Up @@ -31,6 +31,7 @@
<Source>mdfreader/tests/tests.py</Source>
<Source>mdfreader/__main__.py</Source>
<Source>mdfreader/dataRead.pyx</Source>
<Source>mdfreader/mdf.py</Source>
</Sources>
<Forms/>
<Translations/>
Expand Down
2 changes: 1 addition & 1 deletion mdfreader/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
__author__ = 'Aymeric Rateau (aymeric.rateau@gmail.com)'
__copyright__ = 'Copyright (c) 2015 Aymeric Rateau'
__license__ = 'GPLV3'
__version__ = "0.1.9.3"
__version__ = "0.2.0"

#if it's run as a script or imported within python, this happens
if __name__ == 'mdfreader':
Expand Down
102 changes: 33 additions & 69 deletions mdfreader/mdf3reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from struct import pack, Struct
from io import open # for python 3 and 2 consistency
from .mdfinfo3 import info3
from .mdf import mdf_skeleton
PythonVersion = version_info
PythonVersion = PythonVersion[0]

Expand Down Expand Up @@ -542,7 +543,7 @@ def load(self, nameList=None):
return None


class mdf3(dict):
class mdf3(mdf_skeleton):

""" mdf file version 3.0 to 3.3 class
Expand All @@ -562,13 +563,8 @@ class mdf3(dict):
flag to convert raw data to physical just after read
filterChannelNames : bool
flag to filter long channel names from its module names separated by '.'
author : str
organisation : str
project : str
subject : str
comment : str
time : str
date : str
file_metadata : dict
file metadata with minimum keys : author, organisation, project, subject, comment, time, date
Methods
------------
Expand All @@ -584,28 +580,7 @@ class mdf3(dict):
Writes simple mdf 3.3 file
"""

def __init__(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True, filterChannelNames=False):
self.masterChannelList = {}
self.multiProc = False # flag to control multiprocessing, default deactivate, giving priority to mdfconverter
self.author = ''
self.organisation = ''
self.project = ''
self.subject = ''
self.comment = ''
self.time = ''
self.date = ''
self.MDFVersionNumber = 300
self.filterChannelNames = False
# clears class from previous reading and avoid to mess up
self.clear()
if fileName is None and info is not None:
self.fileName = info.fileName
self.read3(self.fileName, info, multiProc, channelList, convertAfterRead)
elif fileName is not None:
self.fileName = fileName
self.read3(self.fileName, info, multiProc, channelList, convertAfterRead)

def read3(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True):
def read3(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True, filterChannelNames=False):
""" Reads mdf 3.x file data and stores it in dict
Parameters
Expand Down Expand Up @@ -633,9 +608,9 @@ def read3(self, fileName=None, info=None, multiProc=False, channelList=None, con
if platform == 'win32':
self.multiProc = False # no multiprocessing for windows platform

if self.fileName is None:
if self.fileName is None and info is not None:
self.fileName = info.fileName
else:
elif fileName is not None:
self.fileName = fileName

if channelList is None:
Expand All @@ -648,19 +623,19 @@ def read3(self, fileName=None, info=None, multiProc=False, channelList=None, con
info = info3(self.fileName, None, self.filterChannelNames)

# reads metadata
self.author = info['HDBlock']['Author']
self.organisation = info['HDBlock']['Organization']
self.project = info['HDBlock']['ProjectName']
self.subject = info['HDBlock']['Subject']
self.file_metadata['author'] = info['HDBlock']['Author']
self.file_metadata['organisation'] = info['HDBlock']['Organization']
self.file_metadata['project'] = info['HDBlock']['ProjectName']
self.file_metadata['subject'] = info['HDBlock']['Subject']
try:
self.comment = info['HDBlock']['TXBlock']['Text']
except:
pass
self.time = info['HDBlock']['Time']
self.date = info['HDBlock']['Date']
self.file_metadata['time'] = info['HDBlock']['Time']
self.file_metadata['date'] = info['HDBlock']['Date']
# converts date to be compatible with ISO8601
day, month, year = self.date.split(':')
self.date = year + '-' + month + '-' + day
day, month, year = self.file_metadata['date'].split(':')
self.file_metadata['date'] = year + '-' + month + '-' + day

try:
fid = open(self.fileName, 'rb')
Expand Down Expand Up @@ -697,19 +672,14 @@ def read3(self, fileName=None, info=None, multiProc=False, channelList=None, con
channelName = info['CNBlock'][dataGroup][channelGroup][channel]['signalName']
recordName = D[dataGroup][recordID]['record'].recordToChannelMatching[channelName] # in case record is used for several channels
temp = D[dataGroup][recordID]['data'].__getattribute__(str(recordName) + '_title')
master_channel = 'master' + str(dataGroup)
if info['CNBlock'][dataGroup][channelGroup][channel]['channelType'] == 1: # time channel
channelName = 'master' + str(dataGroup)
channelName = master_channel
if (allChannel or channelName in channelList) and len(temp) != 0:
if ('master' + str(dataGroup)) not in list(self.masterChannelList.keys()):
self.masterChannelList['master' + str(dataGroup)] = []
self.masterChannelList['master' + str(dataGroup)].append(channelName)
self[channelName] = {}
self[channelName]['master'] = 'master' + str(dataGroup) # master channel of channel
if 'physicalUnit' in info['CCBlock'][dataGroup][channelGroup][channel]:
self[channelName]['unit'] = info['CCBlock'][dataGroup][channelGroup][channel]['physicalUnit']
unitStr = info['CCBlock'][dataGroup][channelGroup][channel]['physicalUnit']
else:
self[channelName]['unit'] = ''
self[channelName]['description'] = info['CNBlock'][dataGroup][channelGroup][channel]['signalDescription']
unitStr = ''

# Process concatenated bits inside uint8
bitCount = info['CNBlock'][dataGroup][channelGroup][channel]['numberOfBits']
Expand All @@ -719,20 +689,14 @@ def read3(self, fileName=None, info=None, multiProc=False, channelList=None, con
bitOffset = info['CNBlock'][dataGroup][channelGroup][channel]['numberOfTheFirstBits'] % 8
temp = right_shift(temp, bitOffset)
temp = bitwise_and(temp, mask)
self[channelName]['data'] = temp
#self[channelName]['data'] = temp
else: # should not happen
print('bit count and offset not applied to correct data type')
self[channelName]['data'] = temp
else: # data using full bytes
self[channelName]['data'] = temp

convType = info['CCBlock'][dataGroup][channelGroup][channel]['conversionFormulaIdentifier']
if convType in (0, 1, 2, 6, 7, 8, 9, 10, 11, 12): # needs conversion
self[channelName]['conversion'] = {}
self[channelName]['conversion']['type'] = convType
self[channelName]['conversion']['parameters'] = info['CCBlock'][dataGroup][channelGroup][channel]['conversion']
if convType == 0 and (self[channelName]['conversion']['parameters']['P2'] == 1.0 and self[channelName]['conversion']['parameters']['P1'] in (0.0, -0.0)):
self[channelName].pop('conversion')

self.add_channel(channelName, temp, master_channel, \
unit=unitStr, \
description=info['CNBlock'][dataGroup][channelGroup][channel]['signalDescription'], \
conversion=info['CCBlock'][dataGroup][channelGroup][channel])
del D[dataGroup]

if convertAfterRead:
Expand Down Expand Up @@ -903,20 +867,20 @@ def writePointer(f, pointer, value):
fid.write(pack(UINT16, ndataGroup)) # number of data groups
writeChar(fid, strftime("%d:%m:%Y")) # date
writeChar(fid, strftime("%H:%M:%S")) # time
if self.author is not None:
writeChar(fid, self.author, size=31) # Author
if self.file_metadata['author'] is not None:
writeChar(fid, self.file_metadata['author'], size=31) # Author
else:
writeChar(fid, ' ', size=31) # Author
if self.organisation is not None:
writeChar(fid, self.organisation, size=31) # Organization
if self.file_metadata['organisation'] is not None:
writeChar(fid, self.file_metadata['organisation'], size=31) # Organization
else:
writeChar(fid, ' ', size=31)
if self.project is not None:
writeChar(fid, self.project, size=31) # Project
if self.file_metadata['project'] is not None:
writeChar(fid, self.file_metadata['project'], size=31) # Project
else:
writeChar(fid, ' ', size=31)
if self.subject is not None:
writeChar(fid, self.subject, size=31) # Subject
if self.file_metadata['subject'] is not None:
writeChar(fid, self.file_metadata['subject'], size=31) # Subject
else:
writeChar(fid, ' ', size=31)
fid.write(pack(UINT64, int(time() * 1000000000))) # Time Stamp
Expand Down
51 changes: 13 additions & 38 deletions mdfreader/mdf4reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from sys import platform, version_info, exc_info, byteorder
from io import open # for python 3 and 2 consistency
from .mdfinfo4 import info4, MDFBlock, ATBlock
from .mdf import mdf_skeleton
from time import gmtime, strftime
from multiprocessing import Queue, Process
PythonVersion = version_info
Expand Down Expand Up @@ -1077,7 +1078,7 @@ def signedInt(temp, extension):
buf[self[chan].name] = asarray(temp)
return buf

class mdf4(dict):
class mdf4(mdf_skeleton):

""" mdf file reader class from version 4.0 to 4.1
Expand All @@ -1097,13 +1098,8 @@ class mdf4(dict):
flag to convert raw data to physical just after read
filterChannelNames : bool
flag to filter long channel names from its module names separated by '.'
author : str
organisation : str
project : str
subject : str
comment : str
time : str
date : str
file_metadata : dict
file metadata with minimum keys : author, organisation, project, subject, comment, time, date
Methods
------------
Expand All @@ -1117,28 +1113,7 @@ class mdf4(dict):
Converts all channels from raw data to converted data according to CCBlock information
"""

def __init__(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True):
self.masterChannelList = {}
self.multiProc = False # flag to control multiprocessing, default deactivate, giving priority to mdfconverter
self.convert_tables = False # if True converts raw data with expensive loops, not necessary most cases
self.MDFVersionNumber = 400
self.author = ''
self.organisation = ''
self.project = ''
self.subject = ''
self.comment = ''
self.time = ''
self.date = ''
# clears class from previous reading and avoid to mess up
self.clear()
if fileName is None and info is not None:
self.fileName = info.fileName
self.read4(self.fileName, info, multiProc, channelList, convertAfterRead)
elif fileName is not None:
self.fileName = fileName
self.read4(self.fileName, info, multiProc, channelList, convertAfterRead)

def read4(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True):
def read4(self, fileName=None, info=None, multiProc=False, channelList=None, convertAfterRead=True, filterChannelNames=False):
""" Reads mdf 4.x file data and stores it in dict
Parameters
Expand Down Expand Up @@ -1171,9 +1146,9 @@ def read4(self, fileName=None, info=None, multiProc=False, channelList=None, con
print('No multiprocessing module found')
self.multiProc = False

if self.fileName is None:
if self.fileName is None and info is not None:
self.fileName = info.fileName
else:
elif fileName is not None:
self.fileName = fileName

# inttime = time.clock()
Expand All @@ -1187,15 +1162,15 @@ def read4(self, fileName=None, info=None, multiProc=False, channelList=None, con
self.time = strftime('%H:%M:%S', fileDateTime)
if 'Comment' in list(info['HDBlock'].keys()):
if 'author' in list(info['HDBlock']['Comment'].keys()):
self.author = info['HDBlock']['Comment']['author']
self.file_metadata['author'] = info['HDBlock']['Comment']['author']
if 'department' in list(info['HDBlock']['Comment'].keys()):
self.organisation = info['HDBlock']['Comment']['department']
self.file_metadata['organisation'] = info['HDBlock']['Comment']['department']
if 'project' in list(info['HDBlock']['Comment'].keys()):
self.project = info['HDBlock']['Comment']['project']
self.file_metadata['project'] = info['HDBlock']['Comment']['project']
if 'subject' in list(info['HDBlock']['Comment'].keys()):
self.subject = info['HDBlock']['Comment']['subject']
self.file_metadata['subject'] = info['HDBlock']['Comment']['subject']
if 'TX' in list(info['HDBlock']['Comment'].keys()):
self.comment = info['HDBlock']['Comment']['TX']
self.file_metadata['comment'] = info['HDBlock']['Comment']['TX']

try:
fid = open(self.fileName, 'rb')
Expand Down Expand Up @@ -1541,7 +1516,7 @@ def processDataBlocks4(Q, buf, info, dataGroup, channelList, multiProc):
buf : DATA class
contains raw data
info : info class
contains infomation from MDF Blocks
contains information from MDF Blocks
dataGroup : int
data group number according to info class
channelList : list of str, optional
Expand Down
Loading

0 comments on commit 8611323

Please sign in to comment.