From a44547a9632c000fb8b37fa8bb2d42c0bbd0e0f5 Mon Sep 17 00:00:00 2001 From: "Eugeniy E. Mikhailov" Date: Wed, 20 Jul 2022 10:53:57 -0400 Subject: Factor of 3 to 4 spead up for reading of the trace with pandas --- qolab/data/trace.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'qolab/data') diff --git a/qolab/data/trace.py b/qolab/data/trace.py index 700105a..c941fd8 100644 --- a/qolab/data/trace.py +++ b/qolab/data/trace.py @@ -2,6 +2,7 @@ from qolab.file_utils import save_table_with_header import datetime import numpy as np import yaml +import pandas def headerFromDictionary(d, prefix=''): header = [] @@ -29,10 +30,11 @@ def loadTraceRawHeaderAndData(fname): break header=yaml.load(str.join('\n', headerstr), Loader=yaml.BaseLoader) # now we load the data itself - data=np.genfromtxt(fname, comments='%', delimiter='\t') - # Note: panda reads csv faster by factor of 8, but it needs column names provided - # otherwise it eats the first data line and makes meaningless column names from it - # data=panda.read_csv('/home/evmik/hopping_trace_20220706_02141.dat', comment='%', delimiter='\t') + # data=np.genfromtxt(fname, comments='%', delimiter='\t') + # Note: pandas reads csv faster by factor of 8 then numpy.genfromtxt + # data=pandas.read_csv('/home/evmik/hopping_trace_20220706_02141.dat', comment='%', delimiter='\t', header=None) + df = pandas.read_csv(fname, comment='%', delimiter='\t', header=None) + data = df.to_numpy() return(header, data) def loadTrace(fname): -- cgit v1.2.3