Commit f39d44f2 authored by Semih Demir's avatar Semih Demir
Browse files

acquisition merge

parents 478ecade bcb1e42c
......@@ -48,9 +48,11 @@ General:
Acquisition:
hardware_settings:
sampling_frequency: 200000 # in Hz
gain_selection : [10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000]
asdf_settings:
data_folder : 'e:/20181218_sw_test/'
data_folder_tmp : 'e:/20181218_sw_test/tmp/'
use_float32 : False # select between int32 and float32 as asdf data format
data_folder : 'e:/20190206_sw_test/'
data_folder_tmp : 'e:/20190206_sw_test/tmp/'
compression : 'gzip-3' # default: 'gzip-3', 'gzip-0' to 'gzip-9' or None
file_length_sec : 60.0 # in seconds
station_naming : [1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31, 24, 32]
......
......@@ -29,10 +29,12 @@ class DataToASDF:
'location': '00',
'channel': '001',
'starttime': UTCDateTime().timestamp,
'sampling_rate': param['Acquisition']['hardware_settings']['sampling_frequency']
'sampling_rate': param['Acquisition']['hardware_settings']['sampling_frequency'],
'gain': '0'
}
self._sampling_rate = param['Acquisition']['hardware_settings']['sampling_frequency']
self._gain_selection = param['Acquisition']['hardware_settings']['gain_selection']
self._nr_of_datapoints = floor(self.l_notify_size.value / 16 / 2) # nr of channels & 16 bit = 2 bytes
self._file_handle = None
self._time_age_of_file = 0 # keeps track internally how old the file is
......@@ -58,7 +60,12 @@ class DataToASDF:
logging.info("_create_new_file with folder_file_name = {0}".format(folder_file_name))
self._time_age_of_file = time.time()
self._file_handle = pyasdf.ASDFDataSet(folder_file_name, compression=self.compression)
# logging.info("self.compression = {}, type = {}".format(self.compression, type(self.compression)))
if self.compression == 'None':
# logging.info("if self.compression = None: -> true")
self._file_handle = pyasdf.ASDFDataSet(folder_file_name, compression=None)
else:
self._file_handle = pyasdf.ASDFDataSet(folder_file_name, compression=self.compression)
if self._last_used_file_name is not None:
os.rename(self.folder_tmp + self._last_used_file_name, self.folder + self._last_used_file_name)
......@@ -88,6 +95,7 @@ class DataToASDF:
for i in range(16):
self.stats['station'] = str(self.station_naming[i + 16*card_nr]).zfill(3)
self.stats['gain'] = self._gain_selection[i + 16*card_nr]
stream += Trace(np_data[i], header=self.stats) # without transpose np_data[:, i]
# logging.info("{}, {}\n".format(self.stats['station'], self.stats['starttime']))
......
......@@ -26,10 +26,12 @@ def generate_data_for_pv_buffer(size, amount):
if time.time() > ts:
logging.info("all channels random. i: {}, {}%".format(i, int(i / size * 100)))
ts = time.time() + 2
# return pv_buffer
fill_percent = 100 # 100 fills up all buffer
if amount > 0:
# for j in range(0, 32, 2):
# channel_byte_offset = j
channel_byte_offset = 0 # 0 is channel 0
for i in range(0 + channel_byte_offset, int(len(pv_buffer) * fill_percent / 100) + channel_byte_offset, 32):
value = int(sin(i / 1100000) * 20000)
......
......@@ -30,6 +30,7 @@ class Card:
# l_notify_size = c_int32(regs.KILO_B(2 * 1024))
self.l_notify_size = c_int32(param['Acquisition']['bytes_per_transfer'])
self._use_16_bit_mode = param['Acquisition']['hardware_settings']['use_16_bit_mode']
self.use_float32 = param['Acquisition']['asdf_settings']['use_float32']
self.card_nr = card_nr
self.h_card = None
......@@ -39,6 +40,13 @@ class Card:
# nr of channels & 16 bit = 2 bytes
self._nr_of_datapoints = floor(param['Acquisition']['bytes_per_transfer'] / 16 / 2)
gain_selection = param['Acquisition']['hardware_settings']['gain_selection']
if card_nr == 0:
self.scaling_this_card = [i * 2 / 65536 for i in gain_selection[0:16]] # e.g. 16 bit to +- 10'000 mV
else:
self.scaling_this_card = [i * 2 / 65536 for i in gain_selection[16:32]]
# logging.info("scaling_this_card: {}".format(', '.join(str(e) for e in self.scaling_this_card)))
def init_card(self, param):
"""Initialise card. Setup card parameters. Reserve buffers for DMA data transfer."""
logging.info("init card: {}".format(self.card_nr))
......@@ -92,9 +100,12 @@ class Card:
pass
else:
# some possibilities: int16(no conversion needed), int32, float32, float64
np_data = np.require(np_data, dtype=np.int32, requirements=["C"]) # int32 convert to 32 bit
# for i in range(0, 16):
# np_data[i] = np_data[i] / 10000
if self.use_float32:
np_data = np.require(np_data, dtype=np.float32, requirements=["C"]) # int32 convert to 32 bit float
for i in range(0, 16):
np_data[i] = np_data[i] * self.scaling_this_card[i]
else:
np_data = np.require(np_data, dtype=np.int32, requirements=["C"])
return np_data
def data_has_been_read(self):
......
......@@ -44,11 +44,15 @@ def init_card(param, card_nr):
l_notify_size = c_int32(param['Acquisition']['bytes_per_transfer'])
timeout = param['Acquisition']['hardware_settings']['timeout']
gain_selection = param['Acquisition']['hardware_settings']['gain_selection']
""" open card """
if card_nr == 0:
h_card = spcm_hOpen(create_string_buffer(b'/dev/spcm0'))
gain_selection_this_card = gain_selection[0:16]
else:
h_card = spcm_hOpen(create_string_buffer(b'/dev/spcm1'))
gain_selection_this_card = gain_selection[16:32]
if h_card is None:
logging.error("card {} not found...".format(card_nr))
return -1
......@@ -98,7 +102,13 @@ def init_card(param, card_nr):
spcm_dwSetParam_i32(h_card, regs.SPC_CLOCKOUT, 0)
# set input range 50, 100, 250, 500, 1000, 2000, 5000, 10000 mV
# spcm_dwSetParam_i32(h_card, regs.SPC_AMP0, 10000)
range_min = c_int32(0)
range_max = c_int32(0)
for i in range(16):
spcm_dwSetParam_i32(h_card, regs.SPC_AMP0 + i*100, gain_selection_this_card[i])
spcm_dwGetParam_i32(h_card, regs.SPC_READRANGEMIN0 + i, byref(range_min))
spcm_dwGetParam_i32(h_card, regs.SPC_READRANGEMAX0 + i, byref(range_max))
logging.info("card {}, channel {}: {}mV to {}mV".format(card_nr, i, range_min.value, range_max.value))
""" define the data buffer """
# we try to use continuous memory if available and big enough
......
on my dell laptop
expected:
=========
- 2e5*32*16 bit per second
- 2e5 sps * 32 channels * 16 bit / 8 bit / 1024 bit / 1024 bit * 60 sec = 732.4219 MByte per minute (16bit)
np.float32
==========
- file size first: 1 175 592 KB
......@@ -41,6 +48,97 @@ np.float64, division commented out: # np_data[i] = np_data[i] / 10000
- file size first: 864 631 KB
- load ~ 103% (5.295)
dtype=np.int32
==============
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=None
- param['Acquisition']['simulation_amount'] = 1
- 15 channels are zeroes, 1 channel with sine
- file size first: 1 574 944 KB
- load ~ 7.87% (0.4033)
dtype=np.int32
==============
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=None
- param['Acquisition']['simulation_amount'] = 4
- file size first: 1 574 944 KB
- load ~ 7.71% (0.395)
dtype=np.int32
==============
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=gzip-0
- param['Acquisition']['simulation_amount'] = 1
- 15 channels are zeroes, 1 channel with sine
- file size first: 1 575 473 KB
- load ~ 10.1% (0.5175)
dtype=np.int16
==============
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=gzip-0
- param['Acquisition']['simulation_amount'] = 1
- 15 channels are zeroes, 1 channel with sine
- file size first: 788 512 KB
- load ~ 6.62% (0.3391)
dtype=np.float32
================
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=gzip-0
- param['Acquisition']['simulation_amount'] = 1
- 15 channels are zeroes, 1 channel with sine
- file size first: 1 574 944 KB
- load ~ 8.45% (0.4325)
dtype=np.float64
================
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=gzip-0
- param['Acquisition']['simulation_amount'] = 1
- 15 channels are zeroes, 1 channel with sine
- file size first: 3 149 769 KB
- load ~ 14.6% (0.7475)
dtype=np.float32
================
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=gzip-3
- all channels sine wave
- file size first: 93 639 KB
- load ~ 19% (0.975)
dtype=np.int32
================
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=gzip-3
- all channels sine wave
- file size first: 112 329 KB
- load ~ 17.7% (0.907)
dtype=np.int16
================
- division commented out: # np_data[i] = np_data[i] / 10000
- compression=gzip-3
- all channels sine wave
- file size first: 105 997 KB
- load ~ 14.7% (0.7525)
dtype=np.float32
================
- division included: np_data[i] = np_data[i] / 9861
- compression=gzip-3
- all channels sine wave
- file size first: 328 925 KB
- load ~ 33.6% (1.72)
dtype=np.int16
================
- all channels random
- compression=gzip-3
- file size first: 789 041 KB (length 62.9146 sec)
- load ~ 28% (1.435)
settings
========
......
......@@ -4,25 +4,24 @@ from scipy.io import savemat
# from obspy.core import read
file_list = []
for file in os.listdir("c:/20181218_sw_test"):
for file in os.listdir("c:/20190206_sw_test"):
if file.endswith(".h5"):
file_list.append(os.path.join("/20181218_sw_test", file))
file_list.append(os.path.join("/20190206_sw_test", file))
# print(os.path.join("/20181107_sw_test", file))
file_list.sort(reverse=True)
load_file = file_list[0]
print("loading latest file: {}".format(load_file))
ds = pyasdf.ASDFDataSet(load_file)
print(ds.events)
print(ds.waveforms.list())
several_channels = ds.waveforms["XM.001"].raw_recording
several_channels += ds.waveforms["XM.001"].raw_recording
several_channels += ds.waveforms["XM.001"].raw_recording
several_channels = ds.waveforms["GR.000"].raw_recording
several_channels += ds.waveforms["GR.001"].raw_recording
several_channels += ds.waveforms["GR.002"].raw_recording
several_channels += ds.waveforms["GR.004"].raw_recording
several_channels += ds.waveforms["GR.016"].raw_recording
several_channels += ds.waveforms["GR.017"].raw_recording
several_channels += ds.waveforms["GR.018"].raw_recording
dt = ds.waveforms["GR.000"].raw_recording[0].stats.starttime
dt = ds.waveforms["XM.001"].raw_recording[0].stats.starttime
several_channels.plot(size=(800, 600))
# three_channels.plot(size=(800, 600), starttime=dt, endtime=dt+0.02)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment