diff --git a/NuRadioMC/EvtGen/generate_unforced.py b/NuRadioMC/EvtGen/generate_unforced.py index 2c228a68f..718f3fe0f 100644 --- a/NuRadioMC/EvtGen/generate_unforced.py +++ b/NuRadioMC/EvtGen/generate_unforced.py @@ -328,7 +328,7 @@ def points_in_cylinder(pt1, pt2, r, q): 'flavors': [], 'energies': []} # calculate rotation matrix to transform position on area to 3D - mask_int = np.zeros_like(mask, dtype=np.bool) + mask_int = np.zeros_like(mask, dtype=bool) t0 = time.perf_counter() n_cylinder = 0 for j, i in enumerate(np.arange(n_events, dtype=int)[mask]): @@ -586,7 +586,7 @@ def points_in_cylinder(pt1, pt2, r, q): data_sets['flavors'] = np.ones(np.sum(mask_int)) data_sets["event_ids"] = np.arange(np.sum(mask_int)) + start_event_id data_sets["n_interaction"] = np.ones(np.sum(mask_int), dtype=int) - data_sets["vertex_times"] = np.zeros(np.sum(mask_int), dtype=np.float) + data_sets["vertex_times"] = np.zeros(np.sum(mask_int), dtype=float) data_sets["interaction_type"] = inelasticities.get_ccnc(np.sum(mask_int)) data_sets["inelasticity"] = inelasticities.get_neutrino_inelasticity(np.sum(mask_int)) diff --git a/NuRadioMC/EvtGen/generator.py b/NuRadioMC/EvtGen/generator.py index d3a73ce65..ae68676dc 100644 --- a/NuRadioMC/EvtGen/generator.py +++ b/NuRadioMC/EvtGen/generator.py @@ -891,7 +891,7 @@ def generate_surface_muons(filename, n_events, Emin, Emax, data_sets["event_group_ids"] = np.arange(i_batch * max_n_events_batch, i_batch * max_n_events_batch + n_events_batch, dtype=int) + start_event_id data_sets["n_interaction"] = np.ones(n_events_batch, dtype=int) - data_sets["vertex_times"] = np.zeros(n_events_batch, dtype=np.float) + data_sets["vertex_times"] = np.zeros(n_events_batch, dtype=float) # generate neutrino flavors randomly @@ -928,7 +928,7 @@ def generate_surface_muons(filename, n_events, Emin, Emax, if('fiducial_rmax' in attributes): mask_phi = mask_arrival_azimuth(data_sets, attributes['fiducial_rmax']) # this currently only works for cylindrical volumes else: - mask_phi = np.ones(len(data_sets["event_group_ids"]), dtype=np.bool) + mask_phi = np.ones(len(data_sets["event_group_ids"]), dtype=bool) # TODO: combine with `get_intersection_volume_neutrino` function for iE, event_id in enumerate(data_sets["event_group_ids"]): if not mask_phi[iE]: @@ -1230,7 +1230,7 @@ def generate_eventlist_cylinder(filename, n_events, Emin, Emax, data_sets["event_group_ids"] = np.arange(i_batch * max_n_events_batch, i_batch * max_n_events_batch + n_events_batch) + start_event_id logger.debug("generating number of interactions") data_sets["n_interaction"] = np.ones(n_events_batch, dtype=int) - data_sets["vertex_times"] = np.zeros(n_events_batch, dtype=np.float) + data_sets["vertex_times"] = np.zeros(n_events_batch, dtype=float) # generate neutrino flavors randomly logger.debug("generating flavors") diff --git a/NuRadioMC/SignalGen/ARZ/ARZ.py b/NuRadioMC/SignalGen/ARZ/ARZ.py index 4b8523200..92827124c 100644 --- a/NuRadioMC/SignalGen/ARZ/ARZ.py +++ b/NuRadioMC/SignalGen/ARZ/ARZ.py @@ -63,7 +63,7 @@ def get_vector_potential( profile_ce: array of floats charge-excess values of the charge excess profile shower_type: string (default "HAD") - type of shower, either "HAD" (hadronic), "EM" (electromagnetic) or "TAU" (tau lepton induced) + type of shower, either "HAD" (hadronic) or "EM" (electromagnetic) n_index: float (default 1.78) index of refraction where the shower development takes place distance: float (default 1km) @@ -239,7 +239,7 @@ def get_dist_shower(X, z): v[0] = vperp_x v[1] = vperp_y v[2] = vperp_z -# v = np.array([vperp_x, vperp_y, vperp_z], dtype=np.float64) +# v = np.array([vperp_x, vperp_y, vperp_z], dtype=float) """ Function F_p Eq.(15) PRD paper. """ @@ -468,6 +468,37 @@ def set_interpolation_factor2(self, interp_factor): """ self._interp_factor2 = interp_factor + def get_shower_profile(self, shower_energy, shower_type, iN): + """ + Gets a charge-excess profile + + Parameters + ---------- + shower_energy: float + the energy of the shower + shower_type: string (default "HAD") + type of shower, either "HAD" (hadronic) or "EM" (electromagnetic) + iN: int + specify shower number + + Returns + ------- + depth, excess: two arrays of floats + slant depths and charge profile amplitudes + """ + + energies = np.array([*self._library[shower_type]]) + iE = np.argmin(np.abs(energies - shower_energy)) + + rescaling_factor = shower_energy / energies[iE] + + profiles = self._library[shower_type][energies[iE]] + profile_depth = profiles['depth'] + profile_ce = profiles['charge_excess'][iN] * rescaling_factor + + return profile_depth, profile_ce + + def get_time_trace(self, shower_energy, theta, N, dt, shower_type, n_index, R, shift_for_xmax=False, same_shower=False, iN=None, output_mode='trace', maximum_angle=20 * units.deg): """ @@ -483,12 +514,8 @@ def get_time_trace(self, shower_energy, theta, N, dt, shower_type, n_index, R, s number of samples in the time domain dt: float size of one time bin in units of time - profile_depth: array of floats - shower depth values of the charge excess profile - profile_ce: array of floats - charge-excess values of the charge excess profile shower_type: string (default "HAD") - type of shower, either "HAD" (hadronic), "EM" (electromagnetic) or "TAU" (tau lepton induced) + type of shower, either "HAD" (hadronic) or "EM" (electromagnetic) n_index: float (default 1.78) index of refraction where the shower development takes place R: float (default 1km) @@ -537,6 +564,7 @@ def get_time_trace(self, shower_energy, theta, N, dt, shower_type, n_index, R, s rescaling_factor = shower_energy / energies[iE] logger.info("shower energy of {:.3g}eV requested, closest available energy is {:.3g}eV. The amplitude of the charge-excess profile will be rescaled accordingly by a factor of {:.2f}".format(shower_energy / units.eV, energies[iE] / units.eV, rescaling_factor)) profiles = self._library[shower_type][energies[iE]] + N_profiles = len(profiles['charge_excess']) if(iN is None or np.isnan(iN)): @@ -597,7 +625,7 @@ def get_time_trace(self, shower_energy, theta, N, dt, shower_type, n_index, R, s logger.error("Tau showers are not yet implemented") raise NotImplementedError("Tau showers are not yet implemented") else: - msg = "showers of type {} are not implemented. Use 'HAD', 'EM' or 'TAU'".format(shower_type) + msg = "showers of type {} are not implemented. Use 'HAD', 'EM'".format(shower_type) logger.error(msg) raise NotImplementedError(msg) if self._use_numba: @@ -669,7 +697,7 @@ def get_vector_potential_fast(self, shower_energy, theta, N, dt, profile_depth, profile_ce: array of floats charge-excess values of the charge excess profile shower_type: string (default "HAD") - type of shower, either "HAD" (hadronic), "EM" (electromagnetic) or "TAU" (tau lepton induced) + type of shower, either "HAD" (hadronic) or "EM" (electromagnetic) n_index: float (default 1.78) index of refraction where the shower development takes place distance: float (default 1km) @@ -1136,7 +1164,7 @@ def get_time_trace(self, shower_energy, theta, N, dt, shower_type, n_index, R, dt: float size of one time bin in units of time shower_type: string (default "HAD") - type of shower, either "HAD" (hadronic), "EM" (electromagnetic) or "TAU" (tau lepton induced) + type of shower, either "HAD" (hadronic) or "EM" (electromagnetic) n_index: float (default 1.78) index of refraction where the shower development takes place R: float (default 1km) diff --git a/NuRadioMC/SignalProp/CPPAnalyticRayTracing/setup.py b/NuRadioMC/SignalProp/CPPAnalyticRayTracing/setup.py index 932437653..adc09a364 100644 --- a/NuRadioMC/SignalProp/CPPAnalyticRayTracing/setup.py +++ b/NuRadioMC/SignalProp/CPPAnalyticRayTracing/setup.py @@ -16,7 +16,7 @@ Extension('wrapper', ['wrapper.pyx'], include_dirs=[numpy.get_include(), '../../utilities/', str(os.environ['GSLDIR']) + '/include/'], library_dirs=[str(os.environ['GSLDIR']) + '/lib/'], - extra_compile_args=['-O3',"-mfpmath=sse"], + extra_compile_args=['-O3'], libraries=['gsl', 'gslcblas'], language='c++' ), diff --git a/NuRadioMC/SignalProp/analyticraytracing.py b/NuRadioMC/SignalProp/analyticraytracing.py index 0d2d65bcf..87d898668 100644 --- a/NuRadioMC/SignalProp/analyticraytracing.py +++ b/NuRadioMC/SignalProp/analyticraytracing.py @@ -1743,8 +1743,8 @@ def set_start_and_end_point(self, x1, x2): if(self._X2[2] < self._X1[2]): self._swap = True self.__logger.debug('swap = True') - self._X2 = np.array(x1, dtype =np.float) - self._X1 = np.array(x2, dtype =np.float) + self._X2 = np.array(x1, dtype =float) + self._X1 = np.array(x2, dtype =float) dX = self._X2 - self._X1 self._dPhi = -np.arctan2(dX[1], dX[0]) diff --git a/NuRadioMC/SignalProp/propagation_base_class.py b/NuRadioMC/SignalProp/propagation_base_class.py index 376f5d19d..0ba908394 100644 --- a/NuRadioMC/SignalProp/propagation_base_class.py +++ b/NuRadioMC/SignalProp/propagation_base_class.py @@ -92,8 +92,8 @@ def set_start_and_end_point(self, x1, x2): stop point of the ray """ self.reset_solutions() - self._X1 = np.array(x1, dtype =np.float) - self._X2 = np.array(x2, dtype = np.float) + self._X1 = np.array(x1, dtype =float) + self._X2 = np.array(x2, dtype = float) if (self._n_reflections): if (self._X1[2] < self._medium.reflection or self._X2[2] < self._medium.reflection): self.__logger.error("start or stop point is below the reflective bottom layer at {:.1f}m".format( diff --git a/NuRadioMC/examples/03_station_coincidences/A05visualization.py b/NuRadioMC/examples/03_station_coincidences/A05visualization.py index 99095c5c0..e69c6f2d5 100644 --- a/NuRadioMC/examples/03_station_coincidences/A05visualization.py +++ b/NuRadioMC/examples/03_station_coincidences/A05visualization.py @@ -79,7 +79,7 @@ x2 = det.get_relative_position(101, iC) if(plot): ax.plot([x2[0]], [x2[1]], [x2[2]], 'ko') - if(j != 0 and (~(np.array(triggered_deep[iE], dtype=np.bool) & mask))[j]): + if(j != 0 and (~(np.array(triggered_deep[iE], dtype=bool) & mask))[j]): continue vertex = np.array([fin['xx'][iE], fin['yy'][iE], fin['zz'][iE]]) # print(fin.keys()) diff --git a/NuRadioMC/simulation/scripts/T05visualize_sim_output.py b/NuRadioMC/simulation/scripts/T05visualize_sim_output.py index a1c046cbf..9585974db 100644 --- a/NuRadioMC/simulation/scripts/T05visualize_sim_output.py +++ b/NuRadioMC/simulation/scripts/T05visualize_sim_output.py @@ -42,14 +42,14 @@ plot_folder = os.path.join(dirname, 'plots', filename, args.trigger_name[0]) if(not os.path.exists(plot_folder)): os.makedirs(plot_folder) - triggered = np.zeros(len(fin['multiple_triggers'][:, 0]), dtype=np.bool) + triggered = np.zeros(len(fin['multiple_triggers'][:, 0]), dtype=bool) for trigger in args.trigger_name[1:]: iTrigger = np.squeeze(np.argwhere(fin.attrs['trigger_names'] == trigger)) - triggered = triggered | np.array(fin['multiple_triggers'][:, iTrigger], dtype=np.bool) + triggered = triggered | np.array(fin['multiple_triggers'][:, iTrigger], dtype=bool) else: trigger_name = args.trigger_name[0] iTrigger = np.argwhere(fin.attrs['trigger_names'] == trigger_name) - triggered = np.array(fin['multiple_triggers'][:, iTrigger], dtype=np.bool) + triggered = np.array(fin['multiple_triggers'][:, iTrigger], dtype=bool) print("\tyou selected '{}'".format(trigger_name)) plot_folder = os.path.join(dirname, 'plots', filename, trigger_name) if(not os.path.exists(plot_folder)): @@ -147,14 +147,14 @@ def a(theta): if(len(args.trigger_name) > 1): print("trigger {} selected which is a combination of {}".format(args.trigger_name[0], args.trigger_name[1:])) trigger_name = args.trigger_name[0] - triggered = np.zeros(len(station['multiple_triggers'][:, 0]), dtype=np.bool) + triggered = np.zeros(len(station['multiple_triggers'][:, 0]), dtype=bool) for trigger in args.trigger_name[1:]: iTrigger = np.squeeze(np.argwhere(fin.attrs['trigger_names'] == trigger)) - triggered = triggered | np.array(station['multiple_triggers'][:, iTrigger], dtype=np.bool) + triggered = triggered | np.array(station['multiple_triggers'][:, iTrigger], dtype=bool) else: trigger_name = args.trigger_name[0] iTrigger = np.argwhere(fin.attrs['trigger_names'] == trigger_name) - triggered = np.array(station['multiple_triggers'][:, iTrigger], dtype=np.bool) + triggered = np.array(station['multiple_triggers'][:, iTrigger], dtype=bool) print("\tyou selected '{}'".format(trigger_name)) ########################### diff --git a/NuRadioMC/simulation/simulation.py b/NuRadioMC/simulation/simulation.py index 35b6be0ac..42a1d19fa 100644 --- a/NuRadioMC/simulation/simulation.py +++ b/NuRadioMC/simulation/simulation.py @@ -306,7 +306,7 @@ def __init__(self, inputfilename, self._amplification_per_channel[self._station_id] = {} for channel_id in range(self._det.get_number_of_channels(self._station_id)): ff = np.linspace(0, 0.5 / self._dt, 10000) - filt = np.ones_like(ff, dtype=np.complex) + filt = np.ones_like(ff, dtype=complex) for i, (name, instance, kwargs) in enumerate(self._evt.iter_modules(self._station_id)): if hasattr(instance, "get_filter"): filt *= instance.get_filter(ff, self._station_id, channel_id, self._det, **kwargs) @@ -1150,12 +1150,12 @@ def _calculate_signal_properties(self): def _create_empty_multiple_triggers(self): if 'trigger_names' not in self._mout_attrs: self._mout_attrs['trigger_names'] = np.array([]) - self._mout['multiple_triggers'] = np.zeros((self._n_showers, 1), dtype=np.bool) + self._mout['multiple_triggers'] = np.zeros((self._n_showers, 1), dtype=bool) for station_id in self._station_ids: sg = self._mout_groups[station_id] n_showers = sg['launch_vectors'].shape[0] - sg['multiple_triggers'] = np.zeros((n_showers, 1), dtype=np.bool) - sg['triggered'] = np.zeros(n_showers, dtype=np.bool) + sg['multiple_triggers'] = np.zeros((n_showers, 1), dtype=bool) + sg['triggered'] = np.zeros(n_showers, dtype=bool) def _create_trigger_structures(self): @@ -1170,13 +1170,13 @@ def _create_trigger_structures(self): # simulated triggers is unknown at the beginning. So we check if the key already exists and if not, # we first create this data structure if 'multiple_triggers' not in self._mout: - self._mout['multiple_triggers'] = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=np.bool) + self._mout['multiple_triggers'] = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=bool) self._mout['trigger_times'] = np.nan * np.zeros_like(self._mout['multiple_triggers'], dtype=float) # for station_id in self._station_ids: # sg = self._mout_groups[station_id] -# sg['multiple_triggers'] = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=np.bool) +# sg['multiple_triggers'] = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=bool) elif extend_array: - tmp = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=np.bool) + tmp = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=bool) nx, ny = self._mout['multiple_triggers'].shape tmp[:, 0:ny] = self._mout['multiple_triggers'] self._mout['multiple_triggers'] = tmp @@ -1186,7 +1186,7 @@ def _create_trigger_structures(self): self._mout['trigger_times'] = tmp_t # for station_id in self._station_ids: # sg = self._mout_groups[station_id] -# tmp = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=np.bool) +# tmp = np.zeros((self._n_showers, len(self._mout_attrs['trigger_names'])), dtype=bool) # nx, ny = sg['multiple_triggers'].shape # tmp[:, 0:ny] = sg['multiple_triggers'] # sg['multiple_triggers'] = tmp @@ -1199,10 +1199,10 @@ def _save_triggers_to_hdf5(self, sg, local_shower_index, global_shower_index): # the information fo the current station and event group n_showers = sg['launch_vectors'].shape[0] if 'multiple_triggers' not in sg: - sg['multiple_triggers'] = np.zeros((n_showers, len(self._mout_attrs['trigger_names'])), dtype=np.bool) + sg['multiple_triggers'] = np.zeros((n_showers, len(self._mout_attrs['trigger_names'])), dtype=bool) sg['trigger_times'] = np.nan * np.zeros_like(sg['multiple_triggers'], dtype=float) elif extend_array: - tmp = np.zeros((n_showers, len(self._mout_attrs['trigger_names'])), dtype=np.bool) + tmp = np.zeros((n_showers, len(self._mout_attrs['trigger_names'])), dtype=bool) nx, ny = sg['multiple_triggers'].shape tmp[:, 0:ny] = sg['multiple_triggers'] sg['multiple_triggers'] = tmp @@ -1213,7 +1213,7 @@ def _save_triggers_to_hdf5(self, sg, local_shower_index, global_shower_index): self._output_event_group_ids[self._station_id].append(self._evt.get_run_number()) self._output_sub_event_ids[self._station_id].append(self._evt.get_id()) - multiple_triggers = np.zeros(len(self._mout_attrs['trigger_names']), dtype=np.bool) + multiple_triggers = np.zeros(len(self._mout_attrs['trigger_names']), dtype=bool) trigger_times = np.nan*np.zeros_like(multiple_triggers) for iT, trigger_name in enumerate(self._mout_attrs['trigger_names']): if self._station.has_trigger(trigger_name): @@ -1262,8 +1262,8 @@ def _create_meta_output_datastructures(self): self._mout = {} self._mout_attributes = {} self._mout['weights'] = np.zeros(self._n_showers) - self._mout['triggered'] = np.zeros(self._n_showers, dtype=np.bool) -# self._mout['multiple_triggers'] = np.zeros((self._n_showers, self._number_of_triggers), dtype=np.bool) + self._mout['triggered'] = np.zeros(self._n_showers, dtype=bool) +# self._mout['multiple_triggers'] = np.zeros((self._n_showers, self._number_of_triggers), dtype=bool) self._mout_attributes['trigger_names'] = None self._amplitudes = {} self._amplitudes_envelope = {} @@ -1289,7 +1289,7 @@ def _create_meta_output_datastructures(self): def _create_station_output_structure(self, n_showers, n_antennas): nS = self._raytracer.get_number_of_raytracing_solutions() # number of possible ray-tracing solutions sg = {} - sg['triggered'] = np.zeros(n_showers, dtype=np.bool) + sg['triggered'] = np.zeros(n_showers, dtype=bool) # we need the reference to the shower id to be able to find the correct shower in the upper level hdf5 file sg['shower_id'] = np.zeros(n_showers, dtype=int) * -1 sg['event_id_per_shower'] = np.zeros(n_showers, dtype=int) * -1 @@ -1424,7 +1424,7 @@ def _write_output_file(self, empty=False): # the multiple triggeres 2d array might have different number of entries per event # because the number of different triggers can increase dynamically # therefore we first create an array with the right size and then fill it - tmp = np.zeros((n_events_for_station, n_triggers), dtype=np.bool) + tmp = np.zeros((n_events_for_station, n_triggers), dtype=bool) for iE, values in enumerate(self._output_multiple_triggers_station[station_id]): tmp[iE] = values sg['multiple_triggers_per_event'] = tmp @@ -1452,10 +1452,10 @@ def _write_output_file(self, empty=False): fout["station_{:d}".format(station_id)].attrs['Vrms'] = list(self._Vrms_per_channel[station_id].values()) fout["station_{:d}".format(station_id)].attrs['bandwidth'] = list(self._bandwidth_per_channel[station_id].values()) - fout.attrs.create("Tnoise", self._noise_temp, dtype=np.float) - fout.attrs.create("Vrms", self._Vrms, dtype=np.float) - fout.attrs.create("dt", self._dt, dtype=np.float) - fout.attrs.create("bandwidth", self._bandwidth, dtype=np.float) + fout.attrs.create("Tnoise", self._noise_temp, dtype=float) + fout.attrs.create("Vrms", self._Vrms, dtype=float) + fout.attrs.create("dt", self._dt, dtype=float) + fout.attrs.create("bandwidth", self._bandwidth, dtype=float) fout.attrs['n_samples'] = self._n_samples fout.attrs['config'] = yaml.dump(self._cfg) diff --git a/NuRadioMC/test/SingleEvents/1e18_output_noise_reference.hdf5 b/NuRadioMC/test/SingleEvents/1e18_output_noise_reference.hdf5 index 093452baa..83cbb5536 100644 Binary files a/NuRadioMC/test/SingleEvents/1e18_output_noise_reference.hdf5 and b/NuRadioMC/test/SingleEvents/1e18_output_noise_reference.hdf5 differ diff --git a/NuRadioMC/test/SingleEvents/validate.sh b/NuRadioMC/test/SingleEvents/validate.sh index 5aa12b647..233784c04 100755 --- a/NuRadioMC/test/SingleEvents/validate.sh +++ b/NuRadioMC/test/SingleEvents/validate.sh @@ -1,9 +1,8 @@ #!/bin/bash -python T02RunSimulation.py 1e18_output_reference.hdf5 surface_station_1GHz.json config.yaml 1e18_output.hdf5 1e18_output.nur - -python T03validate.py 1e18_output.hdf5 1e18_output_reference.hdf5 +python3 T02RunSimulation.py 1e18_output_reference.hdf5 surface_station_1GHz.json config.yaml 1e18_output.hdf5 1e18_output.nur +python3 T03validate.py 1e18_output.hdf5 1e18_output_reference.hdf5 # cleanup -rm -v NuRadioMC/test/SingleEvents/1e18_output.hdf5 \ No newline at end of file +rm -v NuRadioMC/test/SingleEvents/1e18_output.hdf5 diff --git a/NuRadioMC/test/Veff/1e18eV/T03check_output_noise.py b/NuRadioMC/test/Veff/1e18eV/T03check_output_noise.py index a78fe4bfc..f1ebfc01f 100755 --- a/NuRadioMC/test/Veff/1e18eV/T03check_output_noise.py +++ b/NuRadioMC/test/Veff/1e18eV/T03check_output_noise.py @@ -11,7 +11,7 @@ # the event generation has a fixed seed and I switched to Alvarez2000 (also no randomness) # thus, the Veff has no statistical scatter -Veff_mean = 8.17491 +Veff_mean = 7.86364 Veff_sigma = 0.0001 path = os.path.dirname(os.path.abspath(__file__)) diff --git a/NuRadioMC/test/examples/test_radio_emitting_pulser_example.sh b/NuRadioMC/test/examples/test_radio_emitting_pulser_example.sh index 582f7b17b..2645e463e 100755 --- a/NuRadioMC/test/examples/test_radio_emitting_pulser_example.sh +++ b/NuRadioMC/test/examples/test_radio_emitting_pulser_example.sh @@ -1,8 +1,16 @@ #!/bin/bash set -e python NuRadioMC/examples/05_pulser_calibration_measurement/radioEmitting_pulser_calibration/A01generate_pulser_events.py -python NuRadioMC/examples/05_pulser_calibration_measurement/radioEmitting_pulser_calibration/runARA02.py emitter_event_list.hdf5 NuRadioMC/examples/05_pulser_calibration_measurement/radioEmitting_pulser_calibration/ARA02Alt.json NuRadioMC/examples/05_pulser_calibration_measurement/radioEmitting_pulser_calibration/config.yaml output.hdf5 output.nur -python NuRadioMC/test/examples/validate_radio_emitter_allmost_equal.py output.hdf5 NuRadioMC/test/examples/radio_emitter_output_reference.hdf5 + +python NuRadioMC/examples/05_pulser_calibration_measurement/radioEmitting_pulser_calibration/runARA02.py \ + emitter_event_list.hdf5 \ + NuRadioMC/examples/05_pulser_calibration_measurement/radioEmitting_pulser_calibration/ARA02Alt.json \ + NuRadioMC/examples/05_pulser_calibration_measurement/radioEmitting_pulser_calibration/config.yaml \ + output.hdf5 output.nur + +python NuRadioMC/test/examples/validate_radio_emitter_allmost_equal.py output.hdf5 \ + NuRadioMC/test/examples/radio_emitter_output_reference.hdf5 + rm emitter_event_list.hdf5 # rm output.hdf5 rm output.nur diff --git a/NuRadioMC/utilities/Veff.py b/NuRadioMC/utilities/Veff.py index 9ee5eaa14..eab3a500c 100644 --- a/NuRadioMC/utilities/Veff.py +++ b/NuRadioMC/utilities/Veff.py @@ -335,35 +335,35 @@ def get_Veff_Aeff_single( return out for iT, trigger_name in enumerate(trigger_names): - triggered = np.array(fin['multiple_triggers'][:, iT], dtype=np.bool) + triggered = np.array(fin['multiple_triggers'][:, iT], dtype=bool) triggered = remove_duplicate_triggers(triggered, fin['event_group_ids']) out[veff_aeff][trigger_name] = get_veff_output(volume_proj_area, np.sum(weights[triggered]), n_events) for trigger_name, values in iteritems(trigger_combinations): indiv_triggers = values['triggers'] - triggered = np.zeros_like(fin['multiple_triggers'][:, 0], dtype=np.bool) + triggered = np.zeros_like(fin['multiple_triggers'][:, 0], dtype=bool) if isinstance(indiv_triggers, str): triggered = triggered | \ - np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_triggers]], dtype=np.bool) + np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_triggers]], dtype=bool) else: for indiv_trigger in indiv_triggers: triggered = triggered | \ - np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_trigger]], dtype=np.bool) + np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_trigger]], dtype=bool) if 'triggerAND' in values: triggered = triggered & \ - np.array(fin['multiple_triggers'][:, trigger_names_dict[values['triggerAND']]], dtype=np.bool) + np.array(fin['multiple_triggers'][:, trigger_names_dict[values['triggerAND']]], dtype=bool) if 'notriggers' in values: indiv_triggers = values['notriggers'] if(isinstance(indiv_triggers, str)): triggered = triggered & \ - ~np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_triggers]], dtype=np.bool) + ~np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_triggers]], dtype=bool) else: for indiv_trigger in indiv_triggers: triggered = triggered & \ - ~np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_trigger]], dtype=np.bool) + ~np.array(fin['multiple_triggers'][:, trigger_names_dict[indiv_trigger]], dtype=bool) if 'min_sigma' in values.keys(): if isinstance(values['min_sigma'], list): @@ -396,7 +396,7 @@ def get_Veff_Aeff_single( As = np.array(fin['max_amp_ray_solution']) max_amps = np.argmax(As[:, values['ray_channel']], axis=-1) sol = np.array(fin['ray_tracing_solution_type']) - mask = np.array([sol[i, values['ray_channel'], max_amps[i]] == values['ray_solution'] for i in range(len(max_amps))], dtype=np.bool) + mask = np.array([sol[i, values['ray_channel'], max_amps[i]] == values['ray_solution'] for i in range(len(max_amps))], dtype=bool) triggered = triggered & mask if 'n_reflections' in values.keys(): @@ -644,7 +644,7 @@ def get_Veff_Aeff_array(data): * array of unique trigger names Examples - --------- + -------- To plot the full sky effective volume for 'all_triggers' do diff --git a/NuRadioMC/utilities/merge_hdf5.py b/NuRadioMC/utilities/merge_hdf5.py index cb31765d9..464f50733 100644 --- a/NuRadioMC/utilities/merge_hdf5.py +++ b/NuRadioMC/utilities/merge_hdf5.py @@ -266,7 +266,7 @@ def merge2(filenames, output_filename): # try: input_files = np.array(sorted(glob.glob(filename + '.part????'))) input_files = np.append(input_files, np.array(sorted(glob.glob(filename + '.part??????')))) - mask = np.array([os.path.getsize(x) > 1000 for x in input_files], dtype=np.bool) + mask = np.array([os.path.getsize(x) > 1000 for x in input_files], dtype=bool) if(np.sum(~mask)): logger.warning("{:d} files were deselected because their filesize was to small".format(np.sum(~mask))) input_args.append({'filenames': input_files[mask], 'output_filename': output_filename}) diff --git a/NuRadioMC/utilities/muon_flux.py b/NuRadioMC/utilities/muon_flux.py new file mode 100644 index 000000000..b47b481c4 --- /dev/null +++ b/NuRadioMC/utilities/muon_flux.py @@ -0,0 +1,180 @@ +from NuRadioReco.utilities import units +import numpy as np +import os +import pickle +from scipy.interpolate import interp1d +from MCEq.core import MCEqRun +import crflux.models as crf +from functools import lru_cache + +class MuonFlux: + def __init__(self): + + self.mc_m = 1e2 + self.mc_rad = 180. / np.pi + self.mc_eV = 1.e-9 + self.mc_ns = 1e-9 + + self.__buffer = {} + self.file_buffer = "data/surface_muon_buffer.pkl" + if(os.path.exists(self.file_buffer)): + fin = open(self.file_buffer, "rb") + self.__buffer = pickle.load(fin) + fin.close() + + + @lru_cache(maxsize=5000) + def get_mu_flux(self, theta, altitude=3200, interaction_model='SIBYLL23C', primary_model=(crf.GlobalSplineFitBeta, None), particle_names=("total_mu+", "total_mu-")): + + """ + The function get_mu_flux returns the muon flux at theta, for a given altitude, CR model, and hadronic interaction model. + + Parameters + ---------- + energy: float + energy in eV + theta_min: float + minimum zenith angle in rad + theta_max: float + maximum zenith angle in rad + n_steps: int + number of steps to use for the numerical integration + + Returns + ------- + e_grid: array of floats + energy grid in eV + flux: array of floats + flux in NuRadioReco units 1/(area * time * energy * steradian) + """ + + altitude *= self.mc_m + mceq = MCEqRun(interaction_model=interaction_model, + primary_model=primary_model, + theta_deg=theta/units.deg) + + h_grid = np.linspace(50 * 1e3 * 1e2, 0, 500) + X_grid = mceq.density_model.h2X(h_grid) + + alt_idx = np.abs(h_grid - altitude).argmin() + + mceq.solve(int_grid=X_grid) + flux = None + for particle_name in particle_names: + if flux is None: + flux = mceq.get_solution(particle_name, grid_idx=alt_idx, integrate=False) + else: + flux += mceq.get_solution(particle_name, grid_idx=alt_idx, integrate=False) + + flux *= self.mc_m ** 2 * self.mc_eV * self.mc_ns # convert to NuRadioReco units + e_grid = mceq.e_grid / self.mc_eV + + return e_grid, flux + + + def get_interp_angle_mu_flux(self, theta_min, theta_max, altitude=3200, n_steps=3, primary_model=(crf.GlobalSplineFitBeta, None), + interaction_model='SIBYLL23C', particle_names=("total_mu+", "total_mu-")): + """ + The function get_int_angle_mu_flux returns the interpolation function of the integrated muon flux from theta_min to theta_max, + The integration is just a simple Riemannian sum that should be enough, provided the band is small. + + Returns zenith angle integrated flux in NuRadioReco units 1/(area * time * energy) + + Parameters + ---------- + energy: float + energy in eV + theta_min: float + minimum zenith angle in rad + theta_max: float + maximum zenith angle in rad + n_steps: int + number of steps to use for the numerical integration + + Returns + ------- + interpolator: function + function that returns the flux for a given energy + """ + angle_edges = np.arccos(np.linspace(np.cos(theta_max), np.cos(theta_min), n_steps + 1)) + angle_centers = 0.5 *(angle_edges[1:] + angle_edges[:-1]) + d_cos_theta = np.abs(np.cos(theta_min) - np.cos(theta_max)) + + #print(f"integrating the flux from {theta_min/units.deg:.1f} deg to {theta_max/units.deg:.1f} deg by adding the flux from {angle_centers/units.deg}") + + flux = None + for angle in angle_centers: + e_grid, flux_tmp = self.get_mu_flux(angle, altitude, primary_model=primary_model, + interaction_model=interaction_model, particle_names=particle_names) + + # solid angle element is sin(theta) dtheta dphi in spherical coordinates. + flux_tmp *= np.sin(angle) * (d_cos_theta * 2 * np.pi ) / n_steps + if(flux is None): + flux = flux_tmp + else: + flux += flux_tmp + + return interp1d(np.log10(e_grid), flux, kind='cubic') + + + def get_int_angle_mu_flux_buffered(self, energy, theta_min, theta_max, altitude=3200, n_steps=3, primary_model=(crf.GlobalSplineFitBeta, None),interaction_model='SIBYLL23C', particle_names=("total_mu+", "total_mu-")): + + """ + The function get_int_angle_mu_flux evalueates the integrated muon flux from theta_min to theta_max and caches the result. + + Parameters + ---------- + energy: float + energy in eV + theta_min: float + minimum zenith angle in rad + theta_max: float + maximum zenith angle in rad + altitude: float + altitude in meters + n_steps: int + number of steps to use for the numerical integration + + Returns + ------- + flux: float + zenith angle integrated flux in NuRadioReco units 1/(area * time * energy) + """ + + params = (np.round(energy), np.round(theta_min, 6), np.round(theta_max, 6), np.round(altitude), + n_steps, primary_model, interaction_model, particle_names) + + if(params not in self.__buffer): + print(f"calculating muon flux for {params}") + finterp = self.get_interp_angle_mu_flux(theta_min, theta_max, altitude, n_steps=n_steps, primary_model=primary_model, + interaction_model=interaction_model, particle_names=particle_names) + + flux = finterp(np.log10(energy)) + self.__buffer[params] = flux + with open(self.file_buffer, "wb") as fout: + pickle.dump(self.__buffer, fout, protocol=4) + + return self.__buffer[params] + + + def get_e_grid(self, theta=50*units.deg, interaction_model='SIBYLL23C', primary_model=(crf.GlobalSplineFitBeta, None)): + """ + Returns the energy grid for a given interaction model and primary model. Usually this is the same for all zenith angles. + + Parameters + ---------- + theta_deg: float + minimum zenith angle in rad + interaction_model: str + hadronic interaction model + primary_model: tuple + cosmic ray model + + Returns + ------- + energies: array of floats + energy grid in eV + """ + mceq = MCEqRun(interaction_model=interaction_model, primary_model=primary_model, theta_deg=theta/units.deg) + e_grid = mceq.e_grid / self.mc_eV + return e_grid \ No newline at end of file diff --git a/NuRadioReco/detector/ARA/ara_detector_db.json b/NuRadioReco/detector/ARA/ara_detector_db.json index ec5fa08bb..b0f194bb9 100644 --- a/NuRadioReco/detector/ARA/ara_detector_db.json +++ b/NuRadioReco/detector/ARA/ara_detector_db.json @@ -1,4 +1,3 @@ -{ { "_default": {}, "channels": { diff --git a/NuRadioReco/detector/RNO_G/RNO_cr_array.json b/NuRadioReco/detector/RNO_G/RNO_cr_array.json new file mode 100644 index 000000000..631ee1a52 --- /dev/null +++ b/NuRadioReco/detector/RNO_G/RNO_cr_array.json @@ -0,0 +1,341 @@ +{ + "channels": { + "0": { + "amp_type": "rno_surface", + "ant_comment": "upward LPDA north from DAQ", + "ant_orientation_phi": 0.0, + "ant_orientation_theta": 0.0, + "ant_position_x": 0.0, + "ant_position_y": -11, + "ant_position_z": -2.0, + "ant_rotation_phi": 90.0, + "ant_rotation_theta": 90.0, + "ant_type": "createLPDA_100MHz_InfFirn", + "cab_time_delay": 19.8, + "commission_time": "{TinyDate}:2017-10-01T00:00:00", + "decommission_time": "{TinyDate}:2020-11-01T00:00:00", + "channel_id": 0, + "station_id": 11 + }, + + "1": { + "amp_type": "rno_surface", + "ant_comment": "upward LPDA south-east from DAQ", + "ant_orientation_phi": 0.0, + "ant_orientation_theta": 0.0, + "ant_position_x": -9.526, + "ant_position_y": 5.5, + "ant_position_z": -2.0, + "ant_rotation_phi": 330.0, + "ant_rotation_theta": 90.0, + "ant_type": "createLPDA_100MHz_InfFirn", + "cab_time_delay": 19.8, + "commission_time": "{TinyDate}:2017-10-01T00:00:00", + "decommission_time": "{TinyDate}:2020-11-01T00:00:00", + "channel_id": 1, + "station_id": 11 + }, + + + "2": { + "amp_type": "rno_surface", + "ant_comment": "upward LPDA south-west from DAQ", + "ant_orientation_phi": 0.0, + "ant_orientation_theta": 0.0, + "ant_position_x": 9.526, + "ant_position_y": 5.5, + "ant_position_z": -2.0, + "ant_rotation_phi": 210.0, + "ant_rotation_theta": 90.0, + "ant_type": "createLPDA_100MHz_InfFirn", + "cab_time_delay": 19.8, + "commission_time": "{TinyDate}:2017-10-01T00:00:00", + "decommission_time": "{TinyDate}:2020-11-01T00:00:00", + "channel_id": 2, + "station_id": 11 + } + }, + "stations": { + "01": { + "pos_easting": -1401.32, + "pos_northing": 1154.08, + "pos_altitude": 3200, + "pos_site": "summit", + "station_id": 11, + "station_type": "Nanoq", + "reference_station": 11 + }, + "02": { + "pos_easting": -1197.62, + "pos_northing": 2366.47, + "pos_site": "summit", + "station_id": 12, + "station_type": "Terianniaq", + "reference_station": 11 + }, + "03": { + "pos_easting": -993.895, + "pos_northing": 3578.9, + "pos_site": "summit", + "station_id": 13, + "station_type": "Ukaleq", + "reference_station": 11 + }, + "04": { + "pos_easting": -790.163, + "pos_northing": 4791.36, + "pos_site": "summit", + "station_id": 14, + "station_type": "Tuttu", + "reference_station": 11 + }, + "05": { + "pos_easting": -586.42, + "pos_northing": 6003.86, + "pos_site": "summit", + "station_id": 15, + "station_type": "Umimmak", + "reference_station": 11 + }, + "06": { + "pos_easting": -382.664, + "pos_northing": 7216.39, + "pos_site": "summit", + "station_id": 16, + "station_type": "Aarluk", + "reference_station": 11 + }, + "07": { + "pos_easting": -178.897, + "pos_northing": 8428.95, + "pos_site": "summit", + "station_id": 17, + "station_type": "Ussuk", + "reference_station": 11 + }, + "08": { + "pos_easting": -188.953, + "pos_northing": 950.358, + "pos_site": "summit", + "station_id": 21, + "station_type": "Amaroq", + "reference_station": 11 + }, + "09": { + "pos_easting": 14.789, + "pos_northing": 2162.74, + "pos_site": "summit", + "station_id": 22, + "station_type": "Avinngaq", + "reference_station": 11 + }, + "10": { + "pos_easting": 218.543, + "pos_northing": 3375.16, + "pos_site": "summit", + "station_id": 23, + "station_type": "Ukaliatsiaq", + "reference_station": 11 + }, + "11": { + "pos_easting": 422.309, + "pos_northing": 4587.61, + "pos_site": "summit", + "station_id": 24, + "station_type": "Qappik", + "reference_station": 11 + }, + "12": { + "pos_easting": 626.087, + "pos_northing": 5800.09, + "pos_site": "summit", + "station_id": 25, + "station_type": "Aataaq", + "reference_station": 11 + }, + "13": { + "pos_easting": 829.877, + "pos_northing": 7012.61, + "pos_site": "summit", + "station_id": 26, + "station_type": "Aaveq", + "reference_station": 11 + }, + "14": { + "pos_easting": 1033.68, + "pos_northing": 8225.16, + "pos_site": "summit", + "station_id": 27, + "station_type": "Eqalussuaq", + "reference_station": 11 + }, + "15": { + "pos_easting": 1430.97, + "pos_northing": 3171.38, + "pos_site": "summit", + "station_id": 33, + "station_type": "Ippernaq", + "reference_station": 11 + }, + "16": { + "pos_easting": 1634.77, + "pos_northing": 4383.82, + "pos_site": "summit", + "station_id": 34, + "station_type": "Isunngaq", + "reference_station": 11 + }, + "17": { + "pos_easting": 1838.58, + "pos_northing": 5596.29, + "pos_site": "summit", + "station_id": 35, + "station_type": "Natseq", + "reference_station": 11 + }, + "18": { + "pos_easting": 2042.41, + "pos_northing": 6808.79, + "pos_site": "summit", + "station_id": 36, + "station_type": "Niisa", + "reference_station": 11 + }, + "19": { + "pos_easting": 2246.24, + "pos_northing": 8021.33, + "pos_site": "summit", + "station_id": 37, + "station_type": "Uppik", + "reference_station": 11 + }, + "20": { + "pos_easting": 2847.22, + "pos_northing": 4179.99, + "pos_site": "summit", + "station_id": 44, + "station_type": "Nattoralik", + "reference_station": 11 + }, + "21": { + "pos_easting": 3051.07, + "pos_northing": 5392.45, + "pos_site": "summit", + "station_id": 45, + "station_type": "Qilanngaq", + "reference_station": 11 + }, + "22": { + "pos_easting": 3254.92, + "pos_northing": 6604.95, + "pos_site": "summit", + "station_id": 46, + "station_type": "Aqisseq", + "reference_station": 11 + }, + "23": { + "pos_easting": 3458.79, + "pos_northing": 7817.48, + "pos_site": "summit", + "station_id": 47, + "station_type": "Natsersuaq", + "reference_station": 11 + }, + "24": { + "pos_easting": 4059.66, + "pos_northing": 3976.14, + "pos_site": "summit", + "station_id": 54, + "station_type": "Qipoqqaq", + "reference_station": 11 + }, + "25": { + "pos_easting": 4263.54, + "pos_northing": 5188.58, + "pos_site": "summit", + "station_id": 55, + "station_type": "Arfiviit", + "reference_station": 11 + }, + "26": { + "pos_easting": 4467.43, + "pos_northing": 6401.07, + "pos_site": "summit", + "station_id": 56, + "station_type": "Alleq", + "reference_station": 11 + }, + "27": { + "pos_easting": 4671.33, + "pos_northing": 7613.58, + "pos_site": "summit", + "station_id": 57, + "station_type": "Qarsaaq", + "reference_station": 11 + }, + "28": { + "pos_easting": 5272.08, + "pos_northing": 3772.24, + "pos_site": "summit", + "station_id": 64, + "station_type": "Tikaagullik", + "reference_station": 11 + }, + "29": { + "pos_easting": 5476, + "pos_northing": 4984.68, + "pos_site": "summit", + "station_id": 65, + "station_type": "Kapisillik", + "reference_station": 11 + }, + "30": { + "pos_easting": 5679.92, + "pos_northing": 6197.15, + "pos_site": "summit", + "station_id": 66, + "station_type": "Eqaluk", + "reference_station": 11 + }, + "31": { + "pos_easting": 5883.86, + "pos_northing": 7409.66, + "pos_site": "summit", + "station_id": 67, + "station_type": "Qeeraq", + "reference_station": 11 + }, + "32": { + "pos_easting": 6484.5, + "pos_northing": 3568.32, + "pos_site": "summit", + "station_id": 74, + "station_type": "Qaleralik", + "reference_station": 11 + }, + "33": { + "pos_easting": 6688.44, + "pos_northing": 4780.74, + "pos_site": "summit", + "station_id": 75, + "station_type": "Uugaq", + "reference_station": 11 + }, + "34": { + "pos_easting": 6892.4, + "pos_northing": 5993.2, + "pos_site": "summit", + "station_id": 76, + "station_type": "Amiqok", + "reference_station": 11 + }, + "35": { + "pos_easting": 7096.38, + "pos_northing": 7205.69, + "pos_site": "summit", + "station_id": 77, + "station_type": "Nerleq", + "reference_station": 11 + } + } + } \ No newline at end of file diff --git a/NuRadioReco/detector/RNO_G/RNO_single_station.json b/NuRadioReco/detector/RNO_G/RNO_single_station.json index e012f5a66..d89458930 100644 --- a/NuRadioReco/detector/RNO_G/RNO_single_station.json +++ b/NuRadioReco/detector/RNO_G/RNO_single_station.json @@ -194,7 +194,6 @@ "ant_rotation_phi": 90.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", - "cab_time_delay": 19.8, "reference_channel": 0, "channel_id": 12, "station_id": 11 @@ -211,7 +210,6 @@ "ant_rotation_phi": 90.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", - "cab_time_delay": 19.8, "reference_channel": 0, "channel_id": 13, "station_id": 11 @@ -228,7 +226,6 @@ "ant_rotation_phi": 90.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", - "cab_time_delay": 19.8, "reference_channel": 0, "channel_id": 14, "station_id": 11 @@ -236,15 +233,14 @@ "15": { "amp_type": "rno_surface", "ant_comment": "second arm south-east from DAQ, north-east position in arm, pointing side-downwards", - "ant_orientation_phi": 240.0, + "ant_orientation_phi": 120.0, "ant_orientation_theta": 135.0, "ant_position_x": -10.276, "ant_position_y": -4.2, "ant_position_z": -2.0, - "ant_rotation_phi": 330.0, + "ant_rotation_phi": 210.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", - "cab_time_delay": 19.8, "reference_channel": 0, "channel_id": 15, "station_id": 11 @@ -258,10 +254,9 @@ "ant_position_x": -9.526, "ant_position_y": -5.5, "ant_position_z": -2.0, - "ant_rotation_phi": 330.0, + "ant_rotation_phi": 210.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", - "cab_time_delay": 19.8, "reference_channel": 0, "channel_id": 16, "station_id": 11 @@ -270,12 +265,12 @@ "17": { "amp_type": "rno_surface", "ant_comment": "second arm south-east from DAQ, south-west position in arm, pointing side-downwards", - "ant_orientation_phi": 60.0, + "ant_orientation_phi": 300.0, "ant_orientation_theta": 135.0, "ant_position_x": -8.776, "ant_position_y": -6.8, "ant_position_z": -2, - "ant_rotation_phi": 330.0, + "ant_rotation_phi": 210.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", "reference_channel": 0, @@ -286,12 +281,12 @@ "18": { "amp_type": "rno_surface", "ant_comment": "third arm south-west from DAQ, south-east position in arm, pointing side-downwards", - "ant_orientation_phi": 120.0, + "ant_orientation_phi": 240.0, "ant_orientation_theta": 135.0, "ant_position_x": 8.776, "ant_position_y": -6.8, "ant_position_z": -2.0, - "ant_rotation_phi": 210.0, + "ant_rotation_phi": 330.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", "reference_channel": 0, @@ -306,7 +301,7 @@ "ant_position_x": 9.526, "ant_position_y": -5.5, "ant_position_z": -2.0, - "ant_rotation_phi": 210.0, + "ant_rotation_phi": 330.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", "reference_channel": 0, @@ -316,12 +311,12 @@ "20": { "amp_type": "rno_surface", "ant_comment": "third arm south-west from DAQ, north-west position in arm, pointing side-downwards", - "ant_orientation_phi": 300.0, + "ant_orientation_phi": 60.0, "ant_orientation_theta": 135.0, - "ant_position_x": -10.276, - "ant_position_y": 4.2, + "ant_position_x": 10.276, + "ant_position_y": -4.2, "ant_position_z": -2.0, - "ant_rotation_phi": 210.0, + "ant_rotation_phi": 330.0, "ant_rotation_theta": 90.0, "ant_type": "createLPDA_100MHz_InfFirn_n1.4", "reference_channel": 0, @@ -381,7 +376,8 @@ "pos_northing": 0.0, "pos_site": "summit", "station_id": 11, - "station_type": null + "station_type": null, + "reference_station": 11 } } } diff --git a/NuRadioReco/detector/RNO_G/analog_components.py b/NuRadioReco/detector/RNO_G/analog_components.py index 7be975e48..771f7637f 100644 --- a/NuRadioReco/detector/RNO_G/analog_components.py +++ b/NuRadioReco/detector/RNO_G/analog_components.py @@ -58,7 +58,7 @@ def iglu_correction_func(temp, freqs): correction_function = iglu_correction_func elif amp_type == 'phased_array': ph = os.path.join(path, 'HardwareResponses/ULP-216+_Plus25DegC.s2p') - ff, S11gain, S21deg, S21gain, S21deg, S12gain, S12deg, S22gain, S22deg = np.loadtxt(ph, comments=['#', '!'], unpack=True) + ff, S11gain, S11deg, S21gain, S21deg, S12gain, S12deg, S22gain, S22deg = np.loadtxt(ph, comments=['#', '!'], unpack=True) ff *= units.MHz amp_gain_discrete = hp.dB_to_linear(S21gain) amp_phase_discrete = S21deg * units.deg diff --git a/NuRadioReco/detector/amp.py b/NuRadioReco/detector/amp.py index 8133af0d6..04dadc714 100644 --- a/NuRadioReco/detector/amp.py +++ b/NuRadioReco/detector/amp.py @@ -25,7 +25,7 @@ def get_amp_response(frequencies, amp_name): get_phase = intp.interp1d(freqs2, np.unwrap(phase)) get_linmag = intp.interp1d(freqs, linmag) - amp_response = np.zeros_like(frequencies, dtype=np.complex) + amp_response = np.zeros_like(frequencies, dtype=complex) mask = (frequencies > max(freqs.min(), freqs2.min())) & (frequencies < min(freqs.max(), freqs2.max())) amp_response[mask] = get_linmag(frequencies[mask]) * np.exp(1j * get_phase(frequencies[mask])) return amp_response diff --git a/NuRadioReco/detector/antenna_models_hash.json b/NuRadioReco/detector/antenna_models_hash.json index f6ad37bec..ef85bad7f 100644 --- a/NuRadioReco/detector/antenna_models_hash.json +++ b/NuRadioReco/detector/antenna_models_hash.json @@ -16,7 +16,6 @@ "dip7cm_z200m_InFirn_RG.pkl": "49ea79abe0784f216ffc4c88db540785a38b87d9", "dip7cm_InfAir.pkl": "407ab67708d750c1c0e6bb83f4a3b41168fc7d1b", "dip7cm_z270mm_InAir.pkl": "a669d71b79d653c4ea8eb6ab3cbb17bb9026ff9d", - "dip7cm_z260mm_InFirn_RG.pkl": "d72fe75ae0f16f9e40d69796aa015e528735d2ec", "dip7cm_z1m_InAir.pkl": "1c5e4f65bd15b227fb2eebe1e61c1813509ebd97", "dip7cm_z1m_InAir_RG_NearHorizontalHD.pkl": "6a08c240e8162b8193da91e1528d301f9a017e3a", "dip7cm_z1m_InAir_RG_NearHorizontalHD2.pkl": "e880c00f3087ed3b3f8276d78645b4ea13217721", @@ -42,16 +41,13 @@ "createLPDA_100MHz_z1cm_InAir_RG.pkl": "9202fa4842792b241366b38cd9a358c03f83acbd", "XFDTD_Hpol_150mmHole_n1.78.pkl": "cfa1e1d1ddbd146ca3b89c580e137e22300551c4", "XFDTD_Vpol_CrossFeed_150mmHole_n1.78.pkl": "5d87513779cf2cb5d3309a9bb89dbb163d656114", - "trislot_RNOG.pkl": "29ef63bc9f25da743dd85d33e5645fc6330f6675", - "RNOG_vpol_v1_n1.73.pkl": "4bd0b5941b31c6c39fd6b438f005075b025fee8d", - "RNOG_vpol_v1_n1.4.pkl": "1b8be66b64f241d26e34567167318aa85c396f2a", - "RNOG_vpol_4inch_half_1.73.pkl": - "f08d263233503f79e5be10c18bc7587b8b421c1d", - "RNOG_vpol_4inch_center_1.73.pkl": - "5f429ed9ed08175a7f75fd44422367d2278bf2e1", - "RNOG_vpol_4inch_wall_1.73.pkl": - "da24017eb80f7a68348a674be2c527457fe19992", - "RNOG_quadslot_v1_1.74.pkl": - "e9c4946dd2176489249c862f32ef272ab627dc28", - "RNOG_quadslot_v2_1.74.pkl": "b99d69578f8e7c006dcce3de51e26c4f463872b8" + "trislot_RNOG.pkl": "29ef63bc9f25da743dd85d33e5645fc6330f6675", + "RNOG_vpol_v1_n1.73.pkl": "4bd0b5941b31c6c39fd6b438f005075b025fee8d", + "RNOG_vpol_v1_n1.4.pkl": "1b8be66b64f241d26e34567167318aa85c396f2a", + "RNOG_vpol_4inch_half_n1.73.pkl": "f08d263233503f79e5be10c18bc7587b8b421c1d", + "RNOG_vpol_4inch_center_n1.73.pkl": "5f429ed9ed08175a7f75fd44422367d2278bf2e1", + "RNOG_vpol_4inch_wall_n1.73.pkl": "da24017eb80f7a68348a674be2c527457fe19992", + "RNOG_quadslot_v1_n1.74.pkl": "e9c4946dd2176489249c862f32ef272ab627dc28", + "RNOG_quadslot_v2_n1.74.pkl": "b99d69578f8e7c006dcce3de51e26c4f463872b8", + "SKALA_InfFirn": "69b3007da083bb722a4494af97221917973ab297" } diff --git a/NuRadioReco/detector/antennapattern.py b/NuRadioReco/detector/antennapattern.py index 89a6d9795..3403a4cb0 100644 --- a/NuRadioReco/detector/antennapattern.py +++ b/NuRadioReco/detector/antennapattern.py @@ -64,7 +64,7 @@ def interpolate_linear_vectorized(x, x0, x1, y0, y1, interpolation_method='compl """ x = np.array(x) mask = x0 != x1 - result = np.zeros_like(x, dtype=np.complex) + result = np.zeros_like(x, dtype=complex) denominator = x1 - x0 if interpolation_method == 'complex': result[mask] = y0[mask] + (y1[mask] - y0[mask]) * (x[mask] - x0[mask]) / denominator[mask] @@ -623,13 +623,13 @@ def parse_AERA_XML_file(path): # get frequencies and angles frequencies_node = root.find("./frequency") - frequencies = np.array(frequencies_node.text.strip().split(), dtype=np.float) * units.MHz + frequencies = np.array(frequencies_node.text.strip().split(), dtype=float) * units.MHz theta_node = root.find("./theta") - thetas = np.array(theta_node.text.strip().split(), dtype=np.float) * units.deg + thetas = np.array(theta_node.text.strip().split(), dtype=float) * units.deg phi_node = root.find("./phi") - phis = np.array(phi_node.text.strip().split(), dtype=np.float) * units.deg + phis = np.array(phi_node.text.strip().split(), dtype=float) * units.deg n_freqs = len(frequencies) n_angles = len(phis) @@ -650,16 +650,16 @@ def parse_AERA_XML_file(path): freq_string = "%.1f" % freq theta_amp_node = root.find("./EAHTheta_amp[@idfreq='%s']" % freq_string) - theta_amps[iFreq] = np.array(theta_amp_node.text.strip().split(), dtype=np.float) * units.m + theta_amps[iFreq] = np.array(theta_amp_node.text.strip().split(), dtype=float) * units.m theta_phase_node = root.find("./EAHTheta_phase[@idfreq='%s']" % freq_string) - theta_phases[iFreq] = np.deg2rad(np.array(theta_phase_node.text.strip().split(" "), dtype=np.float)) + theta_phases[iFreq] = np.deg2rad(np.array(theta_phase_node.text.strip().split(" "), dtype=float)) phi_amp_node = root.find("./EAHPhi_amp[@idfreq='%s']" % freq_string) - phi_amps[iFreq] = np.array(phi_amp_node.text.strip().split(), dtype=np.float) * units.m + phi_amps[iFreq] = np.array(phi_amp_node.text.strip().split(), dtype=float) * units.m phi_phase_node = root.find("./EAHPhi_phase[@idfreq='%s']" % freq_string) - phi_phases[iFreq] = np.deg2rad(np.array(phi_phase_node.text.strip().split(), dtype=np.float)) + phi_phases[iFreq] = np.deg2rad(np.array(phi_phase_node.text.strip().split(), dtype=float)) return frequencies, phis, thetas, phi_amps, phi_phases, theta_amps, theta_phases @@ -1059,8 +1059,8 @@ def get_antenna_response_vectorized(self, freq, zenith, azimuth, orientation_the of the same length as the frequency input """ if self._notfound: - VEL = {'theta': np.ones(len(freq), dtype=np.complex), - 'phi': np.ones(len(freq), dtype=np.complex)} + VEL = {'theta': np.ones(len(freq), dtype=complex), + 'phi': np.ones(len(freq), dtype=complex)} return VEL if isinstance(freq, (float, int)): @@ -1395,7 +1395,7 @@ def _get_antenna_response_vectorized_raw(self, freq, theta, phi, group_delay='fr # Assuming simple cosine, sine falls-off for dummy module H_eff_t = np.zeros_like(Gain) - fmask = freq >= 0 + fmask = freq > 0 H_eff_t[fmask] = Gain[fmask] * max_gain_cross * 1 / freq[fmask] H_eff_t *= np.cos(theta) * np.sin(phi) H_eff_t *= constants.c * units.m / units.s * Z_ant / Z_0 / np.pi diff --git a/NuRadioReco/detector/detector_browser/detector_provider.py b/NuRadioReco/detector/detector_browser/detector_provider.py index 004acaf21..61ca880f4 100644 --- a/NuRadioReco/detector/detector_browser/detector_provider.py +++ b/NuRadioReco/detector/detector_browser/detector_provider.py @@ -29,8 +29,9 @@ def set_detector(self, filename, assume_inf, antenna_by_depth): filename: string Path to the .json file containing the detector description """ - self.__detector = NuRadioReco.detector.detector.Detector.__new__(NuRadioReco.detector.detector.Detector) - self.__detector.__init__(source='json', json_filename=filename, assume_inf=assume_inf, antenna_by_depth=antenna_by_depth) + self.__detector = NuRadioReco.detector.detector.Detector.__new__( + NuRadioReco.detector.detector.Detector, source='json', json_filename=filename, assume_inf=assume_inf, + antenna_by_depth=antenna_by_depth) def set_generic_detector(self, filename, default_station, default_channel, assume_inf, antenna_by_depth): """ @@ -46,8 +47,9 @@ def set_generic_detector(self, filename, default_station, default_channel, assum ID of the channel to be set as default channel """ import NuRadioReco.detector.generic_detector - self.__detector = NuRadioReco.detector.generic_detector.GenericDetector.__new__(NuRadioReco.detector.generic_detector.GenericDetector) - self.__detector.__init__(filename, default_station, default_channel, assume_inf=assume_inf, antenna_by_depth=antenna_by_depth) + self.__detector = NuRadioReco.detector.generic_detector.GenericDetector.__new__( + NuRadioReco.detector.generic_detector.GenericDetector, filename, default_station, + default_channel, assume_inf=assume_inf, antenna_by_depth=antenna_by_depth) def set_event_file(self, filename): """ diff --git a/NuRadioReco/detector/filterresponse.py b/NuRadioReco/detector/filterresponse.py index 66dbd1d79..58008e251 100644 --- a/NuRadioReco/detector/filterresponse.py +++ b/NuRadioReco/detector/filterresponse.py @@ -24,7 +24,7 @@ def get_filter_response_mini_circuits(frequencies, filter_name): get_S21 = intp.interp1d(ff, S21) - response = np.zeros_like(frequencies, dtype=np.complex) + response = np.zeros_like(frequencies, dtype=complex) mask = (frequencies > ff.min()) & (frequencies < ff.max()) response[mask] = get_S21(frequencies[mask]) return response @@ -54,7 +54,7 @@ def get_filter_response_mini_circuits2(frequencies, filter_name): phase2 = -2 * np.pi * np.cumsum(get_group_delay(fff2) * df) get_phase = intp.interp1d(fff2, phase2) - response = np.zeros_like(frequencies, dtype=np.complex) + response = np.zeros_like(frequencies, dtype=complex) mask = (frequencies > max(ff.min(), ff2.min())) & (frequencies < min(ff.max(), ff2.max())) response[mask] = get_insertion_loss(frequencies[mask]) * np.exp(1j * get_phase(frequencies[mask])) return response @@ -83,7 +83,7 @@ def get_filter_response(frequencies, filter_name): get_phase = intp.interp1d(ff2, np.unwrap(phase)) get_insertion_loss = intp.interp1d(ff, insertion_loss) - response = np.zeros_like(frequencies, dtype=np.complex) + response = np.zeros_like(frequencies, dtype=complex) mask = (frequencies > max(ff.min(), ff2.min())) & (frequencies < min(ff.max(), ff2.max())) response[mask] = get_insertion_loss(frequencies[mask]) * np.exp(1j * get_phase(frequencies[mask])) return response diff --git a/NuRadioReco/detector/generic_detector.py b/NuRadioReco/detector/generic_detector.py index 0c83f74ed..c70078b26 100644 --- a/NuRadioReco/detector/generic_detector.py +++ b/NuRadioReco/detector/generic_detector.py @@ -24,7 +24,7 @@ class GenericDetector(NuRadioReco.detector.detector_base.DetectorBase): also be defined. The default channel has to be part of the default station. It works the same way as the default station: Any property missing from one of the other channels will be taken from the default channel. - The GenericDetector also ignores commission and decommision times and should + The GenericDetector also ignores commission and decommission times and should therefore not be used for real data, but only for simulation studies. This detector only accepts json detector descriptions or dictionary. """ @@ -73,7 +73,7 @@ def __init__(self, json_filename, default_station=None, default_channel=None, de If 'dictionary' is passed to the parameter source, the dictionary passed to this parameter will be used for the detector description. assume_inf : Bool - Default to True, if true forces antenna madels to have infinite boundary conditions, otherwise the antenna + Default to True, if true forces antenna models to have infinite boundary conditions, otherwise the antenna madel will be determined by the station geometry. antenna_by_depth: bool (default True) if True the antenna model is determined automatically depending on the depth of the antenna. @@ -86,50 +86,72 @@ def __init__(self, json_filename, default_station=None, default_channel=None, de if (default_station is None) and (default_channel is None) and (default_device is None): # load detector super(GenericDetector, self).__init__(source=source, json_filename=json_filename, - dictionary=dictionary, assume_inf=assume_inf, - antenna_by_depth=antenna_by_depth) + dictionary=dictionary, assume_inf=assume_inf, + antenna_by_depth=antenna_by_depth) else: if source == "json": - # load json as dictionary and pass that to the detector, this is needed in order to not overwrite the json - # when updating the table to include reference_station/channel/device + # load json as dictionary and pass that to the detector, this is needed in order to not overwrite the + # json when updating the table to include reference_station/channel/device with open(json_filename, "r") as json_input: dictionary = json.load(json_input) super(GenericDetector, self).__init__(source="dictionary", json_filename=None, - dictionary=dictionary, assume_inf=assume_inf, - antenna_by_depth=antenna_by_depth) + dictionary=dictionary, assume_inf=assume_inf, + antenna_by_depth=antenna_by_depth) if default_station is not None: - logger.warning("DeprecationWarning: replace default_station by setting a 'reference_station' for each station in the detector description. This allows to define multiple default station types") + logger.warning( + "DeprecationWarning: replace default_station by setting a 'reference_station' for each station in " + "the detector description. This allows to define multiple default station types") # fill default station info into 'reference_station' field for all stations in detector for sta in self._stations: if 'reference_station' in sta.keys(): - logger.warning(f"Station already has a reference station {sta['reference_station']}. Ignoring deprecated 'default_station'") + logger.warning( + f"Station already has a reference station {sta['reference_station']}. Ignoring deprecated " + f"'default_station'") else: - logger.warning(f"Setting deprecated 'default_station' as reference station ({default_station}) as requested") + logger.warning( + f"Setting deprecated 'default_station' as reference station ({default_station}) " + f"as requested") Station = Query() - self._stations.update({'reference_station': default_station}, (Station.station_id == sta["station_id"])) + self._stations.update({'reference_station': default_station}, + (Station.station_id == sta["station_id"])) if default_channel is not None: - logger.warning("DeprecationWarning: replace default_channel by setting a 'reference_channel' for each channel in the detector description. This allows to define multiple default channel types") + logger.warning( + "DeprecationWarning: replace default_channel by setting a 'reference_channel' for each channel in " + "the detector description. This allows to define multiple default channel types") # fill default channel info into 'reference_channel' field for all channels in detector for chan in self._channels: if 'reference_channel' in chan.keys(): - logger.warning(f"Channel already has a reference channel {chan['reference_channel']}. Ignoring deprecated 'default_channel'") + logger.warning( + f"Channel already has a reference channel {chan['reference_channel']}. Ignoring " + f"deprecated 'default_channel'") else: - logger.warning(f"Setting deprecated 'default_channel' as reference channel ({default_channel}) as requested") + logger.warning( + f"Setting deprecated 'default_channel' as reference channel ({default_channel}) " + f"as requested") Channel = Query() - self._channels.update({'reference_channel': default_channel}, (Channel.station_id == chan["station_id"]) & (Channel.channel_id == chan["channel_id"])) + self._channels.update({'reference_channel': default_channel}, + (Channel.station_id == chan["station_id"]) & ( + Channel.channel_id == chan["channel_id"])) if default_device is not None: - logger.warning("DeprecationWarning: replace default_device by setting a 'reference_device' for each device in the detector description. This allows to define multiple default device types") + logger.warning( + "DeprecationWarning: replace default_device by setting a 'reference_device' for each device in " + "the detector description. This allows to define multiple default device types") # fill default device info into 'reference_device' field for all devices in detector for dev in self._buffered_devices: if 'reference_device' in dev.keys(): - logger.warning(f"Device already has a reference device {dev['reference_device']}. Ignoring deprecated 'default_device'") + logger.warning( + f"Device already has a reference device {dev['reference_device']}. Ignoring deprecated " + f"'default_device'") else: - logger.warning(f"Setting deprecated 'default_device' as reference device ({default_device}) as requested") + logger.warning( + f"Setting deprecated 'default_device' as reference device ({default_device}) as requested") Device = Query() - self._devices.update({'reference_device': default_device}, (Device.station_id == dev["station_id"]) & (Device.device_id == dev["device_id"])) + self._devices.update({'reference_device': default_device}, + (Device.station_id == dev["station_id"]) & ( + Device.device_id == dev["device_id"])) self.__default_station = default_station # TODO maybe these dicts/lists can be omitted # a lookup with one reference station for each station in the detector description @@ -144,7 +166,8 @@ def __init__(self, json_filename, default_station=None, default_channel=None, de self.__reference_stations = {} Station = Query() for reference_station_id in self.__reference_station_ids: - self.__reference_stations[reference_station_id] = self._stations.get((Station.station_id == reference_station_id)) + self.__reference_stations[reference_station_id] = self._stations.get( + (Station.station_id == reference_station_id)) self.__station_changes_for_event = [] self.__run_number = None @@ -155,32 +178,38 @@ def __init__(self, json_filename, default_station=None, default_channel=None, de for sta in self._stations.all(): if "reference_station" in sta: ref = self._stations.get( - (Station.station_id == sta["reference_station"])) + (Station.station_id == sta["reference_station"])) if ref is None: raise ValueError( 'The reference station {} was not found in the detector description'.format( - ref["reference_station"])) + sta["reference_station"])) Channel = Query() for chan in self._channels.all(): if "reference_channel" in chan: ref = self._channels.get( - (Channel.station_id == chan["station_id"]) & (Channel.channel_id == chan["reference_channel"])) + (Channel.station_id == chan["station_id"]) & (Channel.channel_id == chan["reference_channel"])) if ref is None: - raise ValueError( - 'The reference channel {} of station {} was not found in the detector description'.format( - ref["reference_channel"], ref["station_id"])) + # Check if reference channel sits in reference station + reference_station_id = self.__lookuptable_reference_station[chan["station_id"]] + ref = self._channels.get( + (Channel.station_id == reference_station_id) & + (Channel.channel_id == chan["reference_channel"]) + ) + if ref is None: + raise ValueError( + 'The reference channel {} of station {} was not found in the detector description'.format( + chan["reference_channel"], chan["station_id"])) Device = Query() for dev in self._devices.all(): - if "reference_device" in dev: - ref = self._devices.get( + if "reference_device" in dev: + ref = self._devices.get( (Device.station_id == dev["station_id"]) & (Device.channel_id == dev["reference_device"])) - if ref is None: - raise ValueError( - 'The reference device {} of station {} was not found in the detector description'.format( - ref["reference_device"], ref["station_id"])) - + if ref is None: + raise ValueError( + 'The reference device {} of station {} was not found in the detector description'.format( + dev["reference_device"], dev["station_id"])) def _get_station(self, station_id): if station_id not in self._buffered_stations.keys(): @@ -188,7 +217,9 @@ def _get_station(self, station_id): res = copy.copy(self._buffered_stations[station_id]) if self.__run_number is not None and self.__event_id is not None: for change in self.__station_changes_for_event: - if change['station_id'] == station_id and change['run_number'] == self.__run_number and change['event_id'] == self.__event_id: + if change['station_id'] == station_id and \ + change['run_number'] == self.__run_number and \ + change['event_id'] == self.__event_id: for name, value in change['properties'].items(): res[name] = value return res @@ -228,14 +259,16 @@ def _query_channels(self, station_id, raw=False): new_channel['station_id'] = station_id res.append(new_channel) - # now we look if there are reference fields to fill. Will use reference_station_id, which is either the station or the reference + # now we look if there are reference fields to fill. Will use reference_station_id, which is either the + # station or the reference for channel in res: if 'reference_channel' in channel: # add to dictionary to keep track of reference channels TODO this is not really needed? self.__reference_channel_ids[(station_id, channel['channel_id'])] = channel['reference_channel'] # there is a reference, so we have to get it ref_chan = self._channels.get( - (Channel.station_id == reference_station_id) & (Channel.channel_id == channel['reference_channel'])) + (Channel.station_id == reference_station_id) & ( + Channel.channel_id == channel['reference_channel'])) for key in ref_chan.keys(): if key not in channel.keys() and key != 'station_id' and key != 'channel_id': channel[key] = ref_chan[key] @@ -262,14 +295,15 @@ def _query_devices(self, station_id, raw=False): new_device['station_id'] = station_id res.append(new_device) - # now we look if there are reference fields to fill. Will use reference_station_id, which is either the station or the reference + # now we look if there are reference fields to fill. Will use reference_station_id, which is either the + # station or the reference for device in res: if 'reference_device' in device: # add to dictionary to keep track of reference devices TODO this is not really needed? self.__reference_device_ids[(station_id, device['device_id'])] = device['reference_device'] # there is a reference, so we have to get it ref_dev = self._devices.get( - (Device.station_id == reference_station_id) & (Device.device_id == device['reference_device'])) + (Device.station_id == reference_station_id) & (Device.device_id == device['reference_device'])) for key in ref_dev.keys(): if key not in device.keys() and key != 'station_id' and key != 'device_id': device[key] = ref_dev[key] @@ -313,11 +347,14 @@ def add_generic_station(self, station_dict): reference station """ if "reference_station" not in station_dict and self.__default_station is not None: - logger.warning('DeprecationWarning: Generating a station via `add_generic_station` that has no "reference_station" specified.') - logger.warning(f'DeprecationWarning: Taking the deprecated "default_station" ({self.__default_station}) as "reference_station".') + logger.warning( + 'DeprecationWarning: Generating a station via `add_generic_station` that has no "reference_station" ' + 'specified.') + logger.warning( + f'DeprecationWarning: Taking the deprecated "default_station" ({self.__default_station}) as ' + f'"reference_station".') station_dict["reference_station"] = self.__default_station - if station_dict['station_id'] in self._buffered_stations.keys(): logger.warning('Station with ID {} already exists in buffer. Cannot add station with same ID'.format( station_dict['station_id'])) @@ -347,7 +384,6 @@ def add_generic_station(self, station_dict): new_device['station_id'] = station_dict['station_id'] self._buffered_devices[station_dict['station_id']][device['device_id']] = new_device - def add_station_properties_for_event(self, properties, station_id, run_number, event_id): """ Adds an entry to the list of event-specific changes to the detector @@ -423,7 +459,7 @@ def get_reference_station_id(self, station_id): """ Get the properties of the reference station """ - return self.__reference_station_ids[station_id] + return self.__reference_station_ids[station_id] def get_reference_stations(self): """ @@ -439,7 +475,7 @@ def get_default_station(self): def get_reference_station_ids(self): """ - Get the whole diectionary of reference stations + Get the whole dictionary of reference stations """ return self.__reference_station_ids @@ -455,22 +491,21 @@ def get_default_station_id(self): logger.warning( f'more than one station id set as "reference station": {self.__reference_station_ids},\ continue with first entry: {self.__reference_station_ids[0]}') - return self.__reference_station_ids[0] + return self.__reference_station_ids[0] def get_default_channel(self): """ Get the properties of the default channel """ logger.warning("The use of 'default_channel' is deprecated. returning None") - return None #self.__default_channel + return None # self.__default_channel def get_default_channel_id(self): """ Get the ID of the default channel """ logger.warning("The use of 'default_channel' is deprecated. returning None") - return None #self.__default_channel_id - + return None # self.__default_channel_id def get_raw_station(self, station_id): """ diff --git a/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_spectrum.py b/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_spectrum.py index c535a20d7..f19ea72cd 100644 --- a/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_spectrum.py +++ b/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_spectrum.py @@ -54,7 +54,10 @@ def update_sim_spectrum_plot(i_event, filename, signal_types, station_id, juser_ if sim_station is None: return {} fig = plotly.subplots.make_subplots(rows=1, cols=1) - for i_electric_field, electric_field in enumerate(sim_station.get_electric_fields()): + efields = list(sim_station.get_electric_fields()) + efield_ids = [efield.get_unique_identifier() for efield in efields] + efields_sorted = [k[1] for k in sorted(zip(efield_ids, efields))] + for i_electric_field, electric_field in enumerate(efields_sorted): if electric_field.get_parameter(efp.ray_path_type) in signal_types: for polarization in range(1, 3): fig.append_trace( diff --git a/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_trace.py b/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_trace.py index 892591a8b..8f3a81a98 100644 --- a/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_trace.py +++ b/NuRadioReco/eventbrowser/apps/simulation_plots/sim_electric_field_trace.py @@ -54,7 +54,10 @@ def update_sim_trace_plot(i_event, filename, signal_types, station_id, juser_id) if sim_station is None: return {} fig = plotly.subplots.make_subplots(rows=1, cols=1) - for i_electric_field, electric_field in enumerate(sim_station.get_electric_fields()): + efields = list(sim_station.get_electric_fields()) + efield_ids = [efield.get_unique_identifier() for efield in efields] + efields_sorted = [k[1] for k in sorted(zip(efield_ids, efields))] + for i_electric_field, electric_field in enumerate(efields_sorted): if electric_field.get_parameter(efp.ray_path_type) in signal_types: for polarization in range(1, 3): fig.append_trace( diff --git a/NuRadioReco/eventbrowser/apps/trace_plots/channel_spectrum.py b/NuRadioReco/eventbrowser/apps/trace_plots/channel_spectrum.py index 4b8bfd051..19e0b24ce 100644 --- a/NuRadioReco/eventbrowser/apps/trace_plots/channel_spectrum.py +++ b/NuRadioReco/eventbrowser/apps/trace_plots/channel_spectrum.py @@ -4,9 +4,9 @@ from NuRadioReco.utilities import units from NuRadioReco.eventbrowser.default_layout import default_layout import numpy as np -from dash import dcc +from dash import dcc, callback from dash.dependencies import State -from NuRadioReco.eventbrowser.app import app +# from NuRadioReco.eventbrowser.app import app import NuRadioReco.eventbrowser.dataprovider provider = NuRadioReco.eventbrowser.dataprovider.DataProvider() @@ -16,14 +16,16 @@ ] -@app.callback( +@callback( dash.dependencies.Output('channel-spectrum', 'figure'), [dash.dependencies.Input('trigger-trace', 'children'), dash.dependencies.Input('event-counter-slider', 'value'), dash.dependencies.Input('filename', 'value'), dash.dependencies.Input('station-id-dropdown', 'value')], - [State('user_id', 'children')]) -def update_channel_spectrum(trigger, evt_counter, filename, station_id, juser_id): + [State('user_id', 'children'), + State('channel-spectrum-log-linear-switch', 'children')] + ) +def update_channel_spectrum(trigger, evt_counter, filename, station_id, juser_id, yscale): if filename is None or station_id is None: return {} user_id = json.loads(juser_id) @@ -33,27 +35,22 @@ def update_channel_spectrum(trigger, evt_counter, filename, station_id, juser_id station = evt.get_station(station_id) fig = plotly.subplots.make_subplots(rows=1, cols=1) for i, channel in enumerate(station.iter_channels()): - if channel.get_trace() is None: + spec = channel.get_frequency_spectrum() + if spec is None: continue - tt = channel.get_times() - dt = tt[1] - tt[0] - if channel.get_trace() is None: - continue - trace = channel.get_trace() - ff = np.fft.rfftfreq(len(tt), dt) - spec = np.abs(np.fft.rfft(trace, norm='ortho')) + freqs = channel.get_frequencies() fig.append_trace(plotly.graph_objs.Scatter( - x=ff / units.MHz, - y=spec / units.mV, + x=freqs / units.MHz, + y=np.abs(spec) / units.mV, opacity=0.7, marker={ 'color': colors[i % len(colors)], 'line': {'color': colors[i % len(colors)]} }, - name='Channel {}'.format(i) + name='Channel {}'.format(channel.get_id()) ), 1, 1) fig['layout'].update(default_layout) fig['layout']['legend']['uirevision'] = filename fig['layout']['xaxis1'].update(title='frequency [MHz]') - fig['layout']['yaxis1'].update(title='amplitude [mV]') + fig['layout']['yaxis'].update(title='amplitude [mV]', type=yscale) return fig diff --git a/NuRadioReco/eventbrowser/apps/trace_plots/channel_time_trace.py b/NuRadioReco/eventbrowser/apps/trace_plots/channel_time_trace.py index ebe267dfe..36d1780bf 100644 --- a/NuRadioReco/eventbrowser/apps/trace_plots/channel_time_trace.py +++ b/NuRadioReco/eventbrowser/apps/trace_plots/channel_time_trace.py @@ -4,9 +4,9 @@ from NuRadioReco.utilities import units from NuRadioReco.eventbrowser.default_layout import default_layout import numpy as np -from dash import dcc +from dash import dcc, callback from dash.dependencies import State -from NuRadioReco.eventbrowser.app import app +# from NuRadioReco.eventbrowser.app import app import NuRadioReco.eventbrowser.dataprovider provider = NuRadioReco.eventbrowser.dataprovider.DataProvider() @@ -16,7 +16,7 @@ ] -@app.callback( +@callback( dash.dependencies.Output('time-trace', 'figure'), [dash.dependencies.Input('trigger-trace', 'children'), dash.dependencies.Input('event-counter-slider', 'value'), @@ -52,8 +52,8 @@ def update_time_trace(trigger, evt_counter, filename, station_id, juser_id): 'color': colors[i % len(colors)], 'line': {'color': colors[i % len(colors)]} }, - name='Channel {}'.format(i), - uid='Channel {}'.format(i) + name='Channel {}'.format(channel.get_id()), + uid='Channel {}'.format(channel.get_id()) ), 1, 1) fig['layout'].update(default_layout) fig['layout']['legend']['uirevision'] = filename # only update channel selection on changing files. diff --git a/NuRadioReco/eventbrowser/apps/trace_plots/multi_channel_plot.py b/NuRadioReco/eventbrowser/apps/trace_plots/multi_channel_plot.py index bce94c7f3..da96f60bb 100644 --- a/NuRadioReco/eventbrowser/apps/trace_plots/multi_channel_plot.py +++ b/NuRadioReco/eventbrowser/apps/trace_plots/multi_channel_plot.py @@ -12,10 +12,9 @@ from NuRadioReco.utilities import trace_utilities """ import numpy as np -from dash import html -from dash import dcc +from dash import dcc, html, callback from dash.dependencies import Input, Output, State -from NuRadioReco.eventbrowser.app import app +# from NuRadioReco.eventbrowser.app import app import os import NuRadioReco.detector.antennapattern import NuRadioReco.eventbrowser.dataprovider @@ -72,7 +71,7 @@ ] -@app.callback( +@callback( dash.dependencies.Output('dropdown-traces', 'options'), [dash.dependencies.Input('event-counter-slider', 'value'), dash.dependencies.Input('filename', 'value'), @@ -99,7 +98,7 @@ def get_dropdown_traces_options(evt_counter, filename, station_id, juser_id): return options -@app.callback( +@callback( Output('template-input-group', 'style'), [Input('dropdown-traces', 'value')] ) @@ -117,7 +116,7 @@ def get_L1(a): return l1 -@app.callback( +@callback( dash.dependencies.Output('time-traces', 'figure'), [dash.dependencies.Input('event-counter-slider', 'value'), dash.dependencies.Input('filename', 'value'), @@ -126,9 +125,11 @@ def get_L1(a): dash.dependencies.Input('station-id-dropdown', 'value'), dash.dependencies.Input('open-template-button', 'n_clicks_timestamp')], [State('user_id', 'children'), - State('template-directory-input', 'value')]) + State('template-directory-input', 'value'), + State('channel-spectrum-log-linear-switch', 'children')] +) def update_multi_channel_plot(evt_counter, filename, dropdown_traces, dropdown_info, station_id, - open_template_timestamp, juser_id, template_directory): + open_template_timestamp, juser_id, template_directory, yscale): if filename is None or station_id is None: return {} user_id = json.loads(juser_id) @@ -420,6 +421,7 @@ def update_multi_channel_plot(evt_counter, filename, dropdown_traces, dropdown_i fig['layout']['yaxis{:d}'.format(i * 2 + 1)].update( title='Ch. {}
voltage [mV]'.format(channel_id) ) + fig['layout']['yaxis{:d}'.format(i * 2 + 2)].update(type=yscale) if channel.get_trace() is None: continue diff --git a/NuRadioReco/eventbrowser/apps/trace_plots/rec_electric_field_spectrum.py b/NuRadioReco/eventbrowser/apps/trace_plots/rec_electric_field_spectrum.py index 2085c79ba..8cd87ecb7 100644 --- a/NuRadioReco/eventbrowser/apps/trace_plots/rec_electric_field_spectrum.py +++ b/NuRadioReco/eventbrowser/apps/trace_plots/rec_electric_field_spectrum.py @@ -4,7 +4,7 @@ from NuRadioReco.utilities import units from NuRadioReco.eventbrowser.default_layout import default_layout import numpy as np -from dash import dcc +from dash import dcc, html from dash.dependencies import State from NuRadioReco.eventbrowser.app import app import NuRadioReco.eventbrowser.dataprovider @@ -22,8 +22,9 @@ dash.dependencies.Input('event-counter-slider', 'value'), dash.dependencies.Input('filename', 'value'), dash.dependencies.Input('station-id-dropdown', 'value')], - [State('user_id', 'children')]) -def update_efield_spectrum(trigger, evt_counter, filename, station_id, juser_id): + [State('user_id', 'children'), + State('channel-spectrum-log-linear-switch', 'children')]) +def update_efield_spectrum(trigger, evt_counter, filename, station_id, juser_id, yscale): if filename is None or station_id is None: return {} user_id = json.loads(juser_id) @@ -61,5 +62,5 @@ def update_efield_spectrum(trigger, evt_counter, filename, station_id, juser_id) ), 1, 1) fig['layout'].update(default_layout) fig['layout']['xaxis1'].update(title='frequency [MHz]') - fig['layout']['yaxis1'].update(title='amplitude [mV/m]') + fig['layout']['yaxis'].update(title='amplitude [mV/m]', type=yscale) return fig diff --git a/NuRadioReco/eventbrowser/apps/traces.py b/NuRadioReco/eventbrowser/apps/traces.py index c14ccd20d..7eea639d2 100644 --- a/NuRadioReco/eventbrowser/apps/traces.py +++ b/NuRadioReco/eventbrowser/apps/traces.py @@ -1,5 +1,5 @@ from __future__ import absolute_import, division, print_function # , unicode_literals -from dash import html +from dash import html, no_update import NuRadioReco.eventbrowser.apps.trace_plots.rec_electric_field_trace import NuRadioReco.eventbrowser.apps.trace_plots.rec_electric_field_spectrum import NuRadioReco.eventbrowser.apps.trace_plots.channel_time_trace @@ -27,6 +27,9 @@ html.Div([ html.Div([ 'Electric Field Spectrum', + ' (y-scale: ', + html.Button(id='efield-spectrum-log-linear-switch', children='linear'), + ')', html.Button('Show', id='toggle_efield_spectrum', n_clicks=0, style={'float':'right'}) ], className='panel-heading'), html.Div(NuRadioReco.eventbrowser.apps.trace_plots.rec_electric_field_spectrum.layout, @@ -45,6 +48,9 @@ html.Div([ html.Div([ 'Channel Spectrum', + ' (y-scale: ', + html.Button(id='channel-spectrum-log-linear-switch', children='linear'), + ')', html.Button('Show', id='toggle_channel_spectrum', n_clicks=0, style={'float':'right'}) ], className='panel-heading'), html.Div(NuRadioReco.eventbrowser.apps.trace_plots.channel_spectrum.layout, @@ -60,7 +66,7 @@ ]) @app.callback( - [Output('channel_traces_layout', 'children'), + [Output('channel_traces_layout', 'style'), Output('toggle_channel_traces', 'children')], [Input('toggle_channel_traces', 'n_clicks')], State('toggle_channel_traces', 'children'), @@ -68,12 +74,12 @@ ) def toggle_channel_trace_plot(button_clicks, showhide): if showhide == 'Hide': - return [], 'Show' + return {'flex': '1', 'display': 'none'}, 'Show' else: - return NuRadioReco.eventbrowser.apps.trace_plots.channel_time_trace.layout, 'Hide' + return {'flex' : '1'}, 'Hide' @app.callback( - [Output('channel_spectrum_layout', 'children'), + [Output('channel_spectrum_layout', 'style'), Output('toggle_channel_spectrum', 'children')], [Input('toggle_channel_spectrum', 'n_clicks')], State('toggle_channel_spectrum', 'children'), @@ -81,12 +87,12 @@ def toggle_channel_trace_plot(button_clicks, showhide): ) def toggle_channel_spectrum_plot(button_clicks, showhide): if showhide == 'Hide': - return [], 'Show' + return {'flex': '1', 'display': 'none'}, 'Show' else: - return NuRadioReco.eventbrowser.apps.trace_plots.channel_spectrum.layout, 'Hide' + return {'flex': '1'}, 'Hide' @app.callback( - [Output('efield_traces_layout', 'children'), + [Output('efield_traces_layout', 'style'), Output('toggle_efield_traces', 'children')], [Input('toggle_efield_traces', 'n_clicks')], State('toggle_efield_traces', 'children'), @@ -94,12 +100,12 @@ def toggle_channel_spectrum_plot(button_clicks, showhide): ) def toggle_efield_traces_plot(button_clicks, showhide): if showhide == 'Hide': - return [], 'Show' + return {'flex': '1', 'display': 'none'}, 'Show' else: - return NuRadioReco.eventbrowser.apps.trace_plots.rec_electric_field_trace.layout, 'Hide' + return {'flex': '1'}, 'Hide' @app.callback( - [Output('efield_spectrum_layout', 'children'), + [Output('efield_spectrum_layout', 'style'), Output('toggle_efield_spectrum', 'children')], [Input('toggle_efield_spectrum', 'n_clicks')], State('toggle_efield_spectrum', 'children'), @@ -107,6 +113,49 @@ def toggle_efield_traces_plot(button_clicks, showhide): ) def toggle_efield_spectrum_plot(button_clicks, showhide): if showhide == 'Hide': - return [], 'Show' + return {'flex': '1', 'display': 'none'}, 'Show' else: - return NuRadioReco.eventbrowser.apps.trace_plots.rec_electric_field_spectrum.layout, 'Hide' \ No newline at end of file + return {'flex': '1'}, 'Hide' + +# callback to change frequency spectra between linear and log scale +@app.callback( + [Output('efield-spectrum', 'figure', allow_duplicate=True), # this is a 'duplicate' callback - requires dash >= 2.9 + Output('channel-spectrum', 'figure', allow_duplicate=True), # this is a 'duplicate' callback - requires dash >= 2.9 + Output('time-traces', 'figure', allow_duplicate=True), + Output('channel-spectrum-log-linear-switch', 'children'), + Output('efield-spectrum-log-linear-switch', 'children') + ], + [Input('channel-spectrum-log-linear-switch', 'n_clicks'), + Input('efield-spectrum-log-linear-switch', 'n_clicks')], + [State('channel-spectrum-log-linear-switch', 'children'), + State('efield-spectrum', 'figure'), + State('channel-spectrum', 'figure'), + State('time-traces', 'figure'), + ], prevent_initial_call=True +) +def toggle_linear_log_scale(button_clicks, button2_clicks, button_current_value, efield_spectrum, channel_spectrum, multichannel_plot): + outputs = [] + if button_current_value == 'linear': # switch to log + new_value = 'log' + else: + new_value = 'linear' + for spectrum_plot in [efield_spectrum, channel_spectrum]: + try: + spectrum_plot['layout']['yaxis']['type'] = new_value + outputs.append(spectrum_plot) + except KeyError: + outputs.append(no_update) + + try: + yaxes = [key for key in multichannel_plot['layout'].keys() if 'yaxis' in key] + yaxes = [key for key in yaxes if len(key)>5] # omit key 'yaxis' + yaxes_even = [key for key in yaxes if not (int(key.split('yaxis')[-1]) % 2)] + for yaxis in yaxes_even: + multichannel_plot['layout'][yaxis]['type'] = new_value + outputs.append(multichannel_plot) + except KeyError as e: + outputs.append(no_update) + + outputs.append(new_value) + outputs.append(new_value) + return outputs \ No newline at end of file diff --git a/NuRadioReco/eventbrowser/dataprovider.py b/NuRadioReco/eventbrowser/dataprovider.py index 772d422c3..bd07dcc35 100644 --- a/NuRadioReco/eventbrowser/dataprovider.py +++ b/NuRadioReco/eventbrowser/dataprovider.py @@ -1,23 +1,148 @@ -import NuRadioReco.eventbrowser.dataprovider_root -import NuRadioReco.eventbrowser.dataprovider_nur +import threading +import logging +import six +import NuRadioReco.utilities.metaclasses +import os +import time +from NuRadioReco.modules.io import NuRadioRecoio +logging.basicConfig() +logger = logging.getLogger('eventbrowser.dataprovider') +logger.setLevel(logging.INFO) +try: + from NuRadioReco.modules.io.RNO_G.readRNOGDataMattak import _readRNOGData_eventbrowser +except ImportError as e: + logger.error( + msg=( + "Failed to import NuRadioReco.modules.io.RNO_G.readRNOGDataMattak, `.root` files can not be read." + " If you are only trying to read .nur files this error can be ignored.." + ), exc_info=e + ) + _readRNOGData_eventbrowser = None # if we don't define this we'll raise more errors later + +@six.add_metaclass(NuRadioReco.utilities.metaclasses.Singleton) class DataProvider(object): - __instance = None + __lock = threading.Lock() + + def __init__(self, filetype='auto', max_user_instances=6): + """" + Interface to .nur or .root file IO for the eventbrowser + + Parameters + ---------- + filetype: 'auto' | 'nur' | 'root' + Which IO module to use: - def __new__(cls): - if DataProvider.__instance is None: - DataProvider.__instance = object.__new__(cls) - return DataProvider.__instance + * 'nur': The standard `NuRadioRecoio` module for reading `.nur` files + * 'root': Read RNO-G root files with the + :mod:`NuRadioReco.modules.io.RNO_G.readRNOGDataMattak` module (requires + `mattak` to be installed from https://github.com/RNO-G/mattak) + * 'auto' (default): determine the appropriate file reader based on the + file endings - def __init__(self): - self.__data_provider = None + max_user_instances: int (default: 6) + Each unique session id gets its own reader, up to a maximum + of ``max_user_instances`` concurrent readers. Subsequent + requests for new readers drop older readers. + + """ + logger.info("Creating new DataProvider instance") + self.__max_user_instances = max_user_instances + self.__user_instances = {} + self.__filetype = filetype def set_filetype(self, use_root): + """ + DEPRECATED - we use the file endings to determine file type now + + Set the filetype to read in. + + Parameters + ---------- + use_root: bool + If True, use the :mod:`NuRadioReco.modules.io.RNO_G.readRNOGDataMattak` module + to read in RNO-G ROOT files. Otherwise, use the 'standard' NuRadioMC `.nur` file + reader. + """ if use_root: - self.__data_provider = NuRadioReco.eventbrowser.dataprovider_root.DataProviderRoot() + self.__filetype = 'root' else: - self.__data_provider = NuRadioReco.eventbrowser.dataprovider_nur.DataProvider() + self.__filetype = 'nur' def get_file_handler(self, user_id, filename): - return self.__data_provider.get_file_handler(user_id, filename) + """ + Interface to retrieve the actual IO module + + Thread-locked to avoid competing threads reading the same file. + + Parameters + ---------- + user_id: str + unique user_id to allow multiple users to use the dataprovider at once + filename: str | list + path or paths to files to read in + + """ + # because the dash app is multi-threaded, we use a lock to avoid + # multiple simultaneous requests to read the same file + + thread = threading.get_ident() + total_threads = threading.active_count() + logger.debug(f"Thread {thread} out of total {total_threads} requesting file_handler for user {user_id}") + with self.__lock: + logger.debug(f"Thread {thread} locked, getting file_handler for user {user_id}...") + + if filename is None: + return None + + if user_id not in self.__user_instances: + # Occasionally, this gets called before the user_id is initialized (user_id=None). + # We can save a bit of memory by re-using this instance for the first initialized user. + if None in self.__user_instances: + self.__user_instances[user_id] = self.__user_instances.pop(None) + else: + self.__user_instances[user_id] = dict( + reader=None, filename=None) + + if isinstance(filename, str): + filename = [filename] + + # determine which reader to use + use_root = self.__filetype == 'root' + if (self.__filetype == 'auto') and any([f.endswith('.root') for f in filename]): + use_root = True + + if filename != self.__user_instances[user_id]['filename']: + # user is requesting new file -> close current file and open new one + if use_root: + if _readRNOGData_eventbrowser is None: + raise ImportError( + "The .root reading interface `NuRadioReco.modules.io.RNO_G.readRNOGDataMattak`" + " is not available, so .root files can not be read. Make sure you have a working installation" + " of mattak (https://github.com/RNO-G/mattak)." + ) + reader = _readRNOGData_eventbrowser(load_run_table=False) + reader.begin([os.path.dirname(f) for f in filename], overwrite_sampling_rate=3.2) + reader.get_event_ids() + else: + reader = NuRadioRecoio.NuRadioRecoio(filename) # NuRadioRecoio takes filenames as argument to __init__ + self.__user_instances[user_id] = dict( + reader=reader, filename=filename, + ) + + # update last access time + self.__user_instances[user_id]['last_access_time'] = time.time() + + # check if we exceed maximum number of concurrent sessions + if len(self.__user_instances) > self.__max_user_instances: + users = { + self.__user_instances[k]['last_access_time']:k + for k in self.__user_instances.keys() + } + oldest_user = users[min(users)] + # drop oldest session + self.__user_instances.pop(oldest_user) + + logger.debug(f"Returning file_handler and releasing lock") + return self.__user_instances[user_id]['reader'] diff --git a/NuRadioReco/eventbrowser/dataprovider_nur.py b/NuRadioReco/eventbrowser/dataprovider_nur.py deleted file mode 100644 index a9fb2ea80..000000000 --- a/NuRadioReco/eventbrowser/dataprovider_nur.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import absolute_import, division, print_function # , unicode_literals -from NuRadioReco.modules.io import NuRadioRecoio - - -class DataProvider(object): - __instance = None - - def __new__(cls): - if DataProvider.__instance is None: - DataProvider.__instance = object.__new__(cls) - return DataProvider.__instance - - def __init__(self): - self.__user_instances = {} - - def get_file_handler(self, user_id, filename): - if filename is None: - return - if user_id not in self.__user_instances: - self.__user_instances[user_id] = NuRadioRecoio.NuRadioRecoio(filename) - if filename != self.__user_instances[user_id].get_filenames()[0]: - # user is requesting new file -> close current file and open new one - self.__user_instances[user_id].close_files() - self.__user_instances[user_id] = NuRadioRecoio.NuRadioRecoio(filename) - return self.__user_instances[user_id] diff --git a/NuRadioReco/eventbrowser/dataprovider_root.py b/NuRadioReco/eventbrowser/dataprovider_root.py deleted file mode 100644 index fedb8a57f..000000000 --- a/NuRadioReco/eventbrowser/dataprovider_root.py +++ /dev/null @@ -1,26 +0,0 @@ -import NuRadioReco.modules.io.rno_g.rnogDataReader - - -class DataProviderRoot(object): - __instance = None - - def __new__(cls): - if DataProviderRoot.__instance is None: - DataProviderRoot.__instance = object.__new__(cls) - return DataProviderRoot.__instance - - def __init__(self): - self.__user_instances = {} - - def get_file_handler(self, user_id, filename): - if filename is None: - return - if user_id not in self.__user_instances: - self.__user_instances[user_id] = NuRadioReco.modules.io.rno_g.rnogDataReader.RNOGDataReader([filename]) - - if filename != self.__user_instances[user_id].get_filenames()[0]: - # user is requesting new file -> close current file and open new one - self.__user_instances[user_id] = NuRadioReco.modules.io.rno_g.rnogDataReader.RNOGDataReader([filename]) - #TODO begin method does not exist in RNOGDataReader - #self.__user_instances[user_id].begin(filename) - return self.__user_instances[user_id] diff --git a/NuRadioReco/eventbrowser/index.py b/NuRadioReco/eventbrowser/index.py index ad6242136..824ca76fd 100644 --- a/NuRadioReco/eventbrowser/index.py +++ b/NuRadioReco/eventbrowser/index.py @@ -15,7 +15,6 @@ import os import argparse import NuRadioReco.eventbrowser.dataprovider -import NuRadioReco.eventbrowser.dataprovider_root import logging import webbrowser from NuRadioReco.modules.base import module @@ -196,7 +195,7 @@ def get_page_content(selection): Input('event-info-run', 'value')], [State('user_id', 'children')] ) -def set_event_number(next_evt_click_timestamp, prev_evt_click_timestamp, event_id, j_plot_click_info, filename, +def set_event_number(next_evt_click_timestamp, prev_evt_click_timestamp, event_id, j_plot_click_info, filename, i_event, run_number, juser_id): context = dash.callback_context if filename is None: @@ -271,9 +270,9 @@ def update_slider_marks(filename, juser_id): step_size = int(np.power(10., int(np.log10(n_events)))) marks = {} for i in range(0, n_events, step_size): - marks[i] = str(i) + marks[int(i)] = str(i) if n_events % step_size != 0: - marks[n_events] = str(n_events) + marks[int(n_events)] = str(n_events) return marks @@ -424,9 +423,11 @@ def update_event_info_time(event_i, filename, station_id, juser_id): if __name__ == '__main__': - if int(dash.__version__.split('.')[0]) < 2: - print( - 'WARNING: Dash version 2.0.0 or newer is required, you are running version {}. Please update.'.format( + dash_version = [int(i) for i in dash.__version__.split('.')] + if dash_version[0] <= 2: + if (dash_version[1] < 9) or (dash_version[0] < 2): + print( + 'WARNING: Dash version 2.9.2 or newer is required, you are running version {}. Please update.'.format( dash.__version__)) if not parsed_args.debug: werkzeug_logger = logging.getLogger('werkzeug') diff --git a/NuRadioReco/examples/AliasPhasedArray/SNR_study/T01_generate_events_simple.py b/NuRadioReco/examples/AliasPhasedArray/SNR_study/T01_generate_events_simple.py index 9d0947086..8cc768dd2 100644 --- a/NuRadioReco/examples/AliasPhasedArray/SNR_study/T01_generate_events_simple.py +++ b/NuRadioReco/examples/AliasPhasedArray/SNR_study/T01_generate_events_simple.py @@ -47,7 +47,7 @@ vertex_angle_max = 145 * units.deg vertex_angles = np.random.uniform(vertex_angle_min, vertex_angle_max, n_events) -data_sets["yy"] = np.zeros(n_events, dtype=np.float) +data_sets["yy"] = np.zeros(n_events, dtype=float) data_sets["zz"] = z_ant + distance * np.cos(vertex_angles) data_sets["xx"] = distance * np.sin(vertex_angles) @@ -59,7 +59,7 @@ data_sets["event_group_ids"] = np.arange(n_events) data_sets["shower_ids"] = np.arange(n_events) data_sets["n_interaction"] = np.ones(n_events, dtype=int) -data_sets["vertex_times"] = np.zeros(n_events, dtype=np.float) +data_sets["vertex_times"] = np.zeros(n_events, dtype=float) data_sets["flavors"] = np.ones(n_events, dtype=int) * flavor data_sets["energies"] = np.ones(n_events, dtype=int) * energy data_sets["interaction_type"] = ['cc'] * n_events diff --git a/NuRadioReco/examples/RNO_data/read_data_example/read_rnog.py b/NuRadioReco/examples/RNO_data/read_data_example/read_rnog.py new file mode 100644 index 000000000..2855cbe7e --- /dev/null +++ b/NuRadioReco/examples/RNO_data/read_data_example/read_rnog.py @@ -0,0 +1,62 @@ +from NuRadioReco.modules.io.RNO_G import readRNOGDataMattak +from NuRadioReco.modules.io import eventWriter +from NuRadioReco.utilities import units + +import sys +import logging + +""" read in data """ +list_of_root_files = sys.argv[1:-1] +output_filename = sys.argv[-1] + +rnog_reader = readRNOGDataMattak.readRNOGData(log_level=logging.DEBUG) +writer = eventWriter.eventWriter() + +""" +With a selector you can select or reject events based on information in the +Mattak class EventInfo. See https://github.com/RNO-G/mattak/blob/main/py/mattak/Dataset.py + +class EventInfo: + eventNumber: int + station : int + run: int + readoutTime : float + triggerTime : float + triggerType: str + sysclk: int + sysclkLastPPS: Tuple[int, int] # the last 2 PPS sysclks, most recent first + pps: int + radiantStartWindows: numpy.ndarray + sampleRate: float # Sample rate, in GSa/s +""" + +# The following selector selects only events with a forced trigger. +selectors = [lambda einfo: einfo.triggerType == "FORCE"] + +rnog_reader.begin( + list_of_root_files, + selectors=selectors, + # Currently false because Mattak does not contain calibrated data yet + read_calibrated_data=False, + # Only used when read_calibrated_data==False, performs a simple baseline subtraction each 128 bins + apply_baseline_correction=True, + # Only used when read_calibrated_data==False, performs a linear voltage calibration with hardcoded values + convert_to_voltage=True, + # Can be used instead of defining a selector (only for triggers) + select_triggers=None, + # If true, and if the RunTable database is available select runs based on the following criteria + select_runs=True, + # Only use runs of a certain run type + run_types=["physics"], + # Only use runs with a maximum trigger rate of 1 Hz + max_trigger_rate=1 * units.Hz) + +writer.begin(filename=output_filename) + +for i_event, event in enumerate(rnog_reader.run()): + writer.run(event) + +rnog_reader.end() +writer.end() + + diff --git a/NuRadioReco/examples/RNO_data/read_data_example/run_reconstruction.py b/NuRadioReco/examples/RNO_data/read_data_example/run_reconstruction.py index fbe26c1bb..10fe88283 100644 --- a/NuRadioReco/examples/RNO_data/read_data_example/run_reconstruction.py +++ b/NuRadioReco/examples/RNO_data/read_data_example/run_reconstruction.py @@ -1,6 +1,6 @@ import NuRadioReco import matplotlib.pyplot as plt -from NuRadioReco.modules.io.rno_g.readRNOGData import readRNOGData +from NuRadioReco.modules.io.RNO_G.readRNOGDataMattak import readRNOGData import pandas as pd import numpy as np from NuRadioReco.utilities import units @@ -37,7 +37,7 @@ list_of_root_files = ['pulser_data_21.root'] -readRNOGData = NuRadioReco.modules.io.rno_g.readRNOGData.readRNOGData() +readRNOGData = NuRadioReco.modules.io.RNO_G.readRNOGDataMattak.readRNOGData() readRNOGData.begin(list_of_root_files) for i_event, event in enumerate(readRNOGData.run()): @@ -61,8 +61,8 @@ fig.tight_layout() fig.savefig("trace_{}.pdf".format(i_event)) - - + + diff --git a/NuRadioReco/examples/RNO_data/run_reconstruction.py b/NuRadioReco/examples/RNO_data/run_reconstruction.py new file mode 100644 index 000000000..9dde74415 --- /dev/null +++ b/NuRadioReco/examples/RNO_data/run_reconstruction.py @@ -0,0 +1,71 @@ +import NuRadioReco +import matplotlib.pyplot as plt +from NuRadioReco.modules.io.rno_g.readRNOGData import readRNOGData +import pandas as pd +import numpy as np +from NuRadioReco.utilities import units +from NuRadioReco.modules import channelBandPassFilter +from NuRadioReco.detector import detector +import datetime +from NuRadioReco.modules import sphericalWaveFitter +from NuRadioReco.modules import channelAddCableDelay + +""" An example to show how to read RNO-G data and how to perform simple reconstructions. The data used is pulser data and a simple (brute force, not optimized) spherical wave reconstruction is performed to obtain the pulser position """"" + +""" Initiazize modules needed""" + +channelBandPassFilter = NuRadioReco.modules.channelBandPassFilter.channelBandPassFilter() +sphericalWaveFitter = NuRadioReco.modules.sphericalWaveFitter.sphericalWaveFitter() +channelAddCableDelay = NuRadioReco.modules.channelAddCableDelay.channelAddCableDelay() + + +use_channels = [10, 0, 23] +sphericalWaveFitter.begin(channel_ids = use_channels) + +""" Specify the detector. """ + +det = detector.Detector(json_filename = "detector_description.json") +det.update(datetime.datetime(2022, 10, 1)) + +""" Get positions for the pulsers from the detector file as starting positions for the fit """ + +station_id = 21 +pulser_id = 3 #Helper string C +pulser_position = det.get_relative_position(station_id, pulser_id, mode = 'device') +rel_pulser_position = [pulser_position[0], pulser_position[1], pulser_position[2]] + + +plots = True +""" read in data """ +list_of_root_files = ['pulser_data_21.root'] + + +readRNOGData = NuRadioReco.modules.io.rno_g.readRNOGData.readRNOGData() +readRNOGData.begin(list_of_root_files) + +for i_event, event in enumerate(readRNOGData.run()): + print("reconstruction for event", i_event) + station_id = event.get_station_ids()[0] + station = event.get_station(station_id) + channelAddCableDelay.run(event, station, det, mode = 'subtract') + channelBandPassFilter.run(event, station, det, passband = [5*units.MHz, 450*units.MHz]) + sphericalWaveFitter.run(event, station, det, start_pulser_position = rel_pulser_position, n_index = 1.78, debug =True) + + if plots: + fig = plt.figure() + i = 1 + for channel in station.iter_channels(): + if channel.get_id() in use_channels: + ax = fig.add_subplot(3, 2, i) + ax.plot(channel.get_trace()) + ax.title.set_text("channel id {}".format(channel.get_id())) + ax.grid() + i+= 1 + + fig.tight_layout() + fig.savefig("trace_{}.pdf".format(i_event)) + + + + + diff --git a/NuRadioReco/examples/cr_efficiency_analysis/helper_cr_eff.py b/NuRadioReco/examples/cr_efficiency_analysis/helper_cr_eff.py index 244f1a916..0545f2954 100644 --- a/NuRadioReco/examples/cr_efficiency_analysis/helper_cr_eff.py +++ b/NuRadioReco/examples/cr_efficiency_analysis/helper_cr_eff.py @@ -13,9 +13,9 @@ class NumpyEncoder(json.JSONEncoder): """ Special json encoder for numpy types """ def default(self, obj): - if isinstance(obj, np.integer): + if isinstance(obj, int): return int(obj) - elif isinstance(obj, np.floating): + elif isinstance(obj, float): return float(obj) elif isinstance(obj, np.ndarray): return obj.tolist() diff --git a/NuRadioReco/framework/base_station.py b/NuRadioReco/framework/base_station.py index 8ef4ef97a..d1d671e9b 100644 --- a/NuRadioReco/framework/base_station.py +++ b/NuRadioReco/framework/base_station.py @@ -75,22 +75,57 @@ def remove_parameter(self, key): raise ValueError("parameter key needs to be of type NuRadioReco.framework.parameters.stationParameters") self._parameters.pop(key, None) - def set_station_time(self, time): - if time is None: - self._station_time = None - return + def set_station_time(self, time, format=None): + """ + Set the (absolute) time for the station (stored as astropy.time.Time). + Not related to the event._event_time. + + Parameters + ---------- + + time: astropy.time.Time or datetime.datetime or float + If "time" is a float, you have to specify its format. + + format: str + Only used when "time" is a float. Format to interpret "time". (Default: None) + """ + if isinstance(time, datetime.datetime): - time_strings = str(time).split(' ') - self._station_time = astropy.time.Time('{}T{}'.format(time_strings[0], time_strings[1]), format='isot') + self._station_time = astropy.time.Time(time) + elif isinstance(time, astropy.time.Time): + self._station_time = time + elif time is None: + self._station_time = None else: - if time.format == 'datetime': - time_strings = str(time).split(' ') - self._station_time = astropy.time.Time('{}T{}'.format(time_strings[0], time_strings[1]), format='isot') - else: - self._station_time = time + self._station_time = astropy.time.Time(time, format=format) - def get_station_time(self): + def get_station_time(self, format='isot'): + """ + Returns a astropy.time.Time object + + Parameters + ---------- + + format: str + Format in which the time object is displayed. (Default: isot) + + Returns + ------- + + _station_time: astropy.time.Time + """ + if self._station_time is None: + return None + + self._station_time.format = format return self._station_time + + def get_station_time_dict(self): + """ Return the station time as dict {value, format}. Used for reading and writing """ + if self._station_time is None: + return None + else: + return {'value': self._station_time.value, 'format': self._station_time.format} def get_id(self): return self._station_id @@ -235,16 +270,13 @@ def serialize(self, save_efield_traces): trigger_pkls = [] for trigger in self._triggers.values(): trigger_pkls.append(trigger.serialize()) + efield_pkls = [] for efield in self.get_electric_fields(): efield_pkls.append(efield.serialize(save_trace=save_efield_traces)) - if self._station_time is None: - station_time_dict = None - else: - station_time_dict = { - 'value': self._station_time.value, - 'format': self._station_time.format - } + + station_time_dict = self.get_station_time_dict() + data = {'_parameters': NuRadioReco.framework.parameter_serialization.serialize(self._parameters), '_parameter_covariances': NuRadioReco.framework.parameter_serialization.serialize_covariances(self._parameter_covariances), '_ARIANNA_parameters': self._ARIANNA_parameters, @@ -254,15 +286,16 @@ def serialize(self, save_efield_traces): 'triggers': trigger_pkls, '_triggered': self._triggered, 'electric_fields': efield_pkls} + return pickle.dumps(data, protocol=4) def deserialize(self, data_pkl): data = pickle.loads(data_pkl) - if ('triggers' in data): + if 'triggers' in data: self._triggers = NuRadioReco.framework.trigger.deserialize(data['triggers']) - if ('triggers' in data): + if 'triggers' in data: self._triggered = data['_triggered'] for electric_field in data['electric_fields']: @@ -273,8 +306,10 @@ def deserialize(self, data_pkl): self._parameters = NuRadioReco.framework.parameter_serialization.deserialize(data['_parameters'], parameters.stationParameters) - self._parameter_covariances = NuRadioReco.framework.parameter_serialization.deserialize_covariances(data['_parameter_covariances'], parameters.stationParameters) - if ('_ARIANNA_parameters') in data: + self._parameter_covariances = NuRadioReco.framework.parameter_serialization.deserialize_covariances( + data['_parameter_covariances'], parameters.stationParameters) + + if '_ARIANNA_parameters' in data: self._ARIANNA_parameters = data['_ARIANNA_parameters'] self._station_id = data['_station_id'] @@ -285,4 +320,5 @@ def deserialize(self, data_pkl): # For backward compatibility, we also keep supporting station times stored as astropy.time objects else: self.set_station_time(data['_station_time']) + self._particle_type = data['_particle_type'] diff --git a/NuRadioReco/framework/base_trace.py b/NuRadioReco/framework/base_trace.py index 0b735967f..ee3e5443f 100644 --- a/NuRadioReco/framework/base_trace.py +++ b/NuRadioReco/framework/base_trace.py @@ -36,7 +36,7 @@ def get_trace(self): trace: np.array of floats the time trace """ - if(not self.__time_domain_up_to_date): + if not self.__time_domain_up_to_date: self._time_trace = fft.freq2time(self._frequency_spectrum, self._sampling_rate) self.__time_domain_up_to_date = True self._frequency_spectrum = None @@ -62,10 +62,10 @@ def get_filtered_trace(self, passband, filter_type='butter', order=10, rp=None): return fft.freq2time(spec, self.get_sampling_rate()) def get_frequency_spectrum(self): - if(self.__time_domain_up_to_date): + if self.__time_domain_up_to_date: self._frequency_spectrum = fft.time2freq(self._time_trace, self._sampling_rate) self._time_trace = None -# logger.debug("frequency spectrum has shape {}".format(self._frequency_spectrum.shape)) + # logger.debug("frequency spectrum has shape {}".format(self._frequency_spectrum.shape)) self.__time_domain_up_to_date = False return np.copy(self._frequency_spectrum) @@ -82,7 +82,8 @@ def set_trace(self, trace, sampling_rate): """ if trace is not None: if trace.shape[trace.ndim - 1] % 2 != 0: - raise ValueError('Attempted to set trace with an uneven number ({}) of samples. Only traces with an even number of samples are allowed.'.format(trace.shape[trace.ndim - 1])) + raise ValueError(('Attempted to set trace with an uneven number ({}) of samples. ' + 'Only traces with an even number of samples are allowed.').format(trace.shape[trace.ndim - 1])) self.__time_domain_up_to_date = True self._time_trace = np.copy(trace) self._sampling_rate = sampling_rate @@ -109,10 +110,11 @@ def get_times(self): try: length = self.get_number_of_samples() times = np.arange(0, length / self._sampling_rate - 0.1 / self._sampling_rate, 1. / self._sampling_rate) + self._trace_start_time - if(len(times) != length): - logger.error("time array does not have the same length as the trace. n_samples = {:d}, sampling rate = {:.5g}".format(length, self._sampling_rate)) - raise ValueError("time array does not have the same length as the trace") - except: + if len(times) != length: + err = f"time array does not have the same length as the trace. n_samples = {length:d}, sampling rate = {self._sampling_rate:.5g}" + logger.error(err) + raise ValueError(err) + except (ValueError, AttributeError): times = np.array([]) return times @@ -148,7 +150,7 @@ def get_number_of_samples(self): n_samples: int number of samples in time domain """ - if(self.__time_domain_up_to_date): + if self.__time_domain_up_to_date: length = self._time_trace.shape[-1] # returns the correct length independent of the dimension of the array (channels are 1dim, efields are 3dim) else: length = (self._frequency_spectrum.shape[-1] - 1) * 2 @@ -184,24 +186,30 @@ def resample(self, sampling_rate): if resampling_factor.numerator != 1: # resample and use axis -1 since trace might be either shape (N) for analytic trace or shape (3,N) for E-field resampled_trace = scipy.signal.resample(resampled_trace, resampling_factor.numerator * self.get_number_of_samples(), axis=-1) + if resampling_factor.denominator != 1: # resample and use axis -1 since trace might be either shape (N) for analytic trace or shape (3,N) for E-field resampled_trace = scipy.signal.resample(resampled_trace, np.shape(resampled_trace)[-1] // resampling_factor.denominator, axis=-1) if resampled_trace.shape[-1] % 2 != 0: resampled_trace = resampled_trace.T[:-1].T + self.set_trace(resampled_trace, sampling_rate) def serialize(self): + time_trace = self.get_trace() + # if there is no trace, the above will return np.array(None). + if not time_trace.shape: + return None data = {'sampling_rate': self.get_sampling_rate(), - 'time_trace': self.get_trace(), + 'time_trace': time_trace, 'trace_start_time': self.get_trace_start_time()} return pickle.dumps(data, protocol=4) def deserialize(self, data_pkl): data = pickle.loads(data_pkl) self.set_trace(data['time_trace'], data['sampling_rate']) - if('trace_start_time' in data.keys()): + if 'trace_start_time' in data.keys(): self.set_trace_start_time(data['trace_start_time']) def __add__(self, x): @@ -214,10 +222,13 @@ def __add__(self, x): # Some sanity checks if not isinstance(x, BaseTrace): raise TypeError('+ operator is only defined for 2 BaseTrace objects') + if self.get_trace() is None or x.get_trace() is None: raise ValueError('One of the trace objects has no trace set') + if self.get_trace().ndim != x.get_trace().ndim: raise ValueError('Traces have different dimensions') + if self.get_sampling_rate() != x.get_sampling_rate(): # Upsample trace with lower sampling rate # Create new baseTrace object for the resampling so we don't change the originals @@ -249,10 +260,12 @@ def __add__(self, x): first_trace = trace_2 second_trace = trace_1 trace_start = x.get_trace_start_time() + # Calculate the difference in the trace start time between the traces and the number of # samples that time difference corresponds to time_offset = np.abs(x.get_trace_start_time() - self.get_trace_start_time()) i_start = int(round(time_offset * sampling_rate)) + # We have to distinguish 2 cases: Trace is 1D (channel) or 2D(E-field) # and treat them differently if trace_1.ndim == 1: @@ -273,11 +286,13 @@ def __add__(self, x): early_trace[:, :first_trace.shape[1]] = first_trace late_trace = np.zeros((second_trace.shape[0], trace_length)) late_trace[:, :second_trace.shape[1]] = second_trace + # Correct for different trace start times by using fourier shift theorem to # shift the later trace backwards. late_trace_object = BaseTrace() late_trace_object.set_trace(late_trace, sampling_rate) late_trace_object.apply_time_shift(time_offset, True) + # Create new BaseTrace object holding the summed traces new_trace = BaseTrace() new_trace.set_trace(early_trace + late_trace_object.get_trace(), sampling_rate) diff --git a/NuRadioReco/framework/event.py b/NuRadioReco/framework/event.py index 80530be6c..1d4c06128 100644 --- a/NuRadioReco/framework/event.py +++ b/NuRadioReco/framework/event.py @@ -41,7 +41,7 @@ def register_module_event(self, instance, name, kwargs): kwargs: the key word arguments of the run method """ - + self.__modules_event.append([name, instance, kwargs]) def register_module_station(self, station_id, instance, name, kwargs): @@ -59,8 +59,9 @@ def register_module_station(self, station_id, instance, name, kwargs): kwargs: the key word arguments of the run method """ - if(station_id not in self.__modules_station): + if station_id not in self.__modules_station: self.__modules_station[station_id] = [] + iE = len(self.__modules_event) self.__modules_station[station_id].append([iE, name, instance, kwargs]) @@ -427,30 +428,33 @@ def serialize(self, mode): commit_hash = NuRadioReco.utilities.version.get_NuRadioMC_commit_hash() self.set_parameter(parameters.eventParameters.hash_NuRadioMC, commit_hash) except: + logger.warning("Event is serialized without commit hash!") self.set_parameter(parameters.eventParameters.hash_NuRadioMC, None) for station in self.get_stations(): stations_pkl.append(station.serialize(mode)) - showers_pkl = [] - for shower in self.get_showers(): - showers_pkl.append(shower.serialize()) - sim_showers_pkl = [] - for shower in self.get_sim_showers(): - sim_showers_pkl.append(shower.serialize()) - particles_pkl = [] - for particle in self.get_particles(): - particles_pkl.append(particle.serialize()) + showers_pkl = [shower.serialize() for shower in self.get_showers()] + sim_showers_pkl = [shower.serialize() for shower in self.get_sim_showers()] + particles_pkl = [particle.serialize() for particle in self.get_particles()] + hybrid_info = self.__hybrid_information.serialize() + modules_out_event = [] for value in self.__modules_event: # remove module instances (this will just blow up the file size) modules_out_event.append([value[0], None, value[2]]) + invalid_keys = [key for key,val in value[2].items() if isinstance(val, BaseException)] + if len(invalid_keys): + logger.warning(f"The following arguments to module {value[0]} could not be serialized and will not be stored: {invalid_keys}") modules_out_station = {} for key in self.__modules_station: # remove module instances (this will just blow up the file size) modules_out_station[key] = [] for value in self.__modules_station[key]: modules_out_station[key].append([value[0], value[1], None, value[3]]) + invalid_keys = [key for key,val in value[3].items() if isinstance(val, BaseException)] + if len(invalid_keys): + logger.warning(f"The following arguments to module {value[0]} could not be serialized and will not be stored: {invalid_keys}") data = {'_parameters': self._parameters, '__run_number': self.__run_number, @@ -489,9 +493,11 @@ def deserialize(self, data_pkl): particle = NuRadioReco.framework.particle.Particle(None) particle.deserialize(particle_pkl) self.add_particle(particle) + self.__hybrid_information = NuRadioReco.framework.hybrid_information.HybridInformation() if 'hybrid_info' in data.keys(): self.__hybrid_information.deserialize(data['hybrid_info']) + self._parameters = data['_parameters'] self.__run_number = data['__run_number'] self._id = data['_id'] @@ -500,7 +506,7 @@ def deserialize(self, data_pkl): if 'generator_info' in data.keys(): self._generator_info = data['generator_info'] - if("__modules_event" in data): + if "__modules_event" in data: self.__modules_event = data['__modules_event'] - if("__modules_station" in data): + if "__modules_station" in data: self.__modules_station = data['__modules_station'] diff --git a/NuRadioReco/framework/parameters.py b/NuRadioReco/framework/parameters.py index 575d7b402..b234ec112 100644 --- a/NuRadioReco/framework/parameters.py +++ b/NuRadioReco/framework/parameters.py @@ -57,6 +57,7 @@ class channelParameters(Enum): signal_receiving_zenith = 15 #: the zenith angle of direction at which the radio signal arrived at the antenna signal_ray_type = 16 #: type of the ray propagation path of the signal received by this channel. Options are direct, reflected and refracted signal_receiving_azimuth = 17 #: the azimuth angle of direction at which the radio signal arrived at the antenna + block_offsets = 18 #: 'block' or pedestal offsets. See `NuRadioReco.modules.RNO_G.channelBlockOffsetFitter` class electricFieldParameters(Enum): diff --git a/NuRadioReco/framework/trigger.py b/NuRadioReco/framework/trigger.py index 1792f0916..cde481bc6 100644 --- a/NuRadioReco/framework/trigger.py +++ b/NuRadioReco/framework/trigger.py @@ -213,7 +213,10 @@ class SimplePhasedTrigger(Trigger): def __init__(self, name, threshold, channels=None, secondary_channels=None, primary_angles=None, secondary_angles=None, - trigger_delays=None, sec_trigger_delays=None): + trigger_delays=None, sec_trigger_delays=None, + window_size=None, step_size=None, + maximum_amps=None + ): """ initialize trigger class @@ -238,6 +241,12 @@ def __init__(self, name, threshold, channels=None, secondary_channels=None, sec_trigger_delays: dictionary the delays for the secondary channels that have caused a trigger. If there is no trigger or no secondary channels, it's an empty dictionary + window_size: int + the size of the integration window (units of ADC time ticks) + step_size: int + the size of the stride between calculating the phasing (units of ADC time ticks) + maximum_amps: list of floats (length equal to that of `phasing_angles`) + the maximum value of all the integration windows for each of the phased waveforms """ Trigger.__init__(self, name, channels, 'simple_phased') self._primary_channels = channels @@ -247,6 +256,9 @@ def __init__(self, name, threshold, channels=None, secondary_channels=None, self._threshold = threshold self._trigger_delays = trigger_delays self._sec_trigger_delays = sec_trigger_delays + self._window_size = window_size + self._step_side = step_size + self._maximum_amps = maximum_amps class HighLowTrigger(Trigger): diff --git a/NuRadioReco/modules/RNO_G/channelBlockOffsetFitter.py b/NuRadioReco/modules/RNO_G/channelBlockOffsetFitter.py new file mode 100644 index 000000000..e4711e094 --- /dev/null +++ b/NuRadioReco/modules/RNO_G/channelBlockOffsetFitter.py @@ -0,0 +1,263 @@ +""" +Module to remove 'block offsets' from RNO-G voltage traces. + +The function ``fit_block_offsets`` can be used standalone to perform an out-of-band +fit to the block offsets. Alternatively, the ``channelBlockOffsets`` class contains convenience +``add_offsets`` (to add block offsets in simulation) and ``remove_offsets`` methods that can be run +directly on a NuRadioMC/imported ``Event``. The added/removed block offsets are stored per channel +in the `NuRadioReco.framework.parameters.channelParameters.block_offsets` parameter. + +""" + +from NuRadioReco.utilities import units, fft +from NuRadioReco.framework.base_trace import BaseTrace +from NuRadioReco.framework.parameters import channelParameters +import numpy as np +import scipy.optimize + +class channelBlockOffsets: + + def __init__(self, block_size=128, max_frequency=51*units.MHz): + """ + Add or remove block offsets to channel traces + + This module adds, fits or removes 'block offsets' by fitting + them in a specified out-of-band region in frequency space. + + Parameters + ---------- + block_size: int (default: 128) + The size (in samples) of the blocks + max_frequency: float (default: 51 MHz) + The maximum frequency to include in the out-of-band + block offset fit + + """ + self.sampling_rate = None + self.block_size = block_size # the size (in samples) of the blocks + self._offset_fit = dict() + self._offset_inject = dict() + self._max_frequency = max_frequency + + def add_offsets(self, event, station, offsets=1*units.mV, channel_ids=None): + """ + Add (simulated or reconstructed) block offsets to an event. + + Added block offsets for each channel are stored in the + ``channelParameters.block_offsets`` parameter. + + Parameters + ---------- + event: Event object | None + station: Station + The station to add block offsets to + offsets: float | array | dict + offsets to add to the event. Default: 1 mV + + - if a float, add gaussian-distributed of amplitude ``offsets`` + to all channels specified; + - if an array, the length should be the same as the number + of blocks in a single trace, and the entries will be + interpreted as the amplitudes of the offsets; + - if a dict, the keys should be the channel ids, and each + value should contain either a float or an array to add to + each channel as specified above. + + channel_ids: list | None + either a list of channel ids to apply the offsets to, or + None to apply the offsets to all channels in the station + (default: None). + + """ + + if channel_ids is None: + channel_ids = station.get_channel_ids() + for channel_id in channel_ids: + channel = station.get_channel(channel_id) + if isinstance(offsets, dict): + add_offsets = offsets[channel_id] + else: + add_offsets = offsets + if len(np.atleast_1d(add_offsets)) == 1: + add_offsets = np.random.normal( + 0, add_offsets, (channel.get_number_of_samples() // self.block_size) + ) + + # save the added offsets as a channelParameter + if channel.has_parameter(channelParameters.block_offsets): + block_offsets_old = channel.get_parameter(channelParameters.block_offsets) + channel.set_parameter(channelParameters.block_offsets, block_offsets_old + offsets) + else: + channel.set_parameter(channelParameters.block_offsets, offsets) + + channel.set_trace( + channel.get_trace() + np.repeat(add_offsets, self.block_size), + channel.get_sampling_rate() + ) + + def remove_offsets(self, event, station, mode='fit', channel_ids=None, maxiter=5): + """ + Remove block offsets from an event + + Fits and removes the block offsets from an event. The removed + offsets are stored in the ``channelParameters.block_offsets`` + parameter. + + Parameters + ---------- + event: NuRadioReco.framework.event.Event | None + station: NuRadioReco.framework.station.Station + The station to remove the block offsets from + mode: 'fit' | 'approximate' | 'stored' + + - 'fit' (default): fit the block offsets with a minimizer + - 'approximate' : use the first guess from the out-of-band component, + without any fitting (slightly faster) + - 'stored': use the block offsets already stored in the + ``channelParameters.block_offsets`` parameter. Will raise an error + if this parameter is not present. + + channel_ids: list | None + List of channel ids to remove offsets from. If None (default), + remove offsets from all channels in ``station`` + maxiter: int, default 5 + (Only if mode=='fit') The maximum number of fit iterations. + This can be increased to more accurately remove the block offsets + at the cost of performance. (The default value removes 'most' offsets + to about 1%) + + """ + if channel_ids is None: + channel_ids = station.get_channel_ids() + + offsets = {} + if mode == 'stored': # remove offsets stored in channelParameters.block_offsets + offsets = { + channel_id: -station.get_channel(channel_id).get_parameter(channelParameters.block_offsets) + for channel_id in channel_ids} + else: # fit & remove offsets + for channel_id in channel_ids: + channel = station.get_channel(channel_id) + trace = channel.get_trace() + + block_offsets = fit_block_offsets( + trace, self.block_size, + channel.get_sampling_rate(), self._max_frequency, + mode=mode, maxiter=maxiter + ) + offsets[channel_id] = -block_offsets + + self.add_offsets(event, station, offsets, channel_ids) + + +def fit_block_offsets( + trace, block_size=128, sampling_rate=3.2*units.GHz, + max_frequency=50*units.MHz, mode='fit', return_trace = False, + maxiter=5, tol=1e-6): + """ + Fit 'block' offsets for a voltage trace + + Fit block offsets ('rect'-shaped offsets from a baseline) + using a fit to the out-of-band spectrum of a voltage trace. + + Parameters + ---------- + trace: numpy Array + the voltage trace + block_size: int (default: 128) + the number of samples in one block + sampling_rate: float (default: 3.2 GHz) + the sampling rate of the trace + max_frequency: float (default: 50 MHz) + the fit to the block offsets is performed + in the frequency domain, in the band up to + max_frequency + mode: 'fit' | 'approximate' + Whether to fit the block offsets (default) + or just use the first guess from the out-of-band + component (faster) + return_trace: bool (default: False) + if True, return the tuple (offsets, output_trace) + where the output_trace is the input trace with + fitted block offsets removed + maxiter: int (default: 5) + (Only if mode=='fit') The maximum number of fit iterations. + This can be increased to more accurately remove the block offsets + at the cost of performance. (The default value removes 'most' offsets + to about 1%) + + Returns + ------- + block_offsets: numpy array + The fitted block offsets. + output_trace: numpy array or None + The input trace with the fitted block offsets removed. + Returned only if return_trace=True + + Other Parameters + ---------------- + tol: float (default: 1e-6) + tolerance parameter passed on to scipy.optimize.minimize + """ + dt = 1. / sampling_rate + spectrum = fft.time2freq(trace, sampling_rate) + frequencies = np.fft.rfftfreq(len(trace), dt) + n_blocks = len(trace) // block_size + + mask = (frequencies > 0) & (frequencies < max_frequency) # a simple rectangular filter + frequencies_oob = frequencies[mask] + spectrum_oob = spectrum[mask] + + # we use the bandpass-filtered trace to get a first estimate of + # the block offsets, by simply averaging over each block. + filtered_trace_fft = np.copy(spectrum) + filtered_trace_fft[~mask] = 0 + filtered_trace = fft.freq2time(filtered_trace_fft, sampling_rate) + + # obtain guesses for block offsets + a_guess = np.mean(np.split(filtered_trace, n_blocks), axis=1) + if mode == 'approximate': + block_offsets = a_guess + elif mode == 'fit': + # self._offset_guess[channel_id] = a_guess + # we can get rid of one parameter through a global shift + a_guess = a_guess[:-1] - a_guess[-1] + + # we perform the fit out-of-band, in order to avoid + # distorting any actual signal + + # most of the terms in the fit depend only on the frequencies, + # sampling rate and number of blocks. We therefore calculate these + # only once, outside the fit function. + pre_factor_exponent = np.array([ + -2.j * np.pi * frequencies_oob * dt * ((j+.5) * block_size - .5) + for j in range(len(a_guess)) + ]) + const_fft_term = ( + 1 / sampling_rate * np.sqrt(2) # NuRadio FFT normalization + * np.exp(pre_factor_exponent) + * np.sin(np.pi*frequencies_oob*block_size*dt)[None] + / np.sin(np.pi*frequencies_oob*dt)[None] + ) + + def pedestal_fit(a): + fit = np.sum(a[:, None] * const_fft_term, axis=0) + chi2 = np.sum(np.abs(fit-spectrum_oob)**2) + return chi2 + + res = scipy.optimize.minimize(pedestal_fit, a_guess, tol=tol, options=dict(maxiter=maxiter)).x + + block_offsets = np.zeros(len(res) + 1) + block_offsets[:-1] = res + + # the fit is not sensitive to an overall shift, + # so we include the zero-meaning here + block_offsets += np.mean(trace) - np.mean(block_offsets) + else: + raise ValueError(f'Invalid value for mode={mode}. Accepted values are {{"fit", "approximate"}}') + + if return_trace: + output_trace = trace - np.repeat(block_offsets, block_size) + return block_offsets, output_trace + + return block_offsets \ No newline at end of file diff --git a/NuRadioReco/modules/analogToDigitalConverter.py b/NuRadioReco/modules/analogToDigitalConverter.py index 37b229054..5fcd9b30d 100644 --- a/NuRadioReco/modules/analogToDigitalConverter.py +++ b/NuRadioReco/modules/analogToDigitalConverter.py @@ -50,7 +50,7 @@ def perfect_comparator(trace, adc_n_bits, adc_ref_voltage, mode='floor', output= digital_trace = round_to_int(digital_trace) if (output == 'voltage'): - digital_trace = lsb_voltage * digital_trace.astype(np.float) + digital_trace = lsb_voltage * digital_trace.astype(float) elif (output == 'counts'): pass else: @@ -296,7 +296,7 @@ def get_digital_trace(self, station, det, channel, if trigger_filter is not None: trace_fft = np.fft.rfft(trace) - if(len(trace_fft) != trigger_filter): + if len(trace_fft) != len(trigger_filter): raise ValueError("Wrong filter length to apply to traces") trace = np.fft.irfft(trace_fft * trigger_filter) diff --git a/NuRadioReco/modules/base/module.py b/NuRadioReco/modules/base/module.py index 50818fa93..fc55c8657 100644 --- a/NuRadioReco/modules/base/module.py +++ b/NuRadioReco/modules/base/module.py @@ -2,8 +2,10 @@ from timeit import default_timer as timer import NuRadioReco.framework.event import NuRadioReco.framework.base_station +import NuRadioReco.detector.detector_base import logging - +import inspect +import pickle def setup_logger(name="NuRadioReco", level=logging.WARNING): @@ -39,43 +41,59 @@ def register_run_method(self, *args, **kwargs): # generator, so not sure how to access the event. evt = None station = None - # find out type of module automatically - if(len(args) == 1): - if(isinstance(args[0], NuRadioReco.framework.event.Event)): - module_level = "event" - evt = args[0] - else: - # this is a module that creats events - module_level = "reader" - elif(len(args) >= 2): - if(isinstance(args[0], NuRadioReco.framework.event.Event) and isinstance(args[1], NuRadioReco.framework.base_station.BaseStation)): - module_level = "station" - evt = args[0] - station = args[1] - elif(isinstance(args[0], NuRadioReco.framework.event.Event)): - module_level = "event" - evt = args[0] - else: - # this is a module that creates events - module_level = "reader" - raise AttributeError("first argument of run method is not of type NuRadioReco.framework.event.Event") + + signature = inspect.signature(run) + parameters = signature.parameters + # convert args to kwargs to facilitate easier bookkeeping + keys = [key for key in parameters.keys() if key != 'self'] + all_kwargs = {key:value for key,value in zip(keys, args)} + all_kwargs.update(kwargs) # this silently overwrites positional args with kwargs, but this is probably okay as we still raise an error later + + # include parameters with default values + for key,value in parameters.items(): + if key not in all_kwargs.keys(): + if value.default is not inspect.Parameter.empty: + all_kwargs[key] = value.default + + store_kwargs = {} + for idx, (key,value) in enumerate(all_kwargs.items()): + if isinstance(value, NuRadioReco.framework.event.Event) and idx == 0: # event should be the first argument + evt = value + elif isinstance(value, NuRadioReco.framework.base_station.BaseStation) and idx == 1: # station should be second argument + station = value + elif isinstance(value, NuRadioReco.detector.detector_base.DetectorBase): + pass # we don't try to store detectors + else: # we try to store other arguments IF they are pickleable + try: + pickle.dumps(value, protocol=4) + store_kwargs[key] = value + except (TypeError, AttributeError): # object couldn't be pickled - we store the error instead + store_kwargs[key] = TypeError(f"Argument of type {type(value)} could not be serialized") + if station is not None: + module_level = "station" + elif evt is not None: + module_level = "event" else: - # this is a module that creats events module_level = "reader" start = timer() - res = run(self, *args, **kwargs) - if(module_level == "event"): - evt.register_module_event(self, self.__class__.__name__, kwargs) - elif(module_level == "station"): - evt.register_module_station(station.get_id(), self, self.__class__.__name__, kwargs) - elif(module_level == "reader"): + + if module_level == "event": + evt.register_module_event(self, self.__class__.__name__, store_kwargs) + elif module_level == "station": + evt.register_module_station(station.get_id(), self, self.__class__.__name__, store_kwargs) + elif module_level == "reader": # not sure what to do... function returns generator, not sure how to access the event... pass + + res = run(self, *args, **kwargs) + end = timer() + if self not in register_run_method.time: # keep track of timing of modules. We use the module instance as key to time different module instances separately. register_run_method.time[self] = 0 register_run_method.time[self] += (end - start) + return res register_run_method.time = {} diff --git a/NuRadioReco/modules/beamFormingDirectionFitter.py b/NuRadioReco/modules/beamFormingDirectionFitter.py index 5cc431d0a..126596baa 100644 --- a/NuRadioReco/modules/beamFormingDirectionFitter.py +++ b/NuRadioReco/modules/beamFormingDirectionFitter.py @@ -38,7 +38,7 @@ def get_array_of_channels(station, det, zenith, azimuth, polarization): time_shifts = np.zeros(8) t_geos = np.zeros(8) - sampling_rate = station.get_channel(0).get_sampling_rate() + sampling_rate = next(station.iter_channels()).get_sampling_rate() station_id = station.get_id() site = det.get_site(station_id) for iCh, channel in enumerate(station.get_electric_fields()): @@ -185,7 +185,7 @@ def ll_regular_station(angles, evt, station, det, polarization, sampling_rate, p positions = [] for chan in channels: positions.append(det.get_relative_position(station_id, chan)) - sampling_rate = station.get_channel(0).get_sampling_rate() + sampling_rate = station.get_channel(channels[0]).get_sampling_rate() ll = opt.brute( ll_regular_station, diff --git a/NuRadioReco/modules/channelGalacticNoiseAdder.py b/NuRadioReco/modules/channelGalacticNoiseAdder.py index 0638fb423..9afafc989 100644 --- a/NuRadioReco/modules/channelGalacticNoiseAdder.py +++ b/NuRadioReco/modules/channelGalacticNoiseAdder.py @@ -122,7 +122,7 @@ def run( passband_filter = (freqs > passband[0]) & (freqs < passband[1]) noise_spec_sum = np.zeros_like(channel.get_frequency_spectrum()) flux_sum = np.zeros(freqs[passband_filter].shape) - efield_sum = np.zeros((3, freqs.shape[0]), dtype=np.complex) + efield_sum = np.zeros((3, freqs.shape[0]), dtype=complex) if self.__debug: plt.close('all') fig = plt.figure(figsize=(12, 8)) @@ -177,7 +177,7 @@ def run( ax_3.plot(freqs[passband_filter] / units.MHz, E / (units.V / units.m), c='k', alpha=.02) # assign random phases and polarizations to electric field - noise_spectrum = np.zeros((3, freqs.shape[0]), dtype=np.complex) + noise_spectrum = np.zeros((3, freqs.shape[0]), dtype=complex) phases = np.random.uniform(0, 2. * np.pi, len(S)) polarizations = np.random.uniform(0, 2. * np.pi, len(S)) diff --git a/NuRadioReco/modules/channelGenericNoiseAdder.py b/NuRadioReco/modules/channelGenericNoiseAdder.py index e8a4acfd9..e5556c02a 100644 --- a/NuRadioReco/modules/channelGenericNoiseAdder.py +++ b/NuRadioReco/modules/channelGenericNoiseAdder.py @@ -2,7 +2,7 @@ from NuRadioReco.modules.base.module import register_run import numpy as np from NuRadioReco.utilities import units, fft -import numpy.random +from numpy.random import Generator, Philox import logging @@ -27,7 +27,7 @@ def add_random_phases(self, amps, n_samples_time_domain): """ amps = np.array(amps, dtype='complex') Np = (n_samples_time_domain - 1) // 2 - phases = self.__random_generator.rand(Np) * 2 * np.pi + phases = self.__random_generator.random(Np) * 2 * np.pi phases = np.cos(phases) + 1j * np.sin(phases) amps[1:Np + 1] *= phases # Note that the last entry of the index slice is f[Np] ! @@ -45,7 +45,7 @@ def fftnoise_fullfft(self, f): """ f = np.array(f, dtype='complex') Np = (len(f) - 1) // 2 - phases = self.__random_generator.rand(Np) * 2 * np.pi + phases = self.__random_generator.random(Np) * 2 * np.pi phases = np.cos(phases) + 1j * np.sin(phases) f[1:Np + 1] *= phases # Note that the last entry of the index slice is f[Np] ! f[-1:-1 - Np:-1] = np.conj(f[1:Np + 1]) @@ -308,7 +308,7 @@ def __init__(self): def begin(self, debug=False, seed=None): self.__debug = debug - self.__random_generator = np.random.RandomState(seed) + self.__random_generator = Generator(Philox(seed)) if debug: self.logger.setLevel(logging.DEBUG) diff --git a/NuRadioReco/modules/channelMeasuredNoiseAdder.py b/NuRadioReco/modules/channelMeasuredNoiseAdder.py deleted file mode 100644 index 581a316a3..000000000 --- a/NuRadioReco/modules/channelMeasuredNoiseAdder.py +++ /dev/null @@ -1,144 +0,0 @@ -from NuRadioReco.modules.base.module import register_run -import NuRadioReco.modules.io.NuRadioRecoio -import numpy as np -from NuRadioReco.utilities import units -import numpy.random -import logging -import matplotlib.pyplot as plt - - -class channelMeasuredNoiseAdder: - """ - Module that adds measured noise to channel traces - It does so by reading in a set of .nur files, randomly selecting a forced trigger event - and adding the noise from it to the channel waveforms. - The waveforms from the channels in the noise files need to be at least as long as the - waveforms to which the noise is added, so it is recommended to cut them to the right size - first, for example using the channelLengthAdjuster. - """ - def __init__(self): - self.__filenames = None - self.__io = None - self.__random_state = None - self.__max_iterations = None - self.__debug = None - self.logger = logging.getLogger('NuRadioReco.channelMeasuredNoiseAdder') - self.__noise_data = None - - def begin(self, filenames, random_seed=None, max_iterations=100, debug=False, draw_noise_statistics=False): - """ - Set up module parameters - - Parameters - ---------- - filenames: list of strings - List of .nur files containing the measured noise - random_seed: int, default: None - Seed for the random number generator. By default, no seed is set. - max_iterations: int, default: 100 - The module will pick a random event from the noise files, until a suitable event is found - or until the number of iterations exceeds max_iterations. In that case, an error is thrown. - debug: bool, default: False - Set True to get debug output - draw_noise_statistics: boolean, default: False - If true, the values of all samples is stored and a histogram with noise statistics is drawn - be the end() method - """ - self.__filenames = filenames - self.__io = NuRadioReco.modules.io.NuRadioRecoio.NuRadioRecoio(self.__filenames) - self.__random_state = numpy.random.Generator(numpy.random.Philox(random_seed)) - self.__max_iterations = max_iterations - if debug: - self.logger.setLevel(logging.DEBUG) - self.logger.debug('Reading noise from {} files containing {} events'.format(len(filenames), self.__io.get_n_events())) - if draw_noise_statistics: - self.__noise_data = [] - - @register_run() - def run(self, event, station, det): - """ - Add measured noise to station channels - - Parameters - ---------- - event: event object - station: station object - det: detector description - """ - noise_station = None - for i in range(self.__max_iterations): - noise_station = self.get_noise_station(station) - # Get random station from noise file. If we got a suitable station, we continue, - # otherwise we try again - if noise_station is not None: - break - # To avoid infinite loops, if no suitable noise station was found after a number of iterations we raise an error - if noise_station is None: - raise ValueError('Could not find suitable noise event in noise files after {} iterations'.format(self.__max_iterations)) - for channel in station.iter_channels(): - noise_channel = noise_station.get_channel(channel.get_id()) - channel_trace = channel.get_trace() - if noise_channel.get_sampling_rate() != channel.get_sampling_rate(): - noise_channel.resample(channel.get_sampling_rate()) - noise_trace = noise_channel.get_trace() - channel_trace += noise_trace[:channel.get_number_of_samples()] - if self.__noise_data is not None: - self.__noise_data.append(noise_trace) - - def get_noise_station(self, station): - """ - Returns a random station from the noise files that can be used as a noise sample. - The function selects a random event from the noise files and checks if it is suitable. - If it is, the station is returned, otherwise None is returned. The event is suitable if it - fulfills these criteria: - - * It contains a station with the same station ID as the one to which the noise shall be added - * The station does not have a trigger that has triggered. - * The every channel in the station to which the noise shall be added is also present in the station - - Parameters - ---------- - station: Station class - The station to which the noise shall be added - """ - event_i = self.__random_state.randint(self.__io.get_n_events()) - noise_event = self.__io.get_event_i(event_i) - if station.get_id() not in noise_event.get_station_ids(): - self.logger.debug('No station with ID {} found in event'.format(station.get_id())) - return None - noise_station = noise_event.get_station(station.get_id()) - for trigger_name in noise_station.get_triggers(): - trigger = noise_station.get_trigger(trigger_name) - if trigger.has_triggered(): - self.logger.debug('Noise station has triggered') - return None - for channel_id in station.get_channel_ids(): - if channel_id not in noise_station.get_channel_ids(): - self.logger.debug('Channel {} found in station but not in noise file'.format(channel_id)) - return None - noise_channel = noise_station.get_channel(channel_id) - channel = station.get_channel(channel_id) - if noise_channel.get_number_of_samples() / noise_channel.get_sampling_rate() < channel.get_number_of_samples() / channel.get_sampling_rate(): - return None - return noise_station - - def end(self): - """ - End method. Draws a histogram of the noise statistics and fits a - Gaussian distribution to it. - """ - if self.__noise_data is not None: - noise_entries = np.array(self.__noise_data) - noise_bins = np.arange(-150, 150, 5.) * units.mV - noise_entries = noise_entries.flatten() - mean = noise_entries.mean() - sigma = np.sqrt(np.mean((noise_entries - mean)**2)) - plt.close('all') - fig1 = plt.figure() - ax1_1 = fig1.add_subplot(111) - n, bins, pathes = ax1_1.hist(noise_entries / units.mV, bins=noise_bins / units.mV) - ax1_1.plot(noise_bins / units.mV, np.max(n) * np.exp(-.5 * (noise_bins - mean)**2 / sigma**2)) - ax1_1.grid() - ax1_1.set_xlabel('sample value [mV]') - ax1_1.set_ylabel('entries') - plt.show() diff --git a/NuRadioReco/modules/correlationDirectionFitter.py b/NuRadioReco/modules/correlationDirectionFitter.py index 5c6ca4d34..9cb2d8134 100644 --- a/NuRadioReco/modules/correlationDirectionFitter.py +++ b/NuRadioReco/modules/correlationDirectionFitter.py @@ -142,7 +142,7 @@ def ll_regular_station_fft(angles, corr_02_fft, corr_13_fft, sampling_rate, posi station_id = station.get_id() positions_pairs = [[det.get_relative_position(station_id, channel_pairs[0][0]), det.get_relative_position(station_id, channel_pairs[0][1])], [det.get_relative_position(station_id, channel_pairs[1][0]), det.get_relative_position(station_id, channel_pairs[1][1])]] - sampling_rate = station.get_channel(0).get_sampling_rate() # assume that channels have the same sampling rate + sampling_rate = station.get_channel(channel_pairs[0][0]).get_sampling_rate() # assume that channels have the same sampling rate trace_start_time_pairs = [[station.get_channel(channel_pairs[0][0]).get_trace_start_time(), station.get_channel(channel_pairs[0][1]).get_trace_start_time()], [station.get_channel(channel_pairs[1][0]).get_trace_start_time(), station.get_channel(channel_pairs[1][1]).get_trace_start_time()]] # determine automatically if one channel has an inverted waveform with respect to the other diff --git a/NuRadioReco/modules/electricFieldSignalReconstructor.py b/NuRadioReco/modules/electricFieldSignalReconstructor.py index 94252131b..9f4089d2b 100644 --- a/NuRadioReco/modules/electricFieldSignalReconstructor.py +++ b/NuRadioReco/modules/electricFieldSignalReconstructor.py @@ -98,9 +98,9 @@ def run(self, evt, station, det, debug=False): times = electric_field.get_times() mask_signal_window = (times > (signal_time - self.__signal_window_pre)) & (times < (signal_time + self.__signal_window_post)) - mask_noise_window = np.zeros_like(mask_signal_window, dtype=np.bool) + mask_noise_window = np.zeros_like(mask_signal_window, dtype=bool) if(self.__noise_window > 0): - mask_noise_window[int(np.round((-self.__noise_window - 141.) * electric_field.get_sampling_rate())):int(np.round(-141. * electric_field.get_sampling_rate()))] = np.ones(int(np.round(self.__noise_window * electric_field.get_sampling_rate())), dtype=np.bool) # the last n bins + mask_noise_window[int(np.round((-self.__noise_window - 141.) * electric_field.get_sampling_rate())):int(np.round(-141. * electric_field.get_sampling_rate()))] = np.ones(int(np.round(self.__noise_window * electric_field.get_sampling_rate())), dtype=bool) # the last n bins signal_energy_fluence = trace_utilities.get_electric_field_energy_fluence(trace, times, mask_signal_window, mask_noise_window) dt = times[1] - times[0] diff --git a/NuRadioReco/modules/envelope_phasedarray/triggerSimulator.py b/NuRadioReco/modules/envelope_phasedarray/triggerSimulator.py index 0a7aad5fa..e3df099c8 100644 --- a/NuRadioReco/modules/envelope_phasedarray/triggerSimulator.py +++ b/NuRadioReco/modules/envelope_phasedarray/triggerSimulator.py @@ -102,7 +102,7 @@ def envelope_trigger(self, adc_type='perfect_floor_comparator', diode=diode) time_step = 1 / det.get_channel(station_id, channel_id)['trigger_adc_sampling_frequency'] - times = np.arange(len(trace), dtype=np.float) * time_step + times = np.arange(len(trace), dtype=float) * time_step times += channel.get_trace_start_time() else: diff --git a/NuRadioReco/modules/io/NuRadioRecoio.py b/NuRadioReco/modules/io/NuRadioRecoio.py index f7da59216..0544ac553 100644 --- a/NuRadioReco/modules/io/NuRadioRecoio.py +++ b/NuRadioReco/modules/io/NuRadioRecoio.py @@ -3,8 +3,12 @@ import NuRadioReco.detector.detector import NuRadioReco.detector.generic_detector import NuRadioReco.modules.io.event_parser_factory + import numpy as np +import astropy.time + import logging + import time import os @@ -41,14 +45,16 @@ def __init__(self, filenames, parse_header=True, parse_detector=True, fail_on_ve buffer_size: int the size of the read buffer in bytes (default 100MB) """ - if(not isinstance(filenames, list)): + if not isinstance(filenames, list): filenames = [filenames] + self.__file_scanned = False self.logger = logging.getLogger('NuRadioReco.NuRadioRecoio') self.logger.info("initializing NuRadioRecoio with file {}".format(filenames)) t = time.time() if log_level is not None: self.logger.setLevel(log_level) + # Initialize attributes self._filenames = None self.__n_events = None @@ -67,7 +73,7 @@ def __init__(self, filenames, parse_header=True, parse_detector=True, fail_on_ve self.__fail_on_version_mismatch = fail_on_version_mismatch self.__fail_on_minor_version_mismatch = fail_on_minor_version_mismatch self.__parse_header = parse_header - self.__parse_detector = parse_detector + self._parse_detector = parse_detector self.__read_lock = False self.__max_open_files = max_open_files self.__buffer_size = buffer_size @@ -76,43 +82,45 @@ def __init__(self, filenames, parse_header=True, parse_detector=True, fail_on_ve self.logger.info("... finished in {:.0f} seconds".format(time.time() - t)) def _get_file(self, iF): - if(iF not in self.__open_files): + if iF not in self.__open_files: self.logger.debug("file {} is not yet open, opening file".format(iF)) self.__open_files[iF] = {} - self.__open_files[iF]['file'] = open(self._filenames[iF], 'rb', buffering=self.__buffer_size) # 100 MB buffering + self.__open_files[iF]['file'] = open( + self._filenames[iF], 'rb', buffering=self.__buffer_size) # 100 MB buffering self.__open_files[iF]['time'] = time.time() self.__check_file_version(iF) - if(len(self.__open_files) > self.__max_open_files): + + if len(self.__open_files) > self.__max_open_files: self.logger.debug("more than {} file are open, closing oldest file".format(self.__max_open_files)) tnow = time.time() iF_close = 0 for key, value in self.__open_files.items(): - if(value['time'] < tnow): + if value['time'] < tnow: tnow = value['time'] iF_close = key self.logger.debug("closing file {} that was opened at {}".format(iF_close, tnow)) self.__open_files[iF_close]['file'].close() del self.__open_files[iF_close] + return self.__open_files[iF]['file'] def __check_file_version(self, iF): self.__file_version = int.from_bytes(self._get_file(iF).read(6), 'little') self.__file_version_minor = int.from_bytes(self._get_file(iF).read(6), 'little') - if (self.__file_version == 0): + if self.__file_version == 0: self.logger.error( f"File might be corrupt, file has version {self.__file_version}.{self.__file_version} " f"but current version is {VERSION}.{VERSION_MINOR}. " f"This might indicate the file is empty. The file size is {os.stat(self._filenames[iF]).st_size} B.") - elif (self.__file_version > VERSION): + elif self.__file_version > VERSION: self.logger.error( f"File might be corrupt, file has version {self.__file_version}.{self.__file_version} " f"but current version is {VERSION}.{VERSION_MINOR}. " f"This might indicate the file is empty. The file size is {os.stat(self._filenames[iF]).st_size} B.") - - elif(self.__file_version == 1): + elif self.__file_version == 1: self.logger.error( "data file not readable. File has version {}.{} but current version is {}.{}".format( self.__file_version, @@ -121,9 +129,10 @@ def __check_file_version(self, iF): VERSION_MINOR ) ) - if(self.__fail_on_version_mismatch): + if self.__fail_on_version_mismatch: raise IOError - elif(self.__file_version_minor != VERSION_MINOR): + + elif self.__file_version_minor != VERSION_MINOR: self.logger.error( "Data file might not readable, File has version {}.{} but current version is {}.{}".format( self.__file_version, @@ -132,10 +141,13 @@ def __check_file_version(self, iF): VERSION_MINOR ) ) - if(self.__fail_on_minor_version_mismatch): + if self.__fail_on_minor_version_mismatch: raise IOError - self.__scan_files = NuRadioReco.modules.io.event_parser_factory.scan_files_function(self.__file_version, self.__file_version_minor) - self.__iter_events = NuRadioReco.modules.io.event_parser_factory.iter_events_function(self.__file_version, self.__file_version_minor) + + self.__scan_files_versioned = NuRadioReco.modules.io.event_parser_factory.scan_files_function( + self.__file_version, self.__file_version_minor) + self.__iter_events = NuRadioReco.modules.io.event_parser_factory.iter_events_function( + self.__file_version, self.__file_version_minor) def openFile(self, filenames): self._filenames = filenames @@ -151,7 +163,7 @@ def openFile(self, filenames): self._event_specific_detector_changes = {} self.__event_headers = {} - if(self.__parse_header): + if self.__parse_header: self.__scan_files() def close_files(self): @@ -164,28 +176,35 @@ def get_filenames(self): def _parse_event_header(self, evt_header): from NuRadioReco.framework.parameters import stationParameters as stnp self.__event_ids.append(evt_header['event_id']) + for station_id, station in evt_header['stations'].items(): + if station_id not in self.__event_headers: self.__event_headers[station_id] = {} + for key, value in station.items(): # treat sim_station differently - if(key == 'sim_station'): - pass - else: + if key != 'sim_station': + if key not in self.__event_headers[station_id]: self.__event_headers[station_id][key] = [] - if(key == stnp.station_time): - import astropy.time + + if key == stnp.station_time: + station_time = None if value is not None: - if value.format == 'datetime': - time_strings = str(value).split(' ') - station_time = astropy.time.Time('{}T{}'.format(time_strings[0], time_strings[1]), format='isot') + if isinstance(value, dict): + station_time = astropy.time.Time(value["value"], format=value["format"]) + # For backward compatibility, we also keep supporting station times stored + # as astropy.time objects + elif isinstance(value, astropy.time.Time): + station_time = value else: - value.in_subfmt = 'date_hms' - value.out_subfmt = 'date_hms' - station_time=value - else: - station_time = None + err = f"Station time not stored as dict or astropy.time.Time: ({type(value)})" + self.logger.error(err) + raise ValueError(err) + + station_time.format = 'isot' + self.__event_headers[station_id][key].append(station_time) else: self.__event_headers[station_id][key].append(value) @@ -193,11 +212,21 @@ def _parse_event_header(self, evt_header): def __scan_files(self): current_byte = 12 # skip datafile header iF = 0 + iF_prev = None while True: + if iF_prev != iF: + self.logger.debug(f"Start scanning file {iF} ...") + iF_prev = iF + self._get_file(iF).seek(current_byte) - continue_loop, iF, current_byte = self.__scan_files(self, iF, current_byte) + continue_loop, iF, current_byte = self.__scan_files_versioned(self, iF, current_byte) + + if iF_prev != iF: + self.logger.debug(f"Finished scanning file {iF_prev}.") + if not continue_loop: break + self.logger.debug(f"Finished scanning file {iF}. Finished all") self.__event_ids = np.array(self.__event_ids) self.__file_scanned = True @@ -214,36 +243,42 @@ def __scan_files(self): self.__event_headers[station_id][key] = np.array(value) def get_header(self): - if(not self.__file_scanned): + if not self.__file_scanned: self.__scan_files() + return self.__event_headers def get_event_ids(self): """ returns a list of (run, eventid) tuples of all events contained in the data file """ - if(not self.__file_scanned): + if not self.__file_scanned: self.__scan_files() + return self.__event_ids def get_event_i(self, event_number): - while(self.__read_lock): + while self.__read_lock: time.sleep(1) self.logger.debug("read lock waiting 1ms") self.__read_lock = True - if(not self.__file_scanned): + if not self.__file_scanned: self.__scan_files() - if(event_number < 0 or event_number >= self.get_n_events()): - self.logger.error('event number {} out of bounds, only {} present in file'.format(event_number, self.get_n_events())) + + if event_number < 0 or event_number >= self.get_n_events(): + self.logger.error( + 'event number {} out of bounds, only {} present in file'.format( + event_number, self.get_n_events())) self.__read_lock = False return None + # determine in which file event i is istart = 0 file_id = 0 for iF in range(len(self._filenames)): istop = istart + len(self._bytes_start[iF]) - if((event_number >= istart) and (event_number < istop)): + if (event_number >= istart) and (event_number < istop): file_id = iF event_id = event_number - istart break @@ -258,24 +293,23 @@ def get_event_i(self, event_number): self._current_file_id = file_id self._current_event_id = event.get_id() self._current_run_number = event.get_run_number() - # # If the event file contains a detector description that is a - # # generic detector, it might have event-specific properties and we - # # need to set the detector to the right event - if self._current_file_id in self.__detectors.keys(): - if 'generic_detector' in self._detector_dicts[self._current_file_id]: - if self._detector_dicts[self._current_file_id]['generic_detector']: - self.__detectors[self._current_file_id].set_event(self._current_run_number, self._current_event_id) + + self.__set_event_to_detector() return event def get_event(self, event_id): - if(not self.__file_scanned): + if not self.__file_scanned: self.__scan_files() + mask = (self.__event_ids[:, 0] == event_id[0]) & (self.__event_ids[:, 1] == event_id[1]) - if(np.sum(mask) == 0): + if np.sum(mask) == 0: self.logger.error('event number {} not found in file'.format(event_id)) return None - elif(np.sum(mask) > 1): - self.logger.warning(f"{np.sum(mask):d} events with the same run event id pair found. Returning first occurence.") + + elif np.sum(mask) > 1: + self.logger.warning( + f"{np.sum(mask):d} events with the same run event id pair found. Returning first occurence.") + self._current_run_number = event_id[0] self._current_event_id = event_id[1] i = np.argwhere(mask)[0][0] @@ -287,12 +321,18 @@ def get_events(self): for event in self.__iter_events(self): self._current_event_id = event.get_id() self._current_run_number = event.get_run_number() - if self._current_file_id in self.__detectors.keys(): - if 'generic_detector' in self._detector_dicts[self._current_file_id]: - if self._detector_dicts[self._current_file_id]['generic_detector']: - self.__detectors[self._current_file_id].set_event(self._current_run_number, self._current_event_id) + self.__set_event_to_detector() yield event + def __set_event_to_detector(self): + # If the event file contains a detector description that is a + # generic detector, it might have event-specific properties and we + # need to set the detector to the right event + if self._current_file_id in self.__detectors.keys(): + if 'generic_detector' in self._detector_dicts[self._current_file_id]: + if self._detector_dicts[self._current_file_id]['generic_detector']: + self.__detectors[self._current_file_id].set_event(self._current_run_number, self._current_event_id) + def get_detector(self): """ If parse_detector was set True in the __init__() function, this function return @@ -300,47 +340,68 @@ def get_detector(self): files with different detectors are read, the detector for the last returned event is given. """ + + if not self._parse_detector: + self.logger.warn(f"You called \"get_detector\", however, \"parse_detector\" is set to false. Return None!") + return None + # Check if detector object for current file already exists if self._current_file_id not in self.__detectors.keys(): # Detector object for current file does not exist, so we create it if self._current_file_id not in self._detector_dicts: - self.__scan_files() # Maybe we just forgot to scan the file + + if not self.__file_scanned: + self.__scan_files() # Maybe we just forgot to scan the file + if self._current_file_id not in self._detector_dicts: - raise AttributeError('The current file does not contain a detector description.') + self.logger.warn('The current file does not contain a detector description. Return None') + return None + detector_dict = self._detector_dicts[self._current_file_id] if 'generic_detector' in detector_dict.keys(): if detector_dict['generic_detector']: # Detector is a generic detector, so we have to consider default # station/channel and event-specific changes - self.__detectors[self._current_file_id] = NuRadioReco.detector.generic_detector.GenericDetector.__new__(NuRadioReco.detector.generic_detector.GenericDetector) - # the use of default_station and default_channel is deprecated. Allow to set it for now, to ensure backward compatibility + self.__detectors[self._current_file_id] = NuRadioReco.detector.generic_detector.GenericDetector.__new__( + NuRadioReco.detector.generic_detector.GenericDetector) + # the use of default_station and default_channel is deprecated. Allow to + # set it for now, to ensure backward compatibility if 'default_station' not in detector_dict: detector_dict['default_station'] = None if 'default_channel' not in detector_dict: detector_dict['default_channel'] = None - self.__detectors[self._current_file_id].__init__(source='dictionary', json_filename='', dictionary=detector_dict, default_station=detector_dict['default_station'], default_channel=detector_dict['default_channel']) + self.__detectors[self._current_file_id].__init__( + source='dictionary', json_filename='', dictionary=detector_dict, + default_station=detector_dict['default_station'], default_channel=detector_dict['default_channel']) + if self._current_file_id in self._event_specific_detector_changes.keys(): for change in self._event_specific_detector_changes[self._current_file_id]: self.__detectors[self._current_file_id].add_station_properties_for_event( properties=change['properties'], station_id=change['station_id'], run_number=change['run_number'], - event_id=change['event_id'] - ) + event_id=change['event_id']) + self.__detectors[self._current_file_id].set_event(self._current_run_number, self._current_event_id) return self.__detectors[self._current_file_id] + # Detector is a normal detector - self.__detectors[self._current_file_id] = NuRadioReco.detector.detector.Detector.__new__(NuRadioReco.detector.detector.Detector) - self.__detectors[self._current_file_id].__init__(source='dictionary', json_filename='', dictionary=self._detector_dicts[self._current_file_id]) + self.__detectors[self._current_file_id] = NuRadioReco.detector.detector.Detector.__new__( + NuRadioReco.detector.detector.Detector) + self.__detectors[self._current_file_id].__init__( + source='dictionary', json_filename='', dictionary=self._detector_dicts[self._current_file_id]) + # Detector object for current file already exists. If it is a generic detector, # we update it to the run number and ID of the last event that was requested # (in case there are event-specific changes) and return it if 'generic_detector' in self._detector_dicts[self._current_file_id].keys(): if self._detector_dicts[self._current_file_id]['generic_detector']: self.__detectors[self._current_file_id].set_event(self._current_run_number, self._current_event_id) + return self.__detectors[self._current_file_id] def get_n_events(self): - if(not self.__file_scanned): + if not self.__file_scanned: self.__scan_files() + return self.__n_events diff --git a/NuRadioReco/modules/io/noise/__init__.py b/NuRadioReco/modules/io/RNO_G/__init__.py similarity index 100% rename from NuRadioReco/modules/io/noise/__init__.py rename to NuRadioReco/modules/io/RNO_G/__init__.py diff --git a/NuRadioReco/modules/io/RNO_G/readRNOGDataMattak.py b/NuRadioReco/modules/io/RNO_G/readRNOGDataMattak.py new file mode 100644 index 000000000..4d1355026 --- /dev/null +++ b/NuRadioReco/modules/io/RNO_G/readRNOGDataMattak.py @@ -0,0 +1,920 @@ +import numpy as np +import logging +import os +import time +import astropy.time +import math +from functools import lru_cache + +from NuRadioReco.modules.base.module import register_run +from NuRadioReco.modules.RNO_G.channelBlockOffsetFitter import channelBlockOffsets + +import NuRadioReco.framework.event +import NuRadioReco.framework.station +import NuRadioReco.framework.channel +import NuRadioReco.framework.trigger +import NuRadioReco.framework.parameters + +from NuRadioReco.utilities import units +import mattak.Dataset + +import string +import random + + +def create_random_directory_path(prefix="/tmp/", n=7): + """ + Produces a path for a temporary directory with a n letter random suffix + + Parameters + ---------- + + prefix: str + Path prefix, i.e., root directory. (Defaut: /tmp/) + + n: int + Number of letters for the random suffix. (Default: 7) + + Returns + ------- + + path: str + Return path (e.g, /tmp/readRNOGData_XXXXXXX) + """ + # generating random strings + res = ''.join(random.choices(string.ascii_lowercase + string.digits, k=n)) + path = os.path.join(prefix, "readRNOGData_" + res) + + return path + + +def baseline_correction(wfs, n_bins=128, func=np.median, return_offsets=False): + """ + Simple baseline correction function. + + Determines baseline in discrete chuncks of "n_bins" with + the function specified (i.e., mean or median). + + .. Warning:: This function has been deprecated, use :mod:`NuRadioReco.modules.RNO_G.channelBlockOffsetFitter` + instead + + Parameters + ---------- + + wfs: np.array(n_events, n_channels, n_samples) + Waveforms of several events/channels. + + n_bins: int + Number of samples/bins in one "chunck". If None, calculate median/mean over entire trace. (Default: 128) + + func: np.mean or np.median + Function to calculate pedestal + + return_offsets: bool, default False + if True, additionally return the baseline offsets + + Returns + ------- + + wfs_corrected: np.array(n_events, n_channels, n_samples) + Baseline/pedestal corrected waveforms + + baseline_values: np.array of shape (n_samples // n_bins, n_events, n_channels) + (Only if return_offsets==True) The baseline offsets + """ + import warnings + warnings.warn( + 'baseline_correction is deprecated, use NuRadioReco.modules.RNO_G.channelBlockOffsetFitter instead', + DeprecationWarning) + # Example: Get baselines in chunks of 128 bins + # wfs in (n_events, n_channels, 2048) + # np.split -> (16, n_events, n_channels, 128) each waveform split in 16 chuncks + # func -> (16, n_events, n_channels) pedestal for each chunck + if n_bins is not None: + baseline_values = func(np.split(wfs, 2048 // n_bins, axis=-1), axis=-1) + + # np.repeat -> (2048, n_events, n_channels) concatenate the 16 chuncks to one baseline + baseline_traces = np.repeat(baseline_values, n_bins % 2048, axis=0) + else: + baseline_values = func(wfs, axis=-1) + # np.repeat -> (2048, n_events, n_channels) concatenate the 16 chuncks to one baseline + baseline_traces = np.repeat(baseline_values, 2048, axis=0) + + # np.moveaxis -> (n_events, n_channels, 2048) + baseline_traces = np.moveaxis(baseline_traces, 0, -1) + + if return_offsets: + return wfs - baseline_traces, baseline_values + + return wfs - baseline_traces + + +def get_time_offset(trigger_type): + """ + Mapping the offset between trace start time and trigger time (~ signal time). + Temporary use hard-coded values for each trigger type. In the future this + information might be time, station, and channel dependent and should come + from a database (or is already calibrated in mattak) + + Current values motivated by figures posted in PR https://github.com/nu-radio/NuRadioMC/pull/519 + + Parameters + ---------- + + trigger_type: str + Trigger type encoded as string from Mattak + + Returns + ------- + + time_offset: float + trace_start_time = trigger_time - time_offset + + """ + + time_offsets = { + "FORCE": 0, + "LT": 250 * units.ns, + "RADIANT": 475 * units.ns, + "UNKNOWN": 0 # Due to a firmware issue at the beginning of data taking the trigger types were not properly set. + } + + # Should have the same time offset ?! + if trigger_type.startswith("RADIANT"): + trigger_type = "RADIANT" + + if trigger_type in time_offsets: + return time_offsets[trigger_type] + else: + known_trigger_types = ", ".join(time_offsets.keys()) + raise KeyError(f"Unknown trigger type: {trigger_type}. Known are: {known_trigger_types}. Abort ....") + + +def all_files_in_directory(mattak_dir): + """ + Checks if all Mattak root files are in a directory. + Ignoring runinfo.root because (asaik) not all runs have those and information is currently not read by Mattak. + There are mattak directories which produce a ReferenceError when reading. They have a "combined.root" which is + apparently empty but are missing the daqstatus, pedestal, and header file. + + Parameters + ---------- + + mattak_dir: str + Path to a mattak directory + + Returns + ------- + + all_there: bool + True, if all "req_files" are there and waveforms.root or combined.root. Otherwise returns False. + """ + # one or the other has to be present + if not os.path.exists(os.path.join(mattak_dir, "waveforms.root")) and \ + not os.path.exists(os.path.join(mattak_dir, "combined.root")): + return False + + req_files = ["daqstatus.root", "headers.root", "pedestal.root"] + for file in req_files: + if not os.path.exists(os.path.join(mattak_dir, file)): + return False + + return True + + +class readRNOGData: + + def __init__(self, run_table_path=None, load_run_table=True, log_level=logging.INFO): + """ + Parameters + ---------- + + run_table_path: str | None + Path to a run_table.cvs file. If None, the run table is queried from the DB. (Default: None) + + load_run_table: bool + If True, try to load the run_table from run_table_path. Otherwise, skip this. + + log_level: enum + Set verbosity level of logger. If logging.DEBUG, set mattak to verbose (unless specified in mattak_kwargs). + (Default: logging.INFO) + """ + self.logger = logging.getLogger('NuRadioReco.readRNOGData') + self.logger.setLevel(log_level) + + self._blockoffsetfitter = channelBlockOffsets() + + # Initialize run table for run selection + self.__run_table = None + + self.__temporary_dirs = [] + if load_run_table: + if run_table_path is None: + try: + from rnog_data.runtable import RunTable + self.logger.debug("Access RunTable database ...") + try: + self.__run_table = RunTable().get_table() + except: + self.logger.warn("No connect to RunTable database could be established. " + "Runs can not be filtered.") + except ImportError: + self.logger.warn("Import of run table failed. Runs can not be filtered.! \n" + "You can get the interface from GitHub: git@github.com:RNO-G/rnog-runtable.git") + else: + import pandas + self.__run_table = pandas.read_csv(run_table_path) + + + def begin(self, + dirs_files, + read_calibrated_data=False, + select_triggers=None, + select_runs=False, + apply_baseline_correction='approximate', + convert_to_voltage=True, + selectors=[], + run_types=["physics"], + run_time_range=None, + max_trigger_rate=0 * units.Hz, + mattak_kwargs={}, + overwrite_sampling_rate=None): + """ + Parameters + ---------- + + dirs_files: str, list(str) + Path to run directories (i.e. ".../stationXX/runXXX/") or path to root files (have to be "combined" mattak files). + + read_calibrated_data: bool + If True, read calibrated waveforms from Mattak.Dataset. If False, read "raw" ADC traces. + (temp. Default: False, this can/should be switched once the calibration in incorp. into Mattak) + + select_triggers: str or list(str) + Names of triggers which should be selected. Convinence interface instead of passing a selector + (see "selectors" below). (Default: None) + + select_runs: bool + If True, use information in run_table to select runs (based on run_type, run_time, trigger_rate, ...). + If the run_table is not available no selection is performed (and the programm is not interrupted, + only an error message is raised). See parameters to configure run selection. (Default: False) + + Other Parameters + ---------------- + + apply_baseline_correction: 'approximate' | 'fit' | 'none' + Only applies when non-calibrated data are read. Removes the DC (baseline) + block offsets (pedestals). + Options are: + + * 'approximate' (default) - estimate block offsets by looking at the low-pass filtered trace + * 'fit' - do a full out-of-band fit to determine the block offsets; for more details, + see :mod:`NuRadioReco.modules.RNO_G.channelBlockOffsetFitter` + * 'none' - do not apply a baseline correction (faster) + + convert_to_voltage: bool + Only applies when non-calibrated data are read. If true, convert ADC to voltage. + (Default: True) + + selectors: list of lambdas + List of lambda(eventInfo) -> bool to pass to mattak.Dataset.iterate to select events. + Example: trigger_selector = lambda eventInfo: eventInfo.triggerType == "FORCE" + + run_types: list + Used to select/reject runs from information in the RNO-G RunTable. List of run_types to be used. (Default: ['physics']) + + run_time_range: tuple + Specify a time range to select runs (it is sufficient that runs cover the time range partially). + Each value of the tuple has to be in a format which astropy.time.Time understands. A value can be None + which means that the lower or upper bound is unconstrained. If run_time_range is None no time selection is + applied. (Default: None) + + max_trigger_rate: float + Used to select/reject runs from information in the RNO-G RunTable. Maximum allowed trigger rate (per run) in Hz. + If 0, no cut is applied. (Default: 1 Hz) + + mattak_kwargs: dict + Dictionary of arguments for mattak.Dataset.Dataset. (Default: {}) + Example: Select a mattak "backend". Options are "auto", "pyroot", "uproot". If "auto" is selected, + pyroot is used if available otherwise a "fallback" to uproot is used. (Default: "auto") + + overwrite_sampling_rate: float + Set sampling rate of the imported waveforms. This overwrites what is read out from runinfo (i.e., stored in the mattak files). + If None, nothing is overwritten and the sampling rate from the mattak file is used. (Default: None) + NOTE: This option might be necessary when old mattak files are read which have this not set. + """ + + t0 = time.time() + + self._read_calibrated_data = read_calibrated_data + baseline_correction_valid_options = ['approximate', 'fit', 'none'] + if apply_baseline_correction.lower() not in baseline_correction_valid_options: + raise ValueError( + f"Value for apply_baseline_correction ({apply_baseline_correction}) not recognized. " + f"Valid options are {baseline_correction_valid_options}" + ) + self._apply_baseline_correction = apply_baseline_correction + self._convert_to_voltage = convert_to_voltage + + # Temporary solution hard-coded values from Cosmin. Only used when uncalibrated data + # is read and convert_to_voltage is True. + self._adc_ref_voltage_range = 2.5 * units.volt + self._adc_n_bits = 12 + + self._overwrite_sampling_rate = overwrite_sampling_rate + + # Set parameter for run selection + self.__max_trigger_rate = max_trigger_rate + self.__run_types = run_types + + if run_time_range is not None: + convert_time = lambda t: None if t is None else astropy.time.Time(t) + self._time_low = convert_time(run_time_range[0]) + self._time_high = convert_time(run_time_range[1]) + else: + self._time_low = None + self._time_high = None + + if select_runs and self.__run_table is not None: + self.logger.info("\n\tSelect runs with type: {}".format(", ".join(run_types)) + + f"\n\tSelect runs with max. trigger rate of {max_trigger_rate / units.Hz} Hz" + f"\n\tSelect runs which are between {self._time_low} - {self._time_high}") + + self.set_selectors(selectors, select_triggers) + + # Read data + self._time_begin = 0 + self._time_run = 0 + self.__counter = 0 + self.__skipped = 0 + self.__invalid = 0 + + self._events_information = None + self._datasets = [] + self.__n_events_per_dataset = [] + + self.logger.info(f"Parse through / read-in {len(dirs_files)} directory(ies) / file(s).") + + self.__skipped_runs = 0 + self.__n_runs = 0 + + if not isinstance(dirs_files, (list, np.ndarray)): + dirs_files = [dirs_files] + + # Set verbose for mattak + if "verbose" in mattak_kwargs: + verbose = mattak_kwargs.pop("verbose") + else: + verbose = self.logger.level >= logging.DEBUG + + for dir_file in dirs_files: + + if not os.path.exists(dir_file): + self.logger.error(f"The directory/file {dir_file} does not exist") + continue + + if os.path.isdir(dir_file): + + if not all_files_in_directory(dir_file): + self.logger.error(f"Incomplete directory: {dir_file}. Skip ...") + continue + else: + # Providing direct paths to a Mattak combined.root file is not supported by the mattak library yet. + # It only accepts directry paths in which it will look for a file called `combined.root` (or waveforms.root if + # it is not a combined file). To work around this: Create a tmp directory under `/tmp/`, link the file you want to + # read into this directory with the the name `combined.root`, use this path to read the run. + + path = create_random_directory_path() + self.logger.debug(f"Create temporary directory: {path}") + if os.path.exists(path): + raise ValueError(f"Temporary directory {path} already exists.") + + os.mkdir(path) + self.__temporary_dirs.append(path) # for housekeeping + + self.logger.debug(f"Create symlink for {dir_file}") + os.symlink(dir_file, os.path.join(path, "combined.root")) + + dir_file = path # set path to e.g. /tmp/NuRadioReco_XXXXXXX/combined.root + + + try: + dataset = mattak.Dataset.Dataset(station=0, run=0, data_dir=dir_file, verbose=verbose, **mattak_kwargs) + except (ReferenceError, KeyError) as e: + self.logger.error(f"The following exeption was raised reading in the run: {dir_file}. Skip that run ...:\n", exc_info=e) + continue + + # filter runs/datasets based on + if select_runs and self.__run_table is not None and not self.__select_run(dataset): + self.__skipped_runs += 1 + continue + + self.__n_runs += 1 + self._datasets.append(dataset) + self.__n_events_per_dataset.append(dataset.N()) + + if not len(self._datasets): + err = "Found no valid datasets. Stop!" + self.logger.error(err) + raise FileNotFoundError(err) + + # keeps track which event index is in which dataset + self._event_idxs_datasets = np.cumsum(self.__n_events_per_dataset) + self._n_events_total = np.sum(self.__n_events_per_dataset) + self._time_begin = time.time() - t0 + + self.logger.info(f"{self._n_events_total} events in {len(self._datasets)} runs/datasets " + f"have been found using the {self._datasets[0].backend} Mattak backend.") + + if not self._n_events_total: + err = "No runs have been selected. Abort ..." + self.logger.error(err) + raise ValueError(err) + + + def set_selectors(self, selectors, select_triggers=None): + """ + Parameters + ---------- + + selectors: list of lambdas + List of lambda(eventInfo) -> bool to pass to mattak.Dataset.iterate to select events. + Example: trigger_selector = lambda eventInfo: eventInfo.triggerType == "FORCE" + + select_triggers: str or list(str) + Names of triggers which should be selected. Convinence interface instead of passing a selector. (Default: None) + """ + + # Initialize selectors for event filtering + if not isinstance(selectors, (list, np.ndarray)): + selectors = [selectors] + + if select_triggers is not None: + if isinstance(select_triggers, str): + selectors.append(lambda eventInfo: eventInfo.triggerType == select_triggers) + else: + for select_trigger in select_triggers: + selectors.append(lambda eventInfo: eventInfo.triggerType == select_trigger) + + self._selectors = selectors + self.logger.info(f"Set {len(self._selectors)} selector(s)") + + + def __select_run(self, dataset): + """ Filter/select runs/datasets. + + Parameters + ---------- + + dataset: mattak.Dataset.Dataset + + select: bool + Return True to select an dataset, return False to reject/skip it. + """ + + # get first eventInfo + dataset.setEntries(0) + event_info = dataset.eventInfo() + + run_id = event_info.run + station_id = event_info.station + + run_info = self.__run_table.query(f"station == {station_id:d} & run == {run_id:d}") + + if not len(run_info): + self.logger.error(f"Run {run_id:d} (station {station_id:d}) not in run table. Reject...") + return False + + # "time_start/end" is stored in the isot format. datetime is much faster than astropy (~85ns vs 55 mus). + # But using datetime would mean to stip decimals because datetime can only handle mu sec precision and can not cope + # with the additional decimals for ns. + if self._time_low is not None: + time_end = astropy.time.Time(run_info["time_end"].values[0]) + if time_end < self._time_low: + self.logger.info(f"Reject station {station_id} run {run_id} because run ended before {self._time_low}") + return False + + if self._time_high is not None: + time_start = astropy.time.Time(run_info["time_start"].values[0]) + if time_start > self._time_high: + self.logger.info(f"Reject station {station_id} run {run_id} because run started time after {self._time_high}") + return False + + run_type = run_info["run_type"].values[0] + if not run_type in self.__run_types: + self.logger.info(f"Reject station {station_id} run {run_id} because of run type {run_type}") + return False + + trigger_rate = run_info["trigger_rate"].values[0] * units.Hz + if self.__max_trigger_rate and trigger_rate > self.__max_trigger_rate: + self.logger.info(f"Reject station {station_id} run {run_id} because trigger rate is to high ({trigger_rate / units.Hz:.2f} Hz)") + return False + + return True + + + def __get_n_events_of_prev_datasets(self, dataset_idx): + """ Get accumulated number of events from previous datasets """ + dataset_idx_prev = dataset_idx - 1 + return int(self._event_idxs_datasets[dataset_idx_prev]) if dataset_idx_prev >= 0 else 0 + + + def __get_dataset_for_event(self, event_idx): + """ Get correct dataset and set entry accordingly to event index + + Parameters + ---------- + + event_index: int + Same as in read_event(). + + Returns + ------- + + dataset: mattak.Dataset.Dataset + """ + # find correct dataset + dataset_idx = np.digitize(event_idx, self._event_idxs_datasets) + dataset = self._datasets[dataset_idx] + + event_idx_in_dataset = event_idx - self.__get_n_events_of_prev_datasets(dataset_idx) + dataset.setEntries(event_idx_in_dataset) # increment iterator -> point to new event + + return dataset + + + def _filter_event(self, evtinfo, event_idx=None): + """ Filter an event base on its EventInfo and the configured selectors. + + Parameters + ---------- + + event_info: mattak.Dataset.EventInfo + The event info object for one event. + + event_index: int + Same as in read_event(). Only use for logger.info(). (Default: None) + + Returns + ------- + + skip: bool + Returns True to skip/reject event, return False to keep/read event + """ + if self._selectors is not None: + for selector in self._selectors: + if not selector(evtinfo): + self.logger.debug(f"Event {event_idx} (station {evtinfo.station}, run {evtinfo.run}, " + f"event number {evtinfo.eventNumber}) is skipped.") + self.__skipped += 1 + return True + + return False + + + def get_events_information(self, keys=["station", "run", "eventNumber"]): + """ Return information of all events from the EventInfo + + This function is useful to make a pre-selection of events before actually reading them in combination with + self.read_event(). + + Parameters + ---------- + + keys: list(str) + List of the information to receive from each event. Have to match the attributes (member variables) + of the mattak.Dataset.EventInfo class (examples are "station", "run", "triggerTime", "triggerType", "eventNumber", ...). + (Default: ["station", "run", "eventNumber"]) + + Returns + ------- + + data: dict + Keys of the dict are the event indecies (as used in self.read_event(event_index)). The values are dictinaries + them self containing the information specified with "keys" parameter. + """ + + # Read if dict is None ... + do_read = self._events_information is None + + if not do_read: + # ... or when it does not have the desired information + first_event_info = self._events_information[list(self._events_information.keys())[0]] + + for key in keys: + if key not in list(first_event_info.keys()): + do_read = True + + if do_read: + + self._events_information = {} + n_prev = 0 + for dataset in self._datasets: + dataset.setEntries((0, dataset.N())) + + for idx, evtinfo in enumerate(dataset.eventInfo()): # returns a list + + event_idx = idx + n_prev # event index accross all datasets combined + + if self._filter_event(evtinfo, event_idx): + continue + + self._events_information[event_idx] = {key: getattr(evtinfo, key) for key in keys} + + n_prev += dataset.N() + + return self._events_information + + + def _check_for_valid_information_in_event_info(self, event_info): + """ + Checks if certain information (sampling rate, trigger time) in mattak.Dataset.EventInfo are valid + + Parameters + ---------- + + event_info: mattak.Dataset.EventInfo + + Returns + ------- + + is_valid: bool + Returns True if all information valid, false otherwise + """ + + if math.isinf(event_info.triggerTime): + self.logger.error(f"Event {event_info.eventNumber} (st {event_info.station}, run {event_info.run}) " + "has inf trigger time. Skip event...") + self.__invalid += 1 + return False + + + if (event_info.sampleRate == 0 or event_info.sampleRate is None) and self._overwrite_sampling_rate is None: + self.logger.error(f"Event {event_info.eventNumber} (st {event_info.station}, run {event_info.run}) " + f"has a sampling rate of {event_info.sampleRate} GHz. Event is skipped ... " + f"You can avoid this by setting 'overwrite_sampling_rate' in the begin() method.") + self.__invalid += 1 + return False + + return True + + + def _get_event(self, event_info, waveforms): + """ Return a NuRadioReco event + + Parameters + ---------- + + event_info: mattak.Dataset.EventInfo + The event info object for one event. + + waveforms: np.array(n_channel, n_samples) + Typically what dataset.wfs() returns (for one event!) + + Returns + ------- + + evt: NuRadioReco.framework.event + """ + + trigger_time = event_info.triggerTime + if self._overwrite_sampling_rate is not None: + sampling_rate = self._overwrite_sampling_rate + else: + sampling_rate = event_info.sampleRate + + evt = NuRadioReco.framework.event.Event(event_info.run, event_info.eventNumber) + station = NuRadioReco.framework.station.Station(event_info.station) + station.set_station_time(astropy.time.Time(trigger_time, format='unix')) + + trigger = NuRadioReco.framework.trigger.Trigger(event_info.triggerType) + trigger.set_triggered() + trigger.set_trigger_time(trigger_time) + station.set_trigger(trigger) + + for channel_id, wf in enumerate(waveforms): + channel = NuRadioReco.framework.channel.Channel(channel_id) + if self._read_calibrated_data: + channel.set_trace(wf * units.mV, sampling_rate * units.GHz) + else: + # wf stores ADC counts + if self._convert_to_voltage: + # convert adc to voltage + wf = wf * (self._adc_ref_voltage_range / (2 ** (self._adc_n_bits) - 1)) + + channel.set_trace(wf, sampling_rate * units.GHz) + + time_offset = get_time_offset(event_info.triggerType) + channel.set_trace_start_time(-time_offset) # relative to event/trigger time + + station.add_channel(channel) + + evt.set_station(station) + if self._apply_baseline_correction in ['fit', 'approximate'] and not self._read_calibrated_data: + self._blockoffsetfitter.remove_offsets(evt, station, mode=self._apply_baseline_correction) + + return evt + + + @register_run() + def run(self): + """ + Loop over all events. + + Returns + ------- + + evt: generator(NuRadioReco.framework.event) + """ + event_idx = -1 + for dataset in self._datasets: + dataset.setEntries((0, dataset.N())) + + # read all event infos of the entier dataset (= run) + event_infos = dataset.eventInfo() + wfs = None + + for idx, evtinfo in enumerate(event_infos): # returns a list + event_idx += 1 + + self.logger.debug(f"Processing event number {event_idx} out of total {self._n_events_total}") + t0 = time.time() + + if self._filter_event(evtinfo, event_idx): + continue + + if not self._check_for_valid_information_in_event_info(evtinfo): + continue + + # Just read wfs if necessary + if wfs is None: + wfs = dataset.wfs() + + waveforms_of_event = wfs[idx] + + evt = self._get_event(evtinfo, waveforms_of_event) + + self._time_run += time.time() - t0 + self.__counter += 1 + + yield evt + + + + def get_event_by_index(self, event_index): + """ Allows to read a specific event identifed by its index + + Parameters + ---------- + + event_index: int + The index of a particluar event. The index is the chronological number from 0 to + number of total events (across all datasets). + + Returns + ------- + + evt: NuRadioReco.framework.event + """ + + self.logger.debug(f"Processing event number {event_index} out of total {self._n_events_total}") + t0 = time.time() + + dataset = self.__get_dataset_for_event(event_index) + event_info = dataset.eventInfo() # returns a single eventInfo + + if self._filter_event(event_info, event_index): + return None + + # check this before reading the wfs + if not self._check_for_valid_information_in_event_info(event_info): + return None + + # access data + waveforms = dataset.wfs() + + evt = self._get_event(event_info, waveforms) + + self._time_run += time.time() - t0 + self.__counter += 1 + + return evt + + + def get_event(self, run_nr, event_id): + """ Allows to read a specific event identifed by run number and event id + + Parameters + ---------- + + run_nr: int + Run number + + event_id: int + Event Id + + Returns + ------- + + evt: NuRadioReco.framework.event + """ + + self.logger.debug(f"Processing event {event_id}") + t0 = time.time() + + event_infos = self.get_events_information(keys=["eventNumber", "run"]) + event_idx_ids = np.array([[index, ele["eventNumber"], ele["run"]] for index, ele in event_infos.items()]) + mask = np.all([event_idx_ids[:, 1] == event_id, event_idx_ids[:, 2] == run_nr], axis=0) + + if not np.any(mask): + self.logger.info(f"Could not find event with id: {event_id}.") + return None + elif np.sum(mask) > 1: + self.logger.error(f"Found several events with the same id: {event_id}.") + raise ValueError(f"Found several events with the same id: {event_id}.") + else: + pass + + # int(...) necessary to pass it to mattak + event_index = int(event_idx_ids[mask, 0][0]) + + dataset = self.__get_dataset_for_event(event_index) + event_info = dataset.eventInfo() # returns a single eventInfo + + if self._filter_event(event_info, event_index): + return None + + # check this before reading the wfs + if not self._check_for_valid_information_in_event_info(event_info): + return None + + # access data + waveforms = dataset.wfs() + + evt = self._get_event(event_info, waveforms) + + self._time_run += time.time() - t0 + self.__counter += 1 + + return evt + + + def end(self): + if self.__counter: + self.logger.info( + f"\n\tRead {self.__counter} events (skipped {self.__skipped} events, {self.__invalid} invalid events)" + f"\n\tTime to initialize data sets : {self._time_begin:.2f}s" + f"\n\tTime to read all events : {self._time_run:.2f}s" + f"\n\tTime to per event : {self._time_run / self.__counter:.2f}s" + f"\n\tRead {self.__n_runs} runs, skipped {self.__skipped_runs} runs.") + else: + self.logger.info( + f"\n\tRead {self.__counter} events (skipped {self.__skipped} events, {self.__invalid} invalid events)") + + # Clean up links and temporary directories. + for d in self.__temporary_dirs: + self.logger.debug(f"Remove temporary folder: {d}") + os.unlink(os.path.join(d, "combined.root")) + os.rmdir(d) + + +### we create a wrapper for readRNOGData to mirror the interface of the .nur reader +class _readRNOGData_eventbrowser(readRNOGData): + """ + Wrapper for readRNOGData for use in the eventbrowser + + This wrapper mirrors the interface of the .nur reader for use in the eventbrowser, + and uses the lru_cache to speed up IO for parallel plots of the same event. + It should probably not be used outside of the eventbrowser. + + """ + def begin(self, *args, **kwargs): + # We reduce the required amount of IO by caching individual events. + # However, we need to clear the cache every time we change the file + # we're reading. This is implemented here. + self.get_event_i.cache_clear() + self.get_event.cache_clear() + return super().begin(*args, **kwargs) + + def get_event_ids(self): + event_infos = self.get_events_information() + return np.array([(i['run'], i['eventNumber']) for i in event_infos.values()]) + + @lru_cache(maxsize=1) + def get_event_i(self, i): + return self.get_event_by_index(i) + + @lru_cache(maxsize=1) + def get_event(self, event_id): + return super().get_event(*event_id) + + def get_n_events(self): + return self._n_events_total + + def get_detector(self): + """Not implemented in mattak reader""" + return None + + def get_header(self): + """Not implemented in mattak reader""" + return None \ No newline at end of file diff --git a/NuRadioReco/modules/io/coreas/coreas.py b/NuRadioReco/modules/io/coreas/coreas.py index 8898f6ee0..38bee3a2e 100644 --- a/NuRadioReco/modules/io/coreas/coreas.py +++ b/NuRadioReco/modules/io/coreas/coreas.py @@ -1,20 +1,16 @@ import numpy as np import matplotlib.pyplot as plt - from radiotools import helper as hp from radiotools import coordinatesystems - from NuRadioReco.utilities import units import NuRadioReco.framework.sim_station import NuRadioReco.framework.base_trace import NuRadioReco.framework.electric_field import NuRadioReco.framework.radio_shower import radiotools.coordinatesystems - from NuRadioReco.framework.parameters import stationParameters as stnp from NuRadioReco.framework.parameters import electricFieldParameters as efp from NuRadioReco.framework.parameters import showerParameters as shp - import logging logger = logging.getLogger('coreas') @@ -70,7 +66,7 @@ def calculate_simulation_weights(positions, zenith, azimuth, site='summit', debu ax1.set_ylabel(r'Position in $\vec{v} \times \vec{v} \times \vec{B}$ - direction [m]') weights = np.zeros_like(positions[:, 0]) - for p in range(0, weights.shape[0]): # loop over all 240 station positions + for p in range(0, weights.shape[0]): # loop over all observer positions vertices_shower_2d = vor.vertices[vor.regions[vor.point_region[p]]] # x_vertice_ground = x_trafo_from_shower[0] * x_vertice_shower + y_trafo_from_shower[0] * y_vertice_shower + z_trafo_from_shower[0] * z_vertice_shower @@ -87,7 +83,7 @@ def calculate_simulation_weights(positions, zenith, azimuth, site='summit', debu vertices_shower_3d = np.array(vertices_shower_3d) vertices_ground = cstrafo.transform_from_vxB_vxvxB(station_position=vertices_shower_3d) - n_arms = 8 # mask last stations of each arm + n_arms = 8 # mask last observer position of each arm length_shower = np.sqrt(shower[:, 0] ** 2 + shower[:, 1] ** 2) ind = np.argpartition(length_shower, -n_arms)[-n_arms:] @@ -99,7 +95,7 @@ def calculate_simulation_weights(positions, zenith, azimuth, site='summit', debu ax2.plot(vertices_ground[:, 0], vertices_ground[:, 1], c='grey', zorder=1) ax2.scatter(vertices_ground[:, 0], vertices_ground[:, 1], c='tab:orange', zorder=2) if debug: - ax2.scatter(positions[:, 0], positions[:, 1], c='tab:blue', s=10, label='Position of stations') + ax2.scatter(positions[:, 0], positions[:, 1], c='tab:blue', s=10, label='Position of observer') ax2.scatter(vertices_ground[:, 0], vertices_ground[:, 1], c='tab:orange', label='Vertices of cell') ax2.set_aspect('equal') ax2.set_title('On ground, total area {:.2f} $km^2$'.format(sum(weights) / units.km ** 2)) @@ -117,7 +113,7 @@ def calculate_simulation_weights(positions, zenith, azimuth, site='summit', debu ax4.hist(weights) ax4.set_title('Weight distribution') ax4.set_xlabel(r'Weights (here area) $[m^2]$') - ax4.set_ylabel(r'Number of stations') + ax4.set_ylabel(r'Number of observer') ax5.scatter(length_shower ** 2, weights) ax5.set_xlabel(r'$Length^2 [m^2]$') @@ -129,7 +125,7 @@ def calculate_simulation_weights(positions, zenith, azimuth, site='summit', debu def make_sim_station(station_id, corsika, observer, channel_ids, weight=None): """ - creates an NuRadioReco sim station from the observer object of the coreas hdf5 file + creates an NuRadioReco sim station from the (interpolated) observer object of the coreas hdf5 file Parameters ---------- @@ -145,10 +141,11 @@ def make_sim_station(station_id, corsika, observer, channel_ids, weight=None): Returns ------- sim_station: sim station - ARIANNA simulated station object + simulated station object """ - # loop over all coreas stations, rotate to ARIANNA CS and save to simulation branch + zenith, azimuth, magnetic_field_vector = get_angles(corsika) + if(observer is None): data = np.zeros((512, 4)) data[:, 0] = np.arange(0, 512) * units.ns / units.second @@ -201,6 +198,22 @@ def make_sim_station(station_id, corsika, observer, channel_ids, weight=None): def make_sim_shower(corsika, observer=None, detector=None, station_id=None): + """ + creates an NuRadioReco sim shower from the coreas hdf5 file + + Parameters + ---------- + corsika : hdf5 file object + the open hdf5 file object of the corsika hdf5 file + observer : hdf5 observer object + detector : detector object + station_id : station id of the station relativ to which the shower core is given + + Returns + ------- + sim_shower: sim shower + simulated shower object + """ sim_shower = NuRadioReco.framework.radio_shower.RadioShower() zenith, azimuth, magnetic_field_vector = get_angles(corsika) diff --git a/NuRadioReco/modules/io/coreas/readCoREASShower.py b/NuRadioReco/modules/io/coreas/readCoREASShower.py index a9ee174e5..a0c8fd486 100644 --- a/NuRadioReco/modules/io/coreas/readCoREASShower.py +++ b/NuRadioReco/modules/io/coreas/readCoREASShower.py @@ -4,13 +4,11 @@ from NuRadioReco.modules.io.coreas import coreas from NuRadioReco.utilities import units from radiotools import coordinatesystems - import h5py import numpy as np import time import re import os - import logging logger = logging.getLogger('readCoREASShower') @@ -57,15 +55,17 @@ def begin(self, input_files, det=None, logger_level=logging.NOTSET, set_ascendin def run(self): """ - Reads in a CoREAS file and returns an event containing all simulated stations - + Reads in a CoREAS file and returns one event containing all simulated observer positions as stations. + A detector description is not needed to run this module. If a genericDetector is passed to the begin method, + the stations are added to it and the run method returns both the event and the detector. """ - while (self.__current_input_file < len(self.__input_files)): + while self.__current_input_file < len(self.__input_files): t = time.time() t_per_event = time.time() - filesize = os.path.getsize(self.__input_files[self.__current_input_file]) - if(filesize < 18456 * 2): # based on the observation that a file with such a small filesize is corrupt + filesize = os.path.getsize( + self.__input_files[self.__current_input_file]) + if filesize < 18456 * 2: # based on the observation that a file with such a small filesize is corrupt logger.warning( "file {} seems to be corrupt, skipping to next file".format( self.__input_files[self.__current_input_file] @@ -74,9 +74,11 @@ def run(self): self.__current_input_file += 1 continue - logger.info('Reading %s ...' % self.__input_files[self.__current_input_file]) + logger.info('Reading %s ...' % + self.__input_files[self.__current_input_file]) - corsika = h5py.File(self.__input_files[self.__current_input_file], "r") + corsika = h5py.File( + self.__input_files[self.__current_input_file], "r") logger.info("using coreas simulation {} with E={:2g} theta = {:.0f}".format( self.__input_files[self.__current_input_file], corsika['inputs'].attrs["ERANGE"][0] * units.GeV, corsika['inputs'].attrs["THETAP"][0])) @@ -88,13 +90,15 @@ def run(self): self.__ascending_run_and_event_number) self.__ascending_run_and_event_number += 1 else: - evt = NuRadioReco.framework.event.Event(corsika['inputs'].attrs['RUNNR'], corsika['inputs'].attrs['EVTNR']) + evt = NuRadioReco.framework.event.Event( + corsika['inputs'].attrs['RUNNR'], corsika['inputs'].attrs['EVTNR']) evt.__event_time = f_coreas.attrs["GPSSecs"] # create sim shower, no core is set since no external detector description is given sim_shower = coreas.make_sim_shower(corsika) - sim_shower.set_parameter(shp.core, np.array([0, 0, f_coreas.attrs["CoreCoordinateVertical"] / 100])) # set core + sim_shower.set_parameter(shp.core, np.array( + [0, 0, f_coreas.attrs["CoreCoordinateVertical"] / 100])) # set core evt.add_sim_shower(sim_shower) # initialize coordinate transformation @@ -103,20 +107,26 @@ def run(self): # add simulated pulses as sim station for idx, (name, observer) in enumerate(f_coreas['observers'].items()): - station_id = antenna_id(name, idx) # returns proper station id if possible + # returns proper station id if possible + station_id = antenna_id(name, idx) station = NuRadioReco.framework.station.Station(station_id) if self.__det is None: - sim_station = coreas.make_sim_station(station_id, corsika, observer, channel_ids=[0, 1, 2]) + sim_station = coreas.make_sim_station( + station_id, corsika, observer, channel_ids=[0, 1, 2]) else: - sim_station = coreas.make_sim_station(station_id, corsika, observer, channel_ids=self.__det.get_channel_ids(self.__det.get_default_station_id())) + sim_station = coreas.make_sim_station( + station_id, corsika, observer, + channel_ids=self.__det.get_channel_ids(self.__det.get_default_station_id())) + station.set_sim_station(sim_station) evt.set_station(station) + if self.__det is not None: position = observer.attrs['position'] - antenna_position = np.zeros(3) - antenna_position[0], antenna_position[1], antenna_position[2] = -position[1] * units.cm, position[0] * units.cm, position[2] * units.cm + antenna_position = np.array([-position[1], position[0], position[2]]) * units.cm antenna_position = cs.transform_from_magnetic_to_geographic(antenna_position) + if not self.__det.has_station(station_id): self.__det.add_generic_station({ 'station_id': station_id, @@ -147,8 +157,10 @@ def end(self): logger.setLevel(logging.INFO) dt = timedelta(seconds=self.__t) logger.info("total time used by this module is {}".format(dt)) - logger.info("\tcreate event structure {}".format(timedelta(seconds=self.__t_event_structure))) - logger.info("per event {}".format(timedelta(seconds=self.__t_per_event))) + logger.info("\tcreate event structure {}".format( + timedelta(seconds=self.__t_event_structure))) + logger.info("per event {}".format( + timedelta(seconds=self.__t_per_event))) return dt diff --git a/NuRadioReco/modules/io/coreas/readCoREASStation.py b/NuRadioReco/modules/io/coreas/readCoREASStation.py index f47f4313e..45b6d9843 100644 --- a/NuRadioReco/modules/io/coreas/readCoREASStation.py +++ b/NuRadioReco/modules/io/coreas/readCoREASStation.py @@ -5,7 +5,6 @@ import NuRadioReco.framework.station from NuRadioReco.modules.io.coreas import coreas from NuRadioReco.utilities import units - import logging logger = logging.getLogger('readCoREASStation') @@ -34,13 +33,13 @@ def begin(self, input_files, station_id, debug=False): @register_run() def run(self, detector): """ - Reads in all observers in the CoREAS files and returns a new simulated - event for each. + Reads in all observers in the CoREAS files and returns a new simulated event for each observer with + respect to a given detector with a single station. Parameters ---------- detector: Detector object - Detector description of the detector that shall be simulated + Detector description of the detector that shall be simulated containing one station """ for input_file in self.__input_files: diff --git a/NuRadioReco/modules/io/eventWriter.py b/NuRadioReco/modules/io/eventWriter.py index fac065eae..51454e344 100644 --- a/NuRadioReco/modules/io/eventWriter.py +++ b/NuRadioReco/modules/io/eventWriter.py @@ -12,10 +12,14 @@ def get_header(evt): header = {'stations': {}} for iS, station in enumerate(evt.get_stations()): header['stations'][station.get_id()] = station.get_parameters().copy() - if(station.has_sim_station()): + header['stations'][station.get_id( + )][stnp.station_time] = station.get_station_time_dict() + + if station.has_sim_station(): header['stations'][station.get_id()]['sim_station'] = {} - header['stations'][station.get_id()]['sim_station'] = station.get_sim_station().get_parameters() - header['stations'][station.get_id()][stnp.station_time] = station.get_station_time() + header['stations'][station.get_id()]['sim_station'] = station.get_sim_station( + ).get_parameters().copy() + header['event_id'] = (evt.get_run_number(), evt.get_id()) return header @@ -24,6 +28,7 @@ class eventWriter: """ save events to file """ + def __init__(self): # initialize attributes self.__filename = None @@ -38,10 +43,12 @@ def __init__(self): self.__event_ids_and_runs = None self.__events_per_file = None self.__events_in_current_file = 0 + self.__fout = None def __write_fout_header(self): if self.__number_of_files > 1: - self.__fout = open("{}_part{:02d}.nur".format(self.__filename, self.__number_of_files), 'wb') + self.__fout = open("{}_part{:02d}.nur".format( + self.__filename, self.__number_of_files), 'wb') else: self.__fout = open("{}.nur".format(self.__filename), 'wb') b = bytearray() @@ -69,12 +76,15 @@ def begin(self, filename, max_file_size=1024, check_for_duplicates=False, events both set, the file will be split whenever any of the two conditions is fullfilled. """ logger.setLevel(log_level) - if filename[-4:] == '.nur': + if filename.endswith(".nur"): self.__filename = filename[:-4] else: self.__filename = filename - if filename[-4:] == '.ari': - logger.warning('The file ending .ari for NuRadioReco files is deprecated. Please use .nur instead.') + + if filename.endswith('.ari'): + logger.warning( + 'The file ending .ari for NuRadioReco files is deprecated. Please use .nur instead.') + self.__check_for_duplicates = check_for_duplicates self.__number_of_events = 0 self.__current_file_size = 0 @@ -82,8 +92,10 @@ def begin(self, filename, max_file_size=1024, check_for_duplicates=False, events self.__max_file_size = max_file_size * 1024 * 1024 # in bytes self.__stored_stations = [] self.__stored_channels = [] - self.__event_ids_and_runs = [] # Remember which event IDs are already in file to catch duplicates - self.__header_written = False # Remember if we still have to write the current file header + # Remember which event IDs are already in file to catch duplicates + self.__event_ids_and_runs = [] + # Remember if we still have to write the current file header + self.__header_written = False self.__events_per_file = events_per_file @register_run() @@ -129,13 +141,16 @@ def run(self, evt, det=None, mode=None): self.__events_in_current_file += 1 if det is not None: - detector_dict = self.__get_detector_dict(evt, det) # returns None if detector is already saved + # returns None if detector is already saved + detector_dict = self.__get_detector_dict(evt, det) if detector_dict is not None: - detector_bytearray = self.__get_detector_bytearray(detector_dict) + detector_bytearray = self.__get_detector_bytearray( + detector_dict) self.__fout.write(detector_bytearray) self.__current_file_size += detector_bytearray.__sizeof__() if isinstance(det, generic_detector.GenericDetector): - changes_bytearray = self.__get_detector_changes_byte_array(evt, det) + changes_bytearray = self.__get_detector_changes_byte_array( + evt, det) if changes_bytearray is not None: self.__fout.write(changes_bytearray) self.__current_file_size += changes_bytearray.__sizeof__() @@ -143,8 +158,9 @@ def run(self, evt, det=None, mode=None): logger.debug("current file size is {} bytes, event number {}".format(self.__current_file_size, self.__number_of_events)) - if(self.__current_file_size > self.__max_file_size or self.__events_in_current_file == self.__events_per_file): - logger.info("current output file exceeds max file size -> closing current output file and opening new one") + if self.__current_file_size > self.__max_file_size or self.__events_in_current_file == self.__events_per_file: + logger.info( + "current output file exceeds max file size -> closing current output file and opening new one") self.__current_file_size = 0 self.__fout.close() self.__number_of_files += 1 @@ -202,7 +218,8 @@ def __get_detector_dict(self, event, det): for channel in station.iter_channels(): if not self.__is_channel_already_in_file(station.get_id(), channel.get_id(), station.get_station_time()): if not is_generic_detector: - channel_description = det.get_channel(station.get_id(), channel.get_id()) + channel_description = det.get_channel( + station.get_id(), channel.get_id()) self.__stored_channels.append({ 'station_id': station.get_id(), 'channel_id': channel.get_id(), @@ -210,10 +227,11 @@ def __get_detector_dict(self, event, det): 'decommission_time': channel_description['decommission_time'] }) else: - channel_description = det.get_raw_channel(station.get_id(), channel.get_id()) + channel_description = det.get_raw_channel( + station.get_id(), channel.get_id()) self.__stored_channels.append({ - 'station_id': station.get_id(), - 'channel_id': channel.get_id() + 'station_id': station.get_id(), + 'channel_id': channel.get_id() }) det_dict['channels'][str(i_channel)] = channel_description i_channel += 1 @@ -222,16 +240,20 @@ def __get_detector_dict(self, event, det): if is_generic_detector: for reference_station_id in det.get_reference_station_ids(): if not self.__is_station_already_in_file(reference_station_id, None): - station_description = det.get_raw_station(reference_station_id) + station_description = det.get_raw_station( + reference_station_id) self.__stored_stations.append({ 'station_id': reference_station_id }) - det_dict['stations'][str(i_station)] = station_description + det_dict['stations'][str( + i_station)] = station_description i_station += 1 for channel_id in det.get_channel_ids(reference_station_id): if not self.__is_channel_already_in_file(reference_station_id, channel_id, None): - channel_description = det.get_raw_channel(reference_station_id, channel_id) - det_dict['channels'][str(i_channel)] = channel_description + channel_description = det.get_raw_channel( + reference_station_id, channel_id) + det_dict['channels'][str( + i_channel)] = channel_description self.__stored_channels.append({ 'station_id': reference_station_id, 'channel_id': channel_id @@ -276,7 +298,8 @@ def __is_channel_already_in_file(self, station_id, channel_id, station_time): return False def __get_detector_changes_byte_array(self, event, det): - changes = det.get_station_properties_for_event(event.get_run_number(), event.get_id()) + changes = det.get_station_properties_for_event( + event.get_run_number(), event.get_id()) if len(changes) == 0: return None changes_string = pickle.dumps(changes, protocol=4) @@ -295,13 +318,19 @@ def __check_for_duplicate_ids(self, run_number, event_id): Checks if an event with the same ID and run number has already been written to the file and throws an error if that is the case. """ - if(self.__check_for_duplicates): + if self.__check_for_duplicates: if [run_number, event_id] in self.__event_ids_and_runs: - raise ValueError("An event with ID {} and run number {} already exists in the file\nif you don't want unique event ids enforced you can turn it of by passing `check_for_duplicates=True` to the begin method.".format(event_id, run_number)) + raise ValueError("An event with ID {} and run number {} already exists in the " + "file\nif you don't want unique event ids enforced you can " + "turn it of by passing `check_for_duplicates=True` to the " + "begin method.".format(event_id, run_number)) return def end(self): - if(hasattr(self, "__fout")): + if self.__fout is not None: self.__fout.close() - self.debug(f"closing file.") + logger.info(f"closing file {self.__filename}.") + else: + logger.warning( + f"file {self.__filename} does not exist and won't be closed.") return self.__number_of_events diff --git a/NuRadioReco/modules/io/event_parser_factory.py b/NuRadioReco/modules/io/event_parser_factory.py index 1f0a02326..abe9fd68b 100644 --- a/NuRadioReco/modules/io/event_parser_factory.py +++ b/NuRadioReco/modules/io/event_parser_factory.py @@ -9,11 +9,16 @@ def scan_files_function(version_major, version_minor): specifying the first version that function is for """ def scan_files_2_0(self, iF, current_byte): + + if self._parse_detector: + self.logger(f"Scanning a file of version 2.0 which does not contain a detector object. " + f"However, \"parse_detector\" is true.") + bytes_to_read_hex = self._get_file(iF).read(6) bytes_to_read = int.from_bytes(bytes_to_read_hex, 'little') - if(bytes_to_read == 0): + if bytes_to_read == 0: # we are at the end of the file - if(iF < (len(self._filenames) - 1)): # are there more files to be parsed? + if iF < (len(self._filenames) - 1): # are there more files to be parsed? iF += 1 current_byte = 12 # skip datafile header self._get_file(iF).seek(current_byte) @@ -25,6 +30,7 @@ def scan_files_2_0(self, iF, current_byte): self._bytes_length.append([]) else: return False, iF, current_byte + current_byte += 6 self._bytes_start_header[iF].append(current_byte) self._bytes_length_header[iF].append(bytes_to_read) @@ -48,9 +54,9 @@ def scan_files_2_2(self, iF, current_byte): current_byte += 6 bytes_to_read_hex = self._get_file(iF).read(6) bytes_to_read = int.from_bytes(bytes_to_read_hex, 'little') - if(bytes_to_read == 0): + if bytes_to_read == 0: # we are at the end of the file - if(iF < (len(self._filenames) - 1)): # are there more files to be parsed? + if iF < (len(self._filenames) - 1): # are there more files to be parsed? iF += 1 current_byte = 12 # skip datafile header self._get_file(iF).seek(current_byte) @@ -65,8 +71,11 @@ def scan_files_2_2(self, iF, current_byte): current_byte += 6 else: return False, iF, current_byte + current_byte += 6 if object_type == 0: # object is an event + self.logger.debug("Read Event ...") + self._bytes_start_header[iF].append(current_byte) self._bytes_length_header[iF].append(bytes_to_read) current_byte += bytes_to_read @@ -80,18 +89,23 @@ def scan_files_2_2(self, iF, current_byte): bytes_to_read = int.from_bytes(bytes_to_read_hex, 'little') self._bytes_start[iF].append(current_byte) self._bytes_length[iF].append(bytes_to_read) - elif object_type == 1: # object is detector info + + elif object_type == 1 and self._parse_detector: # object is detector info + self.logger.debug("Read detector ...") + detector_dict = pickle.loads(self._get_file(iF).read(bytes_to_read)) if 'generic_detector' not in detector_dict.keys(): is_generic_detector = False else: is_generic_detector = detector_dict['generic_detector'] + if iF not in self._detector_dicts.keys(): self._detector_dicts[iF] = { 'generic_detector': is_generic_detector, 'channels': {}, 'stations': {} } + if is_generic_detector: # add default_station and default_channel to the dict to support older files using these if 'default_station' not in detector_dict: @@ -100,24 +114,28 @@ def scan_files_2_2(self, iF, current_byte): detector_dict['default_channel'] = None self._detector_dicts[iF]['default_station'] = detector_dict['default_station'] self._detector_dicts[iF]['default_channel'] = detector_dict['default_channel'] + for station in detector_dict['stations'].values(): if len(self._detector_dicts[iF]['stations'].keys()) == 0: index = 0 else: index = max(self._detector_dicts[iF]['stations'].keys()) + 1 self._detector_dicts[iF]['stations'][index] = station + for channel in detector_dict['channels'].values(): if len(self._detector_dicts[iF]['channels'].keys()) == 0: index = 0 else: index = max(self._detector_dicts[iF]['channels'].keys()) + 1 self._detector_dicts[iF]['channels'][index] = channel + elif object_type == 2: # object is list of event-specific changes to the detector changes_dict = pickle.loads(self._get_file(iF).read(bytes_to_read)) if iF not in self._event_specific_detector_changes.keys(): self._event_specific_detector_changes[iF] = [] for change in changes_dict: self._event_specific_detector_changes[iF].append(change) + current_byte += bytes_to_read return True, iF, current_byte @@ -126,10 +144,12 @@ def scan_files_2_2(self, iF, current_byte): return scan_files_2_0 else: return scan_files_2_2 + elif version_major == 0: raise ValueError(f'File version is {version_major}.{version_major} which might indicate the file is empty') else: - raise ValueError('File version {}.{} is not supported. Major version needs to be 2 but is {}.'.format(version_major, version_minor, version_major)) + raise ValueError('File version {}.{} is not supported. Major version needs to be 2 but is {}.'.format( + version_major, version_minor, version_major)) diff --git a/NuRadioReco/modules/io/noise/noiseImporter.py b/NuRadioReco/modules/io/noise/noiseImporter.py deleted file mode 100644 index 85f483526..000000000 --- a/NuRadioReco/modules/io/noise/noiseImporter.py +++ /dev/null @@ -1,140 +0,0 @@ -from NuRadioReco.modules.base.module import register_run -from NuRadioReco.utilities import units -import numpy as np -import logging -import glob -import os -import sys -from NuRadioReco.modules.io import NuRadioRecoio -logger = logging.getLogger('noiseImporter') - - -class noiseImporter: - """ - Imports recorded noise from ARIANNA station. The recorded noise needs to match the station geometry and sampling - as chosen with channelResampler and channelLengthAdjuster - - For different stations, new noise files need to be used. - Collect forced triggers from any type of station to use for analysis. - A seizable fraction of data is recommended for accuracy. - - The noise will be random. This module therefore might produce non-reproducible results on a single event basis, - if run several times. - """ - - def __init__(self): - self.__channel_mapping = None - self.__station_id = None - self.__mean_opt = None - self.__noise_files = None - self.__open_files = None - self.__n_tot = None - - def begin(self, noise_folder, station_id=None, noise_files=None, - channel_mapping=None, log_level=logging.WARNING, mean_opt=True): - """ - Parameters - ---------- - noise_folder: string - the folder containing the noise files - station_id: int - the station id, specifies from which station the forced triggers are used - as a noise sample. The data must have the naming convention 'forced_station_.nur' - where is replaced with the station id. If station_id is None, the noiseImporter - will try to find the station with the same iD as the one passed to the run - function in the noise file - noise_files: list of strings (default: None) - List of noisefiles. If None is passed, all files in the noise_folder are used - channel_mapping: dict or None - option relevant for MC studies of new station designs where we do not - have forced triggers for. The channel_mapping dictionary maps the channel - ids of the MC station to the channel ids of the noise data - Default is None which is 1-to-1 mapping - log_level: loggging log level - the log level, default logging.WARNING - mean_opt: boolean - option to subtract mean from trace. Set mean_opt=False to remove mean subtraction from trace. - """ - logger.setLevel(log_level) - self.__channel_mapping = channel_mapping - self.__mean_opt = mean_opt - if(noise_files is not None): - self.__noise_files = noise_files - else: - if(station_id is None): - logger.error("noise_files and station_id can't be both None") - sys.exit(-1) - else: - self.__noise_files = glob.glob(os.path.join(noise_folder, "forced_station_{}.nur".format(station_id))) - if(len(self.__noise_files) == 0): - logger.error("no noise files found for station {} in folder {}".format(station_id, noise_folder)) - sys.exit(-1) - - # open files and scan for number of events - self.__open_files = [] - self.__n_tot = 0 - for file_name in self.__noise_files: - f = NuRadioRecoio.NuRadioRecoio(file_name, parse_header=False) - n = f.get_n_events() - self.__open_files.append({'f': f, 'n_low': self.__n_tot, - 'n_high': self.__n_tot + n - 1}) - self.__n_tot += n - - def __get_noise_event(self, i): - for f in self.__open_files: - if(f['n_low'] <= i <= f['n_high']): - return f['f'].get_event_i(i - f['n_low']) - - def __get_noise_channel(self, channel_id): - if(self.__channel_mapping is None): - return channel_id - else: - return self.__channel_mapping[channel_id] - - @register_run() - def run(self, evt, station, det): - # loop over stations in simulation - i_noise = np.random.randint(0, self.__n_tot) - noise_event = self.__get_noise_event(i_noise) - if self.__station_id is None: - station_id = station.get_id() - else: - station_id = self.__station_id - noise_station = noise_event.get_station(station_id) - if noise_station is None: - raise KeyError('Station whith ID {} not found in noise file'.format(station_id)) - logger.info("choosing noise event {} ({}, run {}, event {}) randomly".format(i_noise, noise_station.get_station_time(), - noise_event.get_run_number(), - noise_event.get_id())) - - for channel in station.iter_channels(): - channel_id = channel.get_id() - - trace = channel.get_trace() - noise_channel = noise_station.get_channel(self.__get_noise_channel(channel_id)) - noise_trace = noise_channel.get_trace() - # check if trace has the same size - if (len(trace) != len(noise_trace)): - logger.error("Mismatch: Noise has {0} and simulation {1} samples".format(len(noise_trace), len(trace))) - sys.exit(-1) - # check sampling rate - if (channel.get_sampling_rate() != noise_channel.get_sampling_rate()): - logger.error("Mismatch in sampling rate: Noise has {0} and simulation {1} GHz".format( - noise_channel.get_sampling_rate() / units.GHz, channel.get_sampling_rate() / units.GHz)) - sys.exit(-1) - - trace = trace + noise_trace - - if self.__mean_opt: - mean = noise_trace.mean() - std = noise_trace.std() - if(mean > 0.05 * std): - logger.warning( - "the noise trace has an offset of {:.2}mV which is more than 5% of the STD of {:.2f}mV. The module corrects for the offset but it might points to an error in the FPN subtraction.".format(mean, std)) - trace = trace - mean - - channel.set_trace(trace, channel.get_sampling_rate()) - - def end(self): - for f in self.__open_files: - f['f'].close_file() diff --git a/NuRadioReco/modules/io/rno_g/readRNOGData.py b/NuRadioReco/modules/io/rno_g/readRNOGData.py deleted file mode 100755 index e1fc23d08..000000000 --- a/NuRadioReco/modules/io/rno_g/readRNOGData.py +++ /dev/null @@ -1,224 +0,0 @@ -import NuRadioReco.framework.event -from NuRadioReco.modules.base.module import register_run -import NuRadioReco.framework.station -import NuRadioReco.framework.channel -import NuRadioReco.framework.trigger -import NuRadioReco.modules.channelSignalReconstructor -signal_reconstructor = NuRadioReco.modules.channelSignalReconstructor.channelSignalReconstructor() - -import uproot -# default uproot readout is awkward arrays, but requires awkward package installed. RNOG data format does not require this. Use numpy instead. -uproot.default_library = "np" - -import numpy as np -from NuRadioReco.utilities import units -import sys -import os -import logging -import time -from scipy import interpolate -import six -from collections import OrderedDict -import astropy.time - - -class readRNOGData: - """ - This is the data reader for RNO-G. Reads RNO-G data from ROOT format using uproot - """ - def __init__(self): - self.logger = logging.getLogger("NuRadioReco.readRNOGdata") - self.__id_current_event = None - self.__t = None - self.__sampling_rate = 3.2 * units.GHz #TODO: 3.2 at the beginning of deployment. Will change to 2.4 GHz after firmware update eventually, but info not yet contained in the .root files. Read out once available. - self._iterator_data = None - self._iterator_header = None - self._data_treename = "waveforms" - self._header_treename = "header" - self.n_events = None - self.input_files = [] - - def begin(self, input_files, input_files_header=None): - - """ - Begin function of the RNO-G reader - - Parameters - ---------- - input_files: list of paths to files containing waveforms - input_files_header: list of paths to files containing header, - if None, headers are expected to be stored in the input_files also - """ - - self.__id_current_event = -1 - self.__t = time.time() - - if isinstance(input_files, six.string_types): - input_files = [input_files] - if isinstance(input_files_header, six.string_types): - input_files_header = [input_files_header] - if input_files_header is None: - input_files_header = input_files - - self.input_files = input_files - self.input_files_header = input_files_header - - self.n_events = 0 - # get the total number of events of all input files - for filename in input_files: - file = uproot.open(filename) - if 'combined' in file: - file = file['combined'] - self.n_events += file[self._data_treename].num_entries - self._set_iterators() - - return self.n_events - - def _set_iterators(self, cut=None): - """ - Set uproot iterators to loop over event trees - - Parameters - ---------- - cut: str - cut string to apply (e.g. for initial event selection based on event_number, ... - e.g. "(event_number==1)" or "(run_number==1)&(event_number<10)" - """ - self.__id_current_event = -1 - - datadict = OrderedDict() - for filename in self.input_files: - if 'combined' in uproot.open(filename): - datadict[filename] = 'combined/' + self._data_treename - else: - datadict[filename] = self._data_treename - - headerdict = OrderedDict() - for filename in self.input_files_header: - if 'combined' in uproot.open(filename): - headerdict[filename] = 'combined/' + self._header_treename - else: - headerdict[filename] = self._header_treename - - # iterator over single events (step 1), for event looping in NuRadioReco dataformat - # may restrict which data to read in the iterator by adding second argument - # read_branches = ['run_number', 'event_number', 'station_number', 'radiant_data[24][2048]'] - self._iterator_data = uproot.iterate(datadict, cut=cut,step_size=1, how=dict, library="np") - self._iterator_header = uproot.iterate(headerdict, cut=cut, step_size=1, how=dict, library="np") - - self.uproot_iterator_data = uproot.iterate(datadict, cut=cut, step_size=1000) - self.uproot_iterator_header = uproot.iterate(headerdict, cut=cut, step_size=1000) - - @register_run() - def run(self, channels=np.arange(24), event_numbers=None, run_numbers=None, cut_string=None): - """ - Run function of the RNOG reader - - Parameters - ---------- - n_channels: int - number of RNOG channels to loop over, default 24 - - event_numbers: None or dict - if dict, use a dict with run number as key and list of event numbers as items - - run_numbers: None or list - list of run numbers to select - Caveat: use only if event_numbers are not set - - cut_string: string - selection string for event pre-selection - Cavieat: use only if event_numbers and run_numbers are not set - """ - - # generate cut string based on passed event_numbers or run_numbers parameters - if not run_numbers is None: - event_cuts = "|".join(["(run_number==%i)" for run_number in run_numbers]) - cut_string = "|".join(event_cuts) - if not event_numbers is None: - event_cuts = [] - for run in event_numbers: - events = event_numbers[run] - for event in events: - event_cuts.append("(run_number==%i)&(event_number==%i)" %(run, event)) - cut_string = "|".join(event_cuts) - self.cut_string = cut_string - - self._set_iterators(cut=self.cut_string) - root_trigger_keys = [ - 'trigger_info.rf_trigger', 'trigger_info.force_trigger', - 'trigger_info.pps_trigger', 'trigger_info.ext_trigger', - 'trigger_info.radiant_trigger', 'trigger_info.lt_trigger', - 'trigger_info.surface_trigger' - ] - self.__t = time.time() - # Note: reading single events is inefficient... - # for event_header, event in zip(self._iterator_header, self._iterator_data): - for event_headers, events in zip(self.uproot_iterator_header, self.uproot_iterator_data): - for event_header, event in zip(event_headers, events): - self.__id_current_event += 1 - #if self.__id_current_event >= self.n_events: - # # all events processed, but iterator should stop before anyways. - # break - if self.__id_current_event % 1000 == 0: - progress = 1. * self.__id_current_event / self.n_events - eta = 0 - if self.__id_current_event > 0: - eta = (time.time() - self.__t) / self.__id_current_event * (self.n_events - self.__id_current_event) / 60. - self.logger.warning("reading in event {}/{} ({:.0f}%) ETA: {:.1f} minutes".format(self.__id_current_event, self.n_events, 100 * progress, eta)) - - run_number = event["run_number"] - evt_number = event["event_number"] - station_id = event_header["station_number"] - self.logger.info("Reading Run: {run_number}, Event {evt_number}, Station {station_id}") - - evt = NuRadioReco.framework.event.Event(run_number, evt_number) - station = NuRadioReco.framework.station.Station(station_id) - #TODO in future: do need to apply calibrations? - - unix_time = event_header["trigger_time"] - event_time = astropy.time.Time(unix_time, format='unix') - - station.set_station_time(event_time) - for trigger_key in root_trigger_keys: - try: - has_triggered = bool(event_header[trigger_key]) - trigger = NuRadioReco.framework.trigger.Trigger(trigger_key.split('.')[-1]) - trigger.set_triggered(has_triggered) - station.set_trigger(trigger) - except ValueError: - pass - - radiant_data = event["radiant_data[24][2048]"] # returns array of n_channels, n_points - # Loop over all requested channels in data - for chan in channels: - channel = NuRadioReco.framework.channel.Channel(chan) - - # Get data from array via graph method - voltage = np.array(radiant_data[chan]) * units.mV - #times = np.arange(len(voltage)) * sampling - - if voltage.shape[0] % 2 != 0: - voltage = voltage[:-1] - - #TODO: need to subtract mean... probably not if running signal reconstructor? - #channel.set_trace(voltage-np.mean(voltage), sampling_rate) - channel.set_trace(voltage, self.__sampling_rate) - station.add_channel(channel) - evt.set_station(station) - # we want to have access to basic signal quantities with implementation from NuRadioReco - #TODO: maybe this should be run in external module? - signal_reconstructor.run(evt, station, None) - yield evt - - def get_events(self): - return self.run() - - def get_n_events(self): - return self.n_events - - def get_filenames(self): - return self.input_files - - def end(self): - pass diff --git a/NuRadioReco/modules/io/rno_g/rnogDataReader.py b/NuRadioReco/modules/io/rno_g/rnogDataReader.py deleted file mode 100644 index cea0be080..000000000 --- a/NuRadioReco/modules/io/rno_g/rnogDataReader.py +++ /dev/null @@ -1,126 +0,0 @@ -import numpy as np -import uproot -import NuRadioReco.framework.event -import NuRadioReco.framework.station -import NuRadioReco.framework.channel -from NuRadioReco.utilities import units -import astropy.time -import glob -import logging -import time -from functools import lru_cache -import six -import NuRadioReco.utilities.metaclasses - -logger = logging.getLogger("RNO-G_IO") -# logger.setLevel(logging.DEBUG) - -# @six.add_metaclass(NuRadioReco.utilities.metaclasses.Singleton) # maybe? -class RNOGDataReader: - - def __init__(self, filenames, *args, **kwargs): - logger.debug("Initializing RNOGDataReader") - self.__filenames = filenames - self.__event_ids = None - self.__sampling_rate = 3.2 * units.GHz #TODO: 3.2 at the beginning of deployment. Will change to 2.4 GHz after firmware update eventually, but info not yet contained in the .root files. Read out once available. - self.__parse_event_ids() - self.__i_events_per_file = np.zeros((len(self.__filenames), 2), dtype=int) - i_event = 0 - for i_file, filename in enumerate(filenames): - file = self.__open_file(filename) - events_in_file = file['waveforms'].num_entries - self.__i_events_per_file[i_file] = [i_event, i_event + events_in_file] - i_event += events_in_file - - self._root_trigger_keys = [ - 'trigger_info.rf_trigger', 'trigger_info.force_trigger', - 'trigger_info.pps_trigger', 'trigger_info.ext_trigger', - 'trigger_info.radiant_trigger', 'trigger_info.lt_trigger', - 'trigger_info.surface_trigger' - ] - - def get_filenames(self): - return self.__filenames - - def get_event_ids(self): - if self.__event_ids is None: - return self.__parse_event_ids() - return self.__event_ids - - def __parse_event_ids(self): - logger.debug('Parsing event ids') - event_ids = np.array([], dtype=int) - run_numbers = np.array([], dtype=int) - for filename in self.__filenames: - file = self.__open_file(filename) - event_ids = np.append(event_ids, file['waveforms']['event_number'].array(library='np').astype(int)) - run_numbers = np.append(run_numbers, file['header']['run_number'].array(library='np').astype(int)) - self.__event_ids = np.array([run_numbers, event_ids]).T - - def __open_file(self, filename): - logger.debug("Opening file {}".format(filename)) - file = uproot.open(filename) - if 'combined' in file: - file = file['combined'] - return file - - def get_n_events(self): - return self.get_event_ids().shape[0] - - # @lru_cache(maxsize=1) # probably not actually relevant outside the data viewer? - def get_event_i(self, i_event): - read_time = time.time() - event = NuRadioReco.framework.event.Event(*self.get_event_ids()[i_event]) - for i_file, filename in enumerate(self.__filenames): - if self.__i_events_per_file[i_file, 0] <= i_event < self.__i_events_per_file[i_file, 1]: - i_event_in_file = i_event - self.__i_events_per_file[i_file, 0] - file = self.__open_file(self.__filenames[i_file]) - station = NuRadioReco.framework.station.Station((file['waveforms']['station_number'].array(library='np', entry_start=i_event_in_file, entry_stop=(i_event_in_file+1))[0])) - # station not set properly in first runs, try from header - if station.get_id() == 0 and 'header' in file: - station = NuRadioReco.framework.station.Station((file['header']['station_number'].array(library='np', entry_start=i_event_in_file, entry_stop=(i_event_in_file+1))[0])) - station.set_is_neutrino() - - if 'header' in file: - unix_time = file['header']['readout_time'].array(library='np', entry_start=i_event_in_file, entry_stop=(i_event_in_file+1))[0] - event_time = astropy.time.Time(unix_time, format='unix') - station.set_station_time(event_time) - ### read in basic trigger data - for trigger_key in self._root_trigger_keys: - try: - has_triggered = bool(file['header'][trigger_key].array(library='np', entry_start=i_event_in_file, entry_stop=(i_event_in_file+1))[0]) - trigger = NuRadioReco.framework.trigger.Trigger(trigger_key.split('.')[-1]) - trigger.set_triggered(has_triggered) - # trigger.set_trigger_time(file['header']['trigger_time']) - station.set_trigger(trigger) - except uproot.exceptions.KeyInFileError: - pass - - waveforms = file['waveforms']['radiant_data[24][2048]'].array(library='np', entry_start=i_event_in_file, entry_stop=(i_event_in_file+1)) - for i_channel in range(waveforms.shape[1]): - channel = NuRadioReco.framework.channel.Channel(i_channel) - channel.set_trace(waveforms[0, i_channel]*units.mV, self.__sampling_rate) - station.add_channel(channel) - event.set_station(station) - logger.debug("Spent {:.0f} ms reading event {}".format((time.time()-read_time) * 1e3, i_event)) - return event - return None - - def get_event(self, event_id): - find_event = np.where((self.get_event_ids()[:,0] == event_id[0]) & (self.get_event_ids()[:,1] == event_id[1]))[0] - if len(find_event) == 0: - return None - elif len(find_event) == 1: - return self.get_event_i(find_event[0]) - else: - raise RuntimeError('There are multiple events with the ID [{}, {}] in the file'.format(event_id[0], event_id[1])) - - def get_events(self): - for ev_i in range(self.get_n_events()): - yield self.get_event_i(ev_i) - - def get_detector(self): - return None - - def get_header(self): - return None diff --git a/NuRadioReco/modules/io/rno_g/__init__.py b/NuRadioReco/modules/measured_noise/ARIANNA/__init__.py similarity index 100% rename from NuRadioReco/modules/io/rno_g/__init__.py rename to NuRadioReco/modules/measured_noise/ARIANNA/__init__.py diff --git a/NuRadioReco/modules/io/noise/noiseImporterROOT.py b/NuRadioReco/modules/measured_noise/ARIANNA/noiseImporterROOT.py similarity index 97% rename from NuRadioReco/modules/io/noise/noiseImporterROOT.py rename to NuRadioReco/modules/measured_noise/ARIANNA/noiseImporterROOT.py index 23645700d..ab4e32f6a 100644 --- a/NuRadioReco/modules/io/noise/noiseImporterROOT.py +++ b/NuRadioReco/modules/measured_noise/ARIANNA/noiseImporterROOT.py @@ -4,7 +4,7 @@ from NuRadioReco.utilities import units import numpy as np import logging -logger = logging.getLogger('noiseImporter') +logger = logging.getLogger('noiseImporterARIANNAROOT') class noiseImporter: diff --git a/NuRadioReco/modules/measured_noise/RNO_G/__init__.py b/NuRadioReco/modules/measured_noise/RNO_G/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/NuRadioReco/modules/measured_noise/RNO_G/noiseImporter.py b/NuRadioReco/modules/measured_noise/RNO_G/noiseImporter.py new file mode 100644 index 000000000..c0c781c8b --- /dev/null +++ b/NuRadioReco/modules/measured_noise/RNO_G/noiseImporter.py @@ -0,0 +1,220 @@ +import numpy as np +import glob +import os +import collections +import time + +from NuRadioReco.modules.io.RNO_G.readRNOGDataMattak import readRNOGData +from NuRadioReco.modules.base.module import register_run +from NuRadioReco.utilities import units + +import logging + + +class noiseImporter: + """ + Imports recorded traces from RNOG stations. + + """ + + + def begin(self, noise_folders, file_pattern="*", + match_station_id=False, station_ids=None, + channel_mapping=None, scramble_noise_file_order=True, + log_level=logging.INFO, random_seed=None, reader_kwargs={}): + """ + + Parameters + ---------- + noise_folders: str or list(str) + Folder(s) containing noise file(s). Search in any subfolder as well. + + file_pattern: str + File pattern used to search for directories, (Default: "*", other examples might be "combined") + + match_station_id: bool + If True, add only noise from stations with the same id. (Default: False) + + station_ids: list(int) + Only add noise from those station ids. If None, use any station. (Default: None) + + channel_mapping: dict or None + option relevant for MC studies of new station designs where we do not + have forced triggers for. The channel_mapping dictionary maps the channel + ids of the MC station to the channel ids of the noise data + Default is None which is 1-to-1 mapping + + scramble_noise_file_order: bool + If True, randomize the order of noise files before reading them. (Default: True) + + log_level: loggging log level + The log level to controll verbosity. (Default: logging.INFO) + + random_seed: int + Seed for the random number generator. (Default: None, no fixed seed). + + reader_kwargs: dict + Optional arguements passed to readRNOGDataMattak + """ + + self.logger = logging.getLogger('NuRadioReco.RNOG.noiseImporter') + self.logger.setLevel(log_level) + self.__random_gen = np.random.Generator(np.random.Philox(random_seed)) + + self._match_station_id = match_station_id + self.__station_ids = station_ids + + self.__channel_mapping = channel_mapping + + self.logger.info(f"\n\tMatch station id: {match_station_id}" + f"\n\tUse noise from only those stations: {station_ids}" + f"\n\tUse the following channel mapping: {channel_mapping}" + f"\n\tRandomize sequence of noise files: {scramble_noise_file_order}") + + if not isinstance(noise_folders, list): + noise_folders = [noise_folders] + + # find all subfolders + noise_files = [] + for noise_folder in noise_folders: + if noise_folder == "": + continue + + noise_files += glob.glob(f"{noise_folder}/**/{file_pattern}root", recursive=True) + self.__noise_folders = np.unique([os.path.dirname(e) for e in noise_files]) + + self.logger.info(f"Found {len(self.__noise_folders)}") + if not len(self.__noise_folders): + self.logger.error("No folders found") + raise FileNotFoundError("No folders found") + + if scramble_noise_file_order: + self.__random_gen.shuffle(self.__noise_folders) + + self._noise_reader = readRNOGData() + + default_reader_kwargs = { + "selectors": [lambda einfo: einfo.triggerType == "FORCE"], + "log_level": log_level, "select_runs": True, "max_trigger_rate": 2 * units.Hz, + "run_types": ["physics"] + } + default_reader_kwargs.update(reader_kwargs) + + self._noise_reader.begin(self.__noise_folders, **default_reader_kwargs) + + # instead of reading all noise events into memory we only get certain information here and read all data in run() + self.logger.info("Get event informations ...") + t0 = time.time() + noise_information = self._noise_reader.get_events_information(keys=["station"]) + self.logger.info(f"... of {len(noise_information)} (selected) events in {time.time() - t0:.2f}s") + + self.__event_index_list = np.array(list(noise_information.keys())) + self.__station_id_list = np.array([ele["station"] for ele in noise_information.values()]) + + self._n_use_event = collections.defaultdict(int) + + + def __get_noise_channel(self, channel_id): + if self.__channel_mapping is None: + return channel_id + else: + return self.__channel_mapping[channel_id] + + + def __draw_noise_event(self, mask): + """ + reader.get_event_by_index can return None when, e.g., the trigger time is inf or the sampling rate 0. + Hence, try again if that happens (should only occur rearly). + + Parameters + ---------- + + mask: np.array(bool) + Mask of which noise events are allowed (e.g. because of matching station ids, ...) + + Returns + ------- + + noise_event: NuRadioReco.framework.event + A event containing noise traces + + i_noise: int + The index of the drawn event + """ + tries = 0 + while tries < 100: + # int(..) necessary because pyroot can not handle np.int64 + i_noise = int(self.__random_gen.choice(self.__event_index_list[mask])) + noise_event = self._noise_reader.get_event_by_index(i_noise) + tries += 1 + if noise_event is not None: + break + + if noise_event is None: + err = "Could not draw a random station which is not None after 100 tries. Stop." + self.logger.error(err) + raise ValueError(err) + + self._n_use_event[i_noise] += 1 + return noise_event, i_noise + + + @register_run() + def run(self, evt, station, det): + + if self._match_station_id: + # select only noise events from simulated station id + station_mask = self.__station_id_list == station.get_id() + if not np.any(station_mask): + raise ValueError(f"No station with id {station.get_id()} in noise data.") + + else: + # select all noise events + station_mask = np.full_like(self.__event_index_list, True) + + noise_event, i_noise = self.__draw_noise_event(station_mask) + + station_id = noise_event.get_station_ids()[0] + noise_station = noise_event.get_station(station_id) + + if self.__station_ids is not None and not station_id in self.__station_ids: + raise ValueError(f"Station id {station_id} not in list of allowed ids: {self.__station_ids}") + + self.logger.debug("Selected noise event {} ({}, run {}, event {})".format( + i_noise, noise_station.get_station_time(), noise_event.get_run_number(), + noise_event.get_id())) + + for channel in station.iter_channels(): + channel_id = channel.get_id() + + trace = channel.get_trace() + noise_channel = noise_station.get_channel(self.__get_noise_channel(channel_id)) + noise_trace = noise_channel.get_trace() + + if len(trace) > 2048: + self.logger.warn("Simulated trace is longer than 2048 bins... trim with :2048") + trace = trace[:2048] + + # sanity checks + if len(trace) != len(noise_trace): + erg_msg = f"Mismatch in trace lenght: Noise has {len(noise_trace)} " + \ + "and simulation has {len(trace)} samples" + self.logger.error(erg_msg) + raise ValueError(erg_msg) + + if channel.get_sampling_rate() != noise_channel.get_sampling_rate(): + erg_msg = "Mismatch in sampling rate: Noise has {} and simulation has {} GHz".format( + noise_channel.get_sampling_rate() / units.GHz, channel.get_sampling_rate() / units.GHz) + self.logger.error(erg_msg) + raise ValueError(erg_msg) + + trace = trace + noise_trace + channel.set_trace(trace, channel.get_sampling_rate()) + + def end(self): + self._noise_reader.end() + n_use = np.array(list(self._n_use_event.values())) + sort = np.flip(np.argsort(n_use)) + self.logger.info("\n\tThe five most used noise events have been used: {}" + .format(", ".join([str(ele) for ele in n_use[sort][:5]]))) + pass diff --git a/NuRadioReco/modules/measured_noise/__init__.py b/NuRadioReco/modules/measured_noise/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/NuRadioReco/modules/measured_noise/channelMeasuredNoiseAdder.py b/NuRadioReco/modules/measured_noise/channelMeasuredNoiseAdder.py new file mode 100644 index 000000000..b09c58a57 --- /dev/null +++ b/NuRadioReco/modules/measured_noise/channelMeasuredNoiseAdder.py @@ -0,0 +1,276 @@ +import numpy as np +import logging +import glob + +from NuRadioReco.modules.base.module import register_run +import NuRadioReco.modules.io.NuRadioRecoio +from NuRadioReco.utilities import units + + +class channelMeasuredNoiseAdder: + """ + Module that adds measured noise to channel traces + It does so by reading in a set of .nur files, randomly selecting a forced trigger event + and adding the noise from it to the channel waveforms. + The waveforms from the channels in the noise files need to be at least as long as the + waveforms to which the noise is added, so it is recommended to cut them to the right size + first, for example using the channelLengthAdjuster. + """ + def __init__(self): + self.__filenames = None + self.__io = None + self.__random_state = None + self.__max_iterations = None + self.logger = logging.getLogger('NuRadioReco.channelMeasuredNoiseAdder') + self.__noise_data = None + + + def begin(self, filenames=None, folder=None, file_pattern="*", + random_seed=None, max_iterations=100, debug=False, + draw_noise_statistics=False, channel_mapping=None, log_level=logging.WARNING, + restrict_station_id=True, station_id=None, allow_noise_resampling=False, + baseline_substraction=True, allowed_triggers=["FORCE"]): + """ + Set up module parameters + + Parameters + ---------- + filenames: list of strings + List of .nur files containing the measured noise. If None, look for .nur files in "folder". + (Default: None) + + folder: str + Only used when "filenames" is None. Directory to search for .nur files matching the "file_pattern" + including subdirectories. (Default: None) + + file_pattern: str + Use ``glob.glob(f"{folder}/**/{file_pattern}.nur", recursive=True)`` to search for files. (Default: "*") + + random_seed: int, default: None + Seed for the random number generator. By default, no seed is set. + + max_iterations: int, default: 100 + The module will pick a random event from the noise files, until a suitable event is found + or until the number of iterations exceeds max_iterations. In that case, an error is thrown. + + debug: bool, default: False + Set True to get debug output + + draw_noise_statistics: boolean, default: False + If true, the values of all samples is stored and a histogram with noise statistics is drawn + be the end() method + + channel_mapping: dict or None + option relevant for MC studies of new station designs where we do not + have forced triggers for. The channel_mapping dictionary maps the channel + ids of the MC station to the channel ids of the noise data + Default is None which is 1-to-1 mapping + + log_level: loggging log level + the log level, default logging.WARNING + + baseline_substraction: boolean + Option to subtract mean from trace. Set mean_opt=False to remove mean subtraction from trace. + + restrict_station_id: bool + Require the station in the noise event to be the same as in the simulated event. (Default: True) + + station_id: int + If restrict_station_id is False specify the station id to be used for the noise. If None take first station + in the noise event. (Default: None) + + allow_noise_resampling: bool + Allow resampling the noise trace to match the simulated trace. (Default: False) + + allowed_triggers: list(str) + List of trigger names which should be used, events with other triggers are not used. (Default: ["FORCE"]) + """ + + self.logger.setLevel(log_level) + + self.__filenames = filenames + if self.__filenames is None: + if folder is None: + err = "Both, \"filenames\" and \"folder\" are None, you have to specify at least one ..." + self.logger.error(err) + raise ValueError(err) + + self.__filenames = glob.glob(f"{folder}/**/{file_pattern}.nur", recursive=True) + + self.logger.info(f"Found {len(self.__filenames)} noise file(s) ...") + + self.__io = NuRadioReco.modules.io.NuRadioRecoio.NuRadioRecoio(self.__filenames) + self.__random_state = np.random.Generator(np.random.Philox(random_seed)) + self.__max_iterations = max_iterations + + self.__channel_mapping = channel_mapping + self.__baseline_substraction = baseline_substraction + + self.__restrict_station_id = restrict_station_id + self.__noise_station_id = station_id + self.__allow_noise_resampling = allow_noise_resampling + + self._allowed_triggers = allowed_triggers + + if debug: + self.logger.setLevel(logging.DEBUG) + self.logger.debug('Reading noise from {} files containing {} events'.format(len(filenames), self.__io.get_n_events())) + + if draw_noise_statistics: + self.__noise_data = [] + + + def __get_noise_channel(self, channel_id): + if self.__channel_mapping is None: + return channel_id + else: + return self.__channel_mapping[channel_id] + + + @register_run() + def run(self, event, station, det): + """ + Add measured noise to station channels + + Parameters + ---------- + event: event object + station: station object + det: detector description + """ + noise_station = None + + for _ in range(self.__max_iterations): + noise_station = self.get_noise_station(station) + # Get random station from noise file. If we got a suitable station, we continue, + # otherwise we try again + if noise_station is not None: + break + + # To avoid infinite loops, if no suitable noise station was found after a number of iterations we raise an error + if noise_station is None: + raise ValueError('Could not find suitable noise event in noise files after {} iterations'.format(self.__max_iterations)) + + for channel in station.iter_channels(): + noise_channel = noise_station.get_channel(self.__get_noise_channel(channel.get_id())) + channel_trace = channel.get_trace() + + # if resampling is not desired no channel with wrong sampling rate is selected in get_noise_station() + resampled = False + if noise_channel.get_sampling_rate() != channel.get_sampling_rate(): + noise_channel.resample(channel.get_sampling_rate()) + resampled = True + + noise_trace = noise_channel.get_trace() + + if self.__baseline_substraction: + mean = noise_trace.mean() + std = noise_trace.std() + if mean > 0.05 * std: + self.logger.warning(( + "The noise trace has an offset/baseline of {:.3f}mV which is more than 5% of the STD of {:.3f}mV. " + "The module corrects for the offset but it might points to an error in the FPN subtraction.").format(mean / units.mV, std / units.mV)) + + noise_trace -= mean + + if len(channel_trace) > len(noise_trace): + err = "{} is shorter than simulated trace. Stop".format("Resampled noise trace" if resampled else "Noise trace") + self.logger.error(err) + raise ValueError(err) + elif len(channel_trace) < len(noise_trace): + self.logger.warn("Noise trace has more samples than the simulated one, clip noise trace at the end ...") + noise_trace = noise_trace[:channel.get_number_of_samples()] # if to long, clip the end + else: + pass + + channel_trace += noise_trace + + # if draw_noise_statistics == True + if self.__noise_data is not None: + self.__noise_data.append(noise_trace) + + + def get_noise_station(self, station): + """ + Returns a random station from the noise files that can be used as a noise sample. + The function selects a random event from the noise files and checks if it is suitable. + If it is, the station is returned, otherwise None is returned. The event is suitable if it + fulfills these criteria: + + * It contains a station with the same station ID as the one to which the noise shall be added + * The station does not have a trigger that has triggered. + * The every channel in the station to which the noise shall be added is also present in the station + + Parameters + ---------- + station: Station class + The station to which the noise shall be added + """ + event_i = self.__random_state.integers(0, self.__io.get_n_events()) + noise_event = self.__io.get_event_i(event_i) + + if self.__restrict_station_id and station.get_id() not in noise_event.get_station_ids(): + self.logger.debug('Event {}: No station with ID {} found.'.format(event_i, station.get_id())) + return None + + if self.__restrict_station_id: + noise_station = noise_event.get_station(station.get_id()) + else: + # If __noise_station_id == None get first station stored in event + noise_station = noise_event.get_station(self.__noise_station_id) + + for trigger_name in noise_station.get_triggers(): + if trigger_name in self._allowed_triggers: + continue + + trigger = noise_station.get_trigger(trigger_name) + if trigger.has_triggered(): + self.logger.debug(f'Noise station has triggered ({trigger_name}), reject noise event.') + return None + + for channel_id in station.get_channel_ids(): + noise_channel_id = self.__get_noise_channel(channel_id) + if noise_channel_id not in noise_station.get_channel_ids(): + if channel_id == noise_channel_id: + self.logger.debug('Event {}: Requested channel {} not found'.format(event_i, noise_channel_id)) + return None + + noise_channel = noise_station.get_channel(noise_channel_id) + channel = station.get_channel(channel_id) + + if channel.get_sampling_rate() != noise_channel.get_sampling_rate() and not self.__allow_noise_resampling: + self.logger.debug('Event {}, Channel {}: Different sampling rates, reject noise event.' + .format(event_i, noise_channel_id)) + return None + + if (noise_channel.get_number_of_samples() / noise_channel.get_sampling_rate() < + channel.get_number_of_samples() / channel.get_sampling_rate()): + # Just add debug message... + self.logger.debug('Event {}, Channel {}: Different sampling rate / trace lenght ratios' + .format(event_i, noise_channel_id)) + + return noise_station + + + def end(self): + """ + End method. Draws a histogram of the noise statistics and fits a + Gaussian distribution to it. + """ + if self.__noise_data is not None: + import matplotlib.pyplot as plt + plt.close('all') + + noise_entries = np.array(self.__noise_data) + noise_bins = np.arange(-150, 150, 5.) * units.mV + noise_entries = noise_entries.flatten() + mean = noise_entries.mean() + sigma = noise_entries.std() + + fig1, ax1_1 = plt.subplots() + n, bins, pathes = ax1_1.hist(noise_entries / units.mV, bins=noise_bins / units.mV) + ax1_1.plot(noise_bins / units.mV, np.max(n) * np.exp(-0.5 * (noise_bins - mean) ** 2 / sigma ** 2)) + ax1_1.grid() + ax1_1.set_xlabel('sample value [mV]') + ax1_1.set_ylabel('entries') + plt.show() diff --git a/NuRadioReco/modules/neutrinoDirectionReconstructor/voltageToEfieldAnalyticConverterForNeutrinos.py b/NuRadioReco/modules/neutrinoDirectionReconstructor/voltageToEfieldAnalyticConverterForNeutrinos.py index e672b51d6..07f7daa19 100644 --- a/NuRadioReco/modules/neutrinoDirectionReconstructor/voltageToEfieldAnalyticConverterForNeutrinos.py +++ b/NuRadioReco/modules/neutrinoDirectionReconstructor/voltageToEfieldAnalyticConverterForNeutrinos.py @@ -187,14 +187,14 @@ def minimizer(params, minimizer=True): order = 5 b, a = signal.butter(order, passband_high[use_channels[iA]], 'bandpass', analog=True) w, h = signal.freqs(b, a, ff[mask]) - f = np.zeros_like(ff, dtype=np.complex) + f = np.zeros_like(ff, dtype=complex) f[mask] = h trace_spectrum *= f order = 10 b, a = signal.butter(order, passband_low[use_channels[iA]], 'bandpass', analog=True) w, h = signal.freqs(b, a, ff[mask]) - f = np.zeros_like(ff, dtype=np.complex) + f = np.zeros_like(ff, dtype=complex) f[mask] = h trace_spectrum *= f @@ -352,8 +352,8 @@ def plotSimulatedVersusReconstructedTraces(nu_zenith, nu_azimuth, shower_energy, travel_distance = np.zeros((n_antennas, maxNumRayTracingSolPerChan)) attenuation = np.zeros((n_antennas, maxNumRayTracingSolPerChan, len(ff))) focusing = np.zeros((n_antennas, maxNumRayTracingSolPerChan, 1)) - reflection_coefficients_theta = np.ones((n_antennas, maxNumRayTracingSolPerChan), dtype=np.complex) - reflection_coefficients_phi = np.ones((n_antennas, maxNumRayTracingSolPerChan), dtype=np.complex) + reflection_coefficients_theta = np.ones((n_antennas, maxNumRayTracingSolPerChan), dtype=complex) + reflection_coefficients_phi = np.ones((n_antennas, maxNumRayTracingSolPerChan), dtype=complex) travel_time_min = float('inf') for iA, position in enumerate(antenna_positions): r = ray.ray_tracing(icemodel, attenuation_model=attenuation_model, n_frequencies_integration=25, n_reflections=n_reflections) diff --git a/NuRadioReco/modules/phasedarray/triggerSimulator.py b/NuRadioReco/modules/phasedarray/triggerSimulator.py index 9fba9b25d..0128d0fe8 100644 --- a/NuRadioReco/modules/phasedarray/triggerSimulator.py +++ b/NuRadioReco/modules/phasedarray/triggerSimulator.py @@ -222,7 +222,7 @@ def power_sum(self, coh_sum, window, step, adc_output='voltage'): num_frames = int(np.floor((len(coh_sum) - window) / step)) if(adc_output == 'voltage'): - coh_sum_squared = (coh_sum * coh_sum).astype(np.float) + coh_sum_squared = (coh_sum * coh_sum).astype(float) elif(adc_output == 'counts'): coh_sum_squared = (coh_sum * coh_sum).astype(int) @@ -230,7 +230,7 @@ def power_sum(self, coh_sum, window, step, adc_output='voltage'): (coh_sum_squared.strides[0] * step, coh_sum_squared.strides[0])) power = np.sum(coh_sum_windowed, axis=1) - return power.astype(np.float) / window, num_frames + return power.astype(float) / window, num_frames def phase_signals(self, traces, beam_rolls): """ @@ -336,6 +336,8 @@ def phased_trigger(self, station, det, the earliest trigger time with respect to first interaction time. trigger_times: dictionary all time bins that fulfil the trigger condition per beam. The key is the beam number. Time with respect to first interaction time. + maximum_amps: list of floats (length equal to that of `phasing_angles`) + the maximum value of all the integration windows for each of the phased waveforms """ if(triggered_channels is None): @@ -375,30 +377,10 @@ def phased_trigger(self, station, det, raise ValueError("Could not convert upsampling_factor to integer. Exiting.") if(upsampling_factor >= 2): - ''' - # Zero inserting and filtering - upsampled_trace = upsampling_fir(trace, adc_sampling_frequency, - int_factor=upsampling_factor, ntaps=1) - - channelBandPassFilter = NuRadioReco.modules.channelBandPassFilter.channelBandPassFilter() - ff = np.fft.rfftfreq(len(upsampled_trace), 1.0 / adc_sampling_frequency / upsampling_factor) - filt = channelBandPassFilter.get_filter(ff, 0, 0, None, passband=[0, 240 * units.MHz], filter_type="cheby1", order=9, rp=.1) - - upsampled_trace = np.fft.irfft(np.fft.rfft(upsampled_trace) * filt) - ''' - # FFT upsampling new_len = len(trace) * upsampling_factor upsampled_trace = scipy.signal.resample(trace, new_len) - ''' - # Linear interpolation - x = np.arange(len(trace)) - f_trace = interp1d(x, trace, kind='linear', fill_value=(trace[0], trace[-1]), bounds_error=False) - x_new = np.arange(len(trace) * upsampling_factor) / upsampling_factor - upsampled_trace = f_trace(x_new) - ''' - # If upsampled is performed, the final sampling frequency changes trace = upsampled_trace[:] @@ -424,11 +406,15 @@ def phased_trigger(self, station, det, channel_trace_start_time = self.get_channel_trace_start_time(station, triggered_channels) trigger_delays = {} + maximum_amps = np.zeros_like(phased_traces) + for iTrace, phased_trace in enumerate(phased_traces): # Create a sliding window squared_mean, num_frames = self.power_sum(coh_sum=phased_trace, window=window, step=step, adc_output=adc_output) + maximum_amps[iTrace] = np.max(squared_mean) + if True in (squared_mean > threshold): trigger_delays[iTrace] = {} @@ -436,18 +422,19 @@ def phased_trigger(self, station, det, trigger_delays[iTrace][channel_id] = beam_rolls[iTrace][channel_id] * time_step triggered_bins = np.atleast_1d(np.squeeze(np.argwhere(squared_mean > threshold))) - # logger.debug(f"Station has triggered, at bins {triggered_bins}") - # logger.debug(trigger_delays) - # logger.debug(f"trigger_delays {trigger_delays[iTrace][triggered_channels[0]]}") + logger.debug(f"Station has triggered, at bins {triggered_bins}") + logger.debug(trigger_delays) + logger.debug(f"trigger_delays {trigger_delays[iTrace][triggered_channels[0]]}") is_triggered = True trigger_times[iTrace] = trigger_delays[iTrace][triggered_channels[0]] + triggered_bins * step * time_step + channel_trace_start_time - # logger.debug(f"trigger times = {trigger_times[iTrace]}") + logger.debug(f"trigger times = {trigger_times[iTrace]}") if is_triggered: - # logger.debug(trigger_times) + logger.debug("Trigger condition satisfied!") + logger.debug("all trigger times", trigger_times) trigger_time = min([x.min() for x in trigger_times.values()]) - # logger.debug(f"minimum trigger time is {trigger_time:.0f}ns") + logger.debug(f"minimum trigger time is {trigger_time:.0f}ns") - return is_triggered, trigger_delays, trigger_time, trigger_times + return is_triggered, trigger_delays, trigger_time, trigger_times, maximum_amps @register_run() def run(self, evt, station, det, @@ -543,25 +530,36 @@ def run(self, evt, station, det, if(set_not_triggered): is_triggered = False trigger_delays = {} + triggered_beams = [] else: - is_triggered, trigger_delays, trigger_time, trigger_times = self.phased_trigger(station=station, - det=det, - Vrms=Vrms, - threshold=threshold, - triggered_channels=triggered_channels, - phasing_angles=phasing_angles, - ref_index=ref_index, - trigger_adc=trigger_adc, - clock_offset=clock_offset, - adc_output=adc_output, - trigger_filter=trigger_filter, - upsampling_factor=upsampling_factor, - window=window, - step=step) + is_triggered, trigger_delays, trigger_time, trigger_times, maximum_amps = self.phased_trigger( + station=station, + det=det, + Vrms=Vrms, + threshold=threshold, + triggered_channels=triggered_channels, + phasing_angles=phasing_angles, + ref_index=ref_index, + trigger_adc=trigger_adc, + clock_offset=clock_offset, + adc_output=adc_output, + trigger_filter=trigger_filter, + upsampling_factor=upsampling_factor, + window=window, + step=step, + ) # Create a trigger object to be returned to the station - trigger = SimplePhasedTrigger(trigger_name, threshold, channels=triggered_channels, - primary_angles=phasing_angles, trigger_delays=trigger_delays) + trigger = SimplePhasedTrigger( + trigger_name, + threshold, + channels=triggered_channels, + primary_angles=phasing_angles, + trigger_delays=trigger_delays, + window_size=window, + step_size=step, + maximum_amps=maximum_amps, + ) trigger.set_triggered(is_triggered) diff --git a/NuRadioReco/modules/sphericalWaveFitter.py b/NuRadioReco/modules/sphericalWaveFitter.py index ce9b8b943..a08c2821d 100644 --- a/NuRadioReco/modules/sphericalWaveFitter.py +++ b/NuRadioReco/modules/sphericalWaveFitter.py @@ -13,25 +13,25 @@ class sphericalWaveFitter: " Fits position x,y, z of a source using spherical fit to channels " - + def __init__(self): pass - - + + def begin(self, channel_ids = [0, 3, 9, 10]): self.__channel_ids = channel_ids pass def run(self, evt, station, det, start_pulser_position, n_index = None, debug = True): - + print("channels used for this reconstruction:", self.__channel_ids) - + def get_distance(x, y): return np.sqrt((x[0] - y[0])**2+ (x[1] - y[1])**2 + (x[2] - y[2])**2) - - + + def get_time_delay_spherical_wave(position, ch_pair, n=n_index): T0 = get_distance(position, det.get_relative_position(station_id, ch_pair[0]))/(constants.c/n_index)*units.s T1 = get_distance(position , det.get_relative_position(station_id, ch_pair[1]))/(constants.c/n_index)*units.s @@ -45,11 +45,11 @@ def get_time_delay_spherical_wave(position, ch_pair, n=n_index): for j in range(i + 1, len(self.__channel_ids)): relative_positions = det.get_relative_position(station_id, self.__channel_ids[i]) - det.get_relative_position(station_id, self.__channel_ids[j]) self.__relative_positions.append(relative_positions) - + self.__channel_pairs.append([self.__channel_ids[i], self.__channel_ids[j]]) - - - self.__sampling_rate = station.get_channel(0).get_sampling_rate() + + + self.__sampling_rate = station.get_channel(self.__channel_ids[0]).get_sampling_rate() if debug: fig, ax = plt.subplots( len(self.__channel_pairs), 2) @@ -73,16 +73,16 @@ def likelihood(pulser_position, x = None, y = None, debug_corr = False): ax.set_ylim((0, max(self.__correlation[ich]))) ax.axvline(pos, label = 'reconstruction', lw = 1, color = 'orange') ax.axvline(self._pos_starting[ich], label = 'starting pos', lw = 1, color = 'green') - ax.set_title("channel pair {}".format( ch_pair), fontsize = 5) + ax.set_title("channel pair {}".format( ch_pair), fontsize = 5) ax.legend(fontsize = 5) - + if debug_corr: fig.tight_layout() fig.savefig("debug.pdf") return -1*corr - - + + trace = np.copy(station.get_channel(self.__channel_pairs[0][0]).get_trace()) self.__correlation = np.zeros((len(self.__channel_pairs), len(np.abs(scipy.signal.correlate(trace, trace))) )) @@ -90,7 +90,7 @@ def likelihood(pulser_position, x = None, y = None, debug_corr = False): for ich, ch_pair in enumerate(self.__channel_pairs): trace1 = np.copy(station.get_channel(self.__channel_pairs[ich][0]).get_trace()) trace2 = np.copy(station.get_channel(self.__channel_pairs[ich][1]).get_trace()) - + t_max1 = station.get_channel(self.__channel_pairs[ich][0]).get_times()[np.argmax(np.abs(trace1))] t_max2 = station.get_channel(self.__channel_pairs[ich][1]).get_times()[np.argmax(np.abs(trace2))] corr_range = 50 * units.ns @@ -101,7 +101,7 @@ def likelihood(pulser_position, x = None, y = None, debug_corr = False): else: trace2[np.abs(station.get_channel(self.__channel_pairs[ich][1]).get_times() - t_max2) > corr_range] = 0 self.__correlation[ich] = np.abs(scipy.signal.correlate(trace1, trace2)) - + #### set positions for starting position #### @@ -124,7 +124,7 @@ def likelihood(pulser_position, x = None, y = None, debug_corr = False): print("reconstructed position: {}".format([ll[0], ll[1], ll[2]])) if debug: - + method = 'Nelder-Mead' x = np.arange(x_start -3, x_start +3, dx) y = np.arange(y_start - 3, y_start +3, dy) @@ -137,7 +137,7 @@ def likelihood(pulser_position, x = None, y = None, debug_corr = False): c = opt.minimize(likelihood, x0 = (start_pulser_position[2]), args = (x_i, y_i, False), method = method) zz[ix, iy] = c.fun zz_values[ix, iy] = c.x[0] - + fig = plt.figure(figsize = (10, 5)) ax1 = fig.add_subplot(121) pax1 = ax1.pcolor(xx, yy, zz.T) @@ -170,4 +170,4 @@ def likelihood(pulser_position, x = None, y = None, debug_corr = False): def end(self): pass - + diff --git a/NuRadioReco/modules/trigger/highLowThreshold.py b/NuRadioReco/modules/trigger/highLowThreshold.py index c08e9d7c4..cff116833 100644 --- a/NuRadioReco/modules/trigger/highLowThreshold.py +++ b/NuRadioReco/modules/trigger/highLowThreshold.py @@ -32,7 +32,7 @@ def get_high_low_triggers(trace, high_threshold, low_threshold, the bins where the trigger condition is satisfied """ n_bins_coincidence = int(np.round(time_coincidence / dt)) + 1 - c = np.ones(n_bins_coincidence, dtype=np.bool) + c = np.ones(n_bins_coincidence, dtype=bool) logger.debug("length of trace {} bins, coincidence window {} bins".format(len(trace), len(c))) c2 = np.array([1, -1]) @@ -70,7 +70,7 @@ def get_majority_logic(tts, number_of_coincidences=2, time_coincidence=32 * unit if(n_bins_coincidence > n): # reduce coincidence window to maximum trace length n_bins_coincidence = n logger.debug("specified coincidence window longer than tracelenght, reducing coincidence window to trace length") - c = np.ones(n_bins_coincidence, dtype=np.bool) + c = np.ones(n_bins_coincidence, dtype=bool) for i in range(len(tts)): logger.debug("get_majority_logic() length of trace {} bins, coincidence window {} bins".format(len(tts[i]), len(c))) diff --git a/NuRadioReco/modules/voltageToAnalyticEfieldConverter.py b/NuRadioReco/modules/voltageToAnalyticEfieldConverter.py index d517b8d22..33bd3cff5 100644 --- a/NuRadioReco/modules/voltageToAnalyticEfieldConverter.py +++ b/NuRadioReco/modules/voltageToAnalyticEfieldConverter.py @@ -314,10 +314,10 @@ def run(self, evt, station, det, debug=False, debug_plotpath=None, efield_antenna_factor, V, V_timedomain = get_array_of_channels(station, use_channels, det, zenith, azimuth, self.antenna_provider, time_domain=True) - sampling_rate = station.get_channel(0).get_sampling_rate() + sampling_rate = station.get_channel(use_channels[0]).get_sampling_rate() n_samples_time = V_timedomain.shape[1] - noise_RMS = det.get_noise_RMS(station.get_id(), 0) + noise_RMS = det.get_noise_RMS(station.get_id(), use_channels[0]) def obj_xcorr(params): if(len(params) == 3): @@ -482,9 +482,10 @@ def obj_amplitude_second_order(params, slope, phase, pos, compare='hilbert', deb ax[iCh][1].axvline(imin, linestyle='--', alpha=.8) ax[iCh][1].axvline(imax, linestyle='--', alpha=.8) if(debug_obj and self.i_slope_fit_iterations % 50 == 0): - sim_channel = station.get_sim_station().get_channel(0)[0] + sim_station = station.get_sim_station() + sim_channel = next(sim_station.iter_channels()) ax[4][0].plot(sim_channel.get_frequencies() / units.MHz, np.abs(pulse.get_analytic_pulse_freq(ampTheta, slope, phase, len(sim_channel.get_times()), sim_channel.get_sampling_rate(), bandpass=bandpass, quadratic_term=second_order)), '--', color='orange') - ax[4][0].plot(sim_channel.get_frequencies() / units.MHz, np.abs(station.get_sim_station().get_channel(0)[0].get_frequency_spectrum()[1]), color='blue') + ax[4][0].plot(sim_channel.get_frequencies() / units.MHz, np.abs(sim_channel.get_frequency_spectrum()[1]), color='blue') ax[4][1].plot(sim_channel.get_frequencies() / units.MHz, np.abs(pulse.get_analytic_pulse_freq(ampPhi, slope, phase, len(sim_channel.get_times()), sim_channel.get_sampling_rate(), bandpass=bandpass, quadratic_term=second_order)), '--', color='orange') ax[4][1].plot(sim_channel.get_frequencies() / units.MHz, np.abs(sim_channel.get_frequency_spectrum()[2]), color='blue') ax[4][0].set_xlim([20, 500]) diff --git a/NuRadioReco/modules/voltageToEfieldConverter.py b/NuRadioReco/modules/voltageToEfieldConverter.py index 2b66efa12..5c2f5fb08 100644 --- a/NuRadioReco/modules/voltageToEfieldConverter.py +++ b/NuRadioReco/modules/voltageToEfieldConverter.py @@ -47,7 +47,7 @@ def get_array_of_channels(station, use_channels, det, zenith, azimuth, tmax = time_shifts.max() logger.debug("adding relative station time = {:.0f}ns".format((t_cables.min() + t_geos.max()) / units.ns)) logger.debug("delta t is {:.2f}".format(delta_t / units.ns)) - trace_length = station.get_channel(0).get_times()[-1] - station.get_channel(0).get_times()[0] + trace_length = station.get_channel(use_channels[0]).get_times()[-1] - station.get_channel(use_channels[0]).get_times()[0] debug_cut = 0 if(debug_cut): fig, ax = plt.subplots(len(use_channels), 1) @@ -77,7 +77,7 @@ def get_array_of_channels(station, use_channels, det, zenith, azimuth, for iCh, trace in enumerate(traces): V_timedomain[iCh] = trace.get_trace() frequencies = traces[0].get_frequencies() # assumes that all channels have the same sampling rate - V = np.zeros((len(use_channels), len(frequencies)), dtype=np.complex) + V = np.zeros((len(use_channels), len(frequencies)), dtype=complex) for iCh, trace in enumerate(traces): V[iCh] = trace.get_frequency_spectrum() @@ -110,11 +110,11 @@ def stacked_lstsq(L, b, rcond=1e-10): class voltageToEfieldConverter: """ - This module reconstructs the electric field by solving the system of equation that related the incident electric field via the antenna response functions to the measured voltages + This module reconstructs the electric field by solving the system of equation that related the incident electric field via the antenna response functions to the measured voltages (see Eq. 4 of the NuRadioReco paper https://link.springer.com/article/10.1140/epjc/s10052-019-6971-5). The module assumed that the electric field is identical at the antennas/channels that are used for the reconstruction. Furthermore, at least two antennas with - orthogonal polarization response are needed to reconstruct the 3dim electric field. - Alternatively, the polarization of the resulting efield could be forced to a single polarization component. In that case, a single antenna is sufficient. + orthogonal polarization response are needed to reconstruct the 3dim electric field. + Alternatively, the polarization of the resulting efield could be forced to a single polarization component. In that case, a single antenna is sufficient. """ def __init__(self): @@ -170,7 +170,7 @@ def run(self, evt, station, det, use_channels=None, use_MC_direction=False, forc E1[mask] = (V[0] * efield_antenna_factor[-1][1] - V[-1] * efield_antenna_factor[0][1])[mask] / denom[mask] E2[mask] = (V[-1] - efield_antenna_factor[-1][0] * E1)[mask] / efield_antenna_factor[-1][1][mask] # solve it in a vectorized way - efield3_f = np.zeros((2, n_frequencies), dtype=np.complex) + efield3_f = np.zeros((2, n_frequencies), dtype=complex) if force_Polarization == 'eTheta': efield3_f[:1, mask] = np.moveaxis(stacked_lstsq(np.moveaxis(efield_antenna_factor[:, 0, mask], 1, 0)[:, :, np.newaxis], np.moveaxis(V[:, mask], 1, 0)), 0, 1) elif force_Polarization == 'ePhi': @@ -178,12 +178,12 @@ def run(self, evt, station, det, use_channels=None, use_MC_direction=False, forc else: efield3_f[:, mask] = np.moveaxis(stacked_lstsq(np.moveaxis(efield_antenna_factor[:, :, mask], 2, 0), np.moveaxis(V[:, mask], 1, 0)), 0, 1) # add eR direction - efield3_f = np.array([np.zeros_like(efield3_f[0], dtype=np.complex), + efield3_f = np.array([np.zeros_like(efield3_f[0], dtype=complex), efield3_f[0], efield3_f[1]]) electric_field = NuRadioReco.framework.electric_field.ElectricField(use_channels, [0, 0, 0]) - electric_field.set_frequency_spectrum(efield3_f, station.get_channel(0).get_sampling_rate()) + electric_field.set_frequency_spectrum(efield3_f, station.get_channel(use_channels[0]).get_sampling_rate()) electric_field.set_parameter(efp.zenith, zenith) electric_field.set_parameter(efp.azimuth, azimuth) # figure out the timing of the E-field diff --git a/NuRadioReco/modules/voltageToEfieldConverterPerChannel.py b/NuRadioReco/modules/voltageToEfieldConverterPerChannel.py index d83319e16..f55f5f875 100644 --- a/NuRadioReco/modules/voltageToEfieldConverterPerChannel.py +++ b/NuRadioReco/modules/voltageToEfieldConverterPerChannel.py @@ -53,19 +53,19 @@ def run(self, evt, station, det, pol=0): zenith = station[stnp.zenith] azimuth = station[stnp.azimuth] - frequencies = station.get_channel(0).get_frequencies() # assuming that all channels have the same sampling rate and length use_channels = det.get_channel_ids(station.get_id()) + frequencies = station.get_channel(use_channels[0]).get_frequencies() # assuming that all channels have the same sampling rate and length efield_antenna_factor = trace_utilities.get_efield_antenna_factor(station, frequencies, use_channels, det, zenith, azimuth, self.antenna_provider) - sampling_rate = station.get_channel(0).get_sampling_rate() + sampling_rate = station.get_channel(use_channels[0]).get_sampling_rate() for iCh, channel in enumerate(station.iter_channels()): efield = ef.ElectricField([iCh]) trace = channel.get_frequency_spectrum() mask1 = np.abs(efield_antenna_factor[iCh][0]) != 0 mask2 = np.abs(efield_antenna_factor[iCh][1]) != 0 - efield_spectrum = np.zeros((3, len(trace)), dtype=np.complex) + efield_spectrum = np.zeros((3, len(trace)), dtype=complex) efield_spectrum[1][mask1] = (1.0 - pol) ** 2 * trace[mask1] / efield_antenna_factor[iCh][0][mask1] efield_spectrum[2][mask2] = pol ** 2 * trace[mask2] / efield_antenna_factor[iCh][1][mask2] efield.set_frequency_spectrum(efield_spectrum, sampling_rate) diff --git a/NuRadioReco/test/tiny_reconstruction/reference.json b/NuRadioReco/test/tiny_reconstruction/reference.json index 40c6b9def..968d3584b 100644 --- a/NuRadioReco/test/tiny_reconstruction/reference.json +++ b/NuRadioReco/test/tiny_reconstruction/reference.json @@ -1,344 +1 @@ -{ - "0" : { - "station_parameters" : { - "nu_zenith" : null, - "nu_azimuth" : null, - "nu_energy" : null, - "nu_flavor" : null, - "ccnc" : null, - "nu_vertex" : null, - "inelasticity" : null, - "triggered" : null, - "cr_energy" : null, - "cr_zenith" : null, - "cr_azimuth" : null, - "channels_max_amplitude" : 0.08853022471996891, - "zenith" : 0.79, - "azimuth" : 3.96, - "zenith_cr_templatefit" : null, - "zenith_nu_templatefit" : null, - "cr_xcorrelations" : null, - "nu_xcorrelations" : null, - "station_time" : null, - "cr_energy_em" : null, - "nu_inttype" : null, - "chi2_efield_time_direction_fit" : null, - "ndf_efield_time_direction_fit" : null, - "cr_xmax" : null, - "vertex_2D_fit" : null, - "distance_correlations" : null - }, - "sim_station_parameters" : { - "nu_zenith" : null, - "nu_azimuth" : null, - "nu_energy" : null, - "nu_flavor" : null, - "ccnc" : null, - "nu_vertex" : null, - "inelasticity" : null, - "triggered" : null, - "cr_energy" : 1.58489319246E18, - "cr_zenith" : null, - "cr_azimuth" : null, - "channels_max_amplitude" : null, - "zenith" : 0.7853981633974483, - "azimuth" : 3.957853280977215, - "zenith_cr_templatefit" : null, - "zenith_nu_templatefit" : null, - "cr_xcorrelations" : null, - "nu_xcorrelations" : null, - "station_time" : null, - "cr_energy_em" : 1.39187479744259994E18, - "nu_inttype" : null, - "chi2_efield_time_direction_fit" : null, - "ndf_efield_time_direction_fit" : null, - "cr_xmax" : 646.2024663, - "vertex_2D_fit" : null, - "distance_correlations" : null - }, - "channel_parameters" : { - "zenith" : [ null, null, null, null ], - "azimuth" : [ null, null, null, null ], - "maximum_amplitude" : [ 0.0873141653492205, 0.08853022471996891, 0.07885012049626394, 0.07486104950522432 ], - "SNR" : [ { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.004149416647868447, - "peak_amplitude" : 0.004225325303006149, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.0061325770929495016, - "peak_amplitude" : 0.006227798651940198, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.0038091911965324092, - "peak_amplitude" : 0.0038590946944723325, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.003328271897442733, - "peak_amplitude" : 0.0034583171387571744, - "Seckel_2_noise" : 5 - } ], - "maximum_amplitude_envelope" : [ 0.08772020520899446, 0.08892789858406401, 0.07966830072878535, 0.08297084302728261 ], - "P2P_amplitude" : [ 0.1724152251327864, 0.16341917302469117, 0.146681542086196, 0.1458075155635604 ], - "cr_xcorrelations" : [ null, null, null, null ], - "nu_xcorrelations" : [ null, null, null, null ], - "signal_time" : [ -168.2559785714568, -171.65597857145679, -183.2559785714568, -173.85597857145672 ], - "noise_rms" : [ 0.008836023847618458, 0.008955465322914774, 0.008881819339363746, 0.009304834436981291 ], - "signal_regions" : [ null, null, null, null ], - "noise_regions" : [ null, null, null, null ], - "signal_time_offset" : [ null, null, null, null ], - "signal_receiving_zenith" : [ null, null, null, null ], - "signal_ray_type" : [ null, null, null, null ], - "signal_receiving_azimuth" : [ null, null, null, null ] - }, - "electric_field_parameters" : { - "ray_path_type" : [ null, null ], - "polarization_angle" : [ 1.437014716504628, 1.505812376998251 ], - "polarization_angle_expectation" : [ 1.3259385237455839, -1.8156541298442093 ], - "signal_energy_fluence" : [ [ 0.0, 0.2505871320257097, 13.83446329404234 ], [ 0.0, 0.07493732342984019, 17.695470024342303 ] ], - "cr_spectrum_slope" : [ null, -5.071257963761185 ], - "zenith" : [ 0.79, 0.79 ], - "azimuth" : [ 3.96, 3.96 ], - "signal_time" : [ -254.95450631365475, null ], - "nu_vertex_distance" : [ null, null ], - "nu_viewing_angle" : [ null, null ], - "max_amp_antenna" : [ null, null ], - "max_amp_antenna_envelope" : [ null, null ], - "reflection_coefficient_theta" : [ null, null ], - "reflection_coefficient_phi" : [ null, null ], - "cr_spectrum_quadratic_term" : [ null, 3.7278692487503235 ], - "energy_fluence_ratios" : [ null, null ] - } - }, - "1" : { - "station_parameters" : { - "nu_zenith" : null, - "nu_azimuth" : null, - "nu_energy" : null, - "nu_flavor" : null, - "ccnc" : null, - "nu_vertex" : null, - "inelasticity" : null, - "triggered" : null, - "cr_energy" : null, - "cr_zenith" : null, - "cr_azimuth" : null, - "channels_max_amplitude" : 0.33359702926988355, - "zenith" : 0.78, - "azimuth" : 3.95, - "zenith_cr_templatefit" : null, - "zenith_nu_templatefit" : null, - "cr_xcorrelations" : null, - "nu_xcorrelations" : null, - "station_time" : null, - "cr_energy_em" : null, - "nu_inttype" : null, - "chi2_efield_time_direction_fit" : null, - "ndf_efield_time_direction_fit" : null, - "cr_xmax" : null, - "vertex_2D_fit" : null, - "distance_correlations" : null - }, - "sim_station_parameters" : { - "nu_zenith" : null, - "nu_azimuth" : null, - "nu_energy" : null, - "nu_flavor" : null, - "ccnc" : null, - "nu_vertex" : null, - "inelasticity" : null, - "triggered" : null, - "cr_energy" : 1.58489319246E18, - "cr_zenith" : null, - "cr_azimuth" : null, - "channels_max_amplitude" : null, - "zenith" : 0.7853981633974483, - "azimuth" : 3.957853280977215, - "zenith_cr_templatefit" : null, - "zenith_nu_templatefit" : null, - "cr_xcorrelations" : null, - "nu_xcorrelations" : null, - "station_time" : null, - "cr_energy_em" : 1.39187479744259994E18, - "nu_inttype" : null, - "chi2_efield_time_direction_fit" : null, - "ndf_efield_time_direction_fit" : null, - "cr_xmax" : 646.2024663, - "vertex_2D_fit" : null, - "distance_correlations" : null - }, - "channel_parameters" : { - "zenith" : [ null, null, null, null ], - "azimuth" : [ null, null, null, null ], - "maximum_amplitude" : [ 0.2835856575324592, 0.33359702926988355, 0.2791941553632823, 0.3256776186469762 ], - "SNR" : [ { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.00291206901456176, - "peak_amplitude" : 0.002929725052658198, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.00340572893988943, - "peak_amplitude" : 0.0034558407562806964, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.006845057653623751, - "peak_amplitude" : 0.00701923788399404, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.007164164413376706, - "peak_amplitude" : 0.007202262656811785, - "Seckel_2_noise" : 5 - } ], - "maximum_amplitude_envelope" : [ 0.3003272394395557, 0.3399598814912192, 0.29236891303881996, 0.35467083189765575 ], - "P2P_amplitude" : [ 0.5632116726571561, 0.6637288453900938, 0.554719957942302, 0.6479974651404222 ], - "cr_xcorrelations" : [ null, null, null, null ], - "nu_xcorrelations" : [ null, null, null, null ], - "signal_time" : [ 587.7440214285432, 589.9440214285432, 574.9440214285432, 571.3440214285433 ], - "noise_rms" : [ 0.009006474050937649, 0.008732105958313228, 0.007788519158762668, 0.009397032099363559 ], - "signal_regions" : [ null, null, null, null ], - "noise_regions" : [ null, null, null, null ], - "signal_time_offset" : [ null, null, null, null ], - "signal_receiving_zenith" : [ null, null, null, null ], - "signal_ray_type" : [ null, null, null, null ], - "signal_receiving_azimuth" : [ null, null, null, null ] - }, - "electric_field_parameters" : { - "ray_path_type" : [ null, null ], - "polarization_angle" : [ 1.423905497514318, 1.4409019604509443 ], - "polarization_angle_expectation" : [ 1.3232105351636871, -1.818382118426106 ], - "signal_energy_fluence" : [ [ 0.0, 6.607535268712784, 301.8361992200022 ], [ 0.0, 5.437314688159896, 318.63935252848177 ] ], - "cr_spectrum_slope" : [ null, -6.6760948093020485 ], - "zenith" : [ 0.78, 0.78 ], - "azimuth" : [ 3.95, 3.95 ], - "signal_time" : [ 499.91208204952255, null ], - "nu_vertex_distance" : [ null, null ], - "nu_viewing_angle" : [ null, null ], - "max_amp_antenna" : [ null, null ], - "max_amp_antenna_envelope" : [ null, null ], - "reflection_coefficient_theta" : [ null, null ], - "reflection_coefficient_phi" : [ null, null ], - "cr_spectrum_quadratic_term" : [ null, -2.215049003838984 ], - "energy_fluence_ratios" : [ null, null ] - } - }, - "2" : { - "station_parameters" : { - "nu_zenith" : null, - "nu_azimuth" : null, - "nu_energy" : null, - "nu_flavor" : null, - "ccnc" : null, - "nu_vertex" : null, - "inelasticity" : null, - "triggered" : null, - "cr_energy" : null, - "cr_zenith" : null, - "cr_azimuth" : null, - "channels_max_amplitude" : 0.28471716111983614, - "zenith" : 0.78, - "azimuth" : 3.95, - "zenith_cr_templatefit" : null, - "zenith_nu_templatefit" : null, - "cr_xcorrelations" : null, - "nu_xcorrelations" : null, - "station_time" : null, - "cr_energy_em" : null, - "nu_inttype" : null, - "chi2_efield_time_direction_fit" : null, - "ndf_efield_time_direction_fit" : null, - "cr_xmax" : null, - "vertex_2D_fit" : null, - "distance_correlations" : null - }, - "sim_station_parameters" : { - "nu_zenith" : null, - "nu_azimuth" : null, - "nu_energy" : null, - "nu_flavor" : null, - "ccnc" : null, - "nu_vertex" : null, - "inelasticity" : null, - "triggered" : null, - "cr_energy" : 1.58489319246E18, - "cr_zenith" : null, - "cr_azimuth" : null, - "channels_max_amplitude" : null, - "zenith" : 0.7853981633974483, - "azimuth" : 3.957853280977215, - "zenith_cr_templatefit" : null, - "zenith_nu_templatefit" : null, - "cr_xcorrelations" : null, - "nu_xcorrelations" : null, - "station_time" : null, - "cr_energy_em" : 1.39187479744259994E18, - "nu_inttype" : null, - "chi2_efield_time_direction_fit" : null, - "ndf_efield_time_direction_fit" : null, - "cr_xmax" : 646.2024663, - "vertex_2D_fit" : null, - "distance_correlations" : null - }, - "channel_parameters" : { - "zenith" : [ null, null, null, null ], - "azimuth" : [ null, null, null, null ], - "maximum_amplitude" : [ 0.2617580175047184, 0.27712092224880147, 0.2555336277305896, 0.28471716111983614 ], - "SNR" : [ { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.0043472269995871535, - "peak_amplitude" : 0.0045710624556043215, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.003710424290754701, - "peak_amplitude" : 0.0037252751132054716, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.00777307702513199, - "peak_amplitude" : 0.007907951982053118, - "Seckel_2_noise" : 5 - }, { - "integrated_power" : 0.0, - "peak_2_peak_amplitude" : 0.008133561831805624, - "peak_amplitude" : 0.008336373693178935, - "Seckel_2_noise" : 5 - } ], - "maximum_amplitude_envelope" : [ 0.2653837960611956, 0.2969067177063586, 0.270376775534144, 0.298365691136913 ], - "P2P_amplitude" : [ 0.5188798774102936, 0.550291044285413, 0.5046288232897831, 0.5481116482309814 ], - "cr_xcorrelations" : [ null, null, null, null ], - "nu_xcorrelations" : [ null, null, null, null ], - "signal_time" : [ -134.05597857145676, -132.85597857145672, -146.85597857145672, -144.85597857145672 ], - "noise_rms" : [ 0.009190485861932177, 0.009505810656701827, 0.00886260343524874, 0.0091532389582909 ], - "signal_regions" : [ null, null, null, null ], - "noise_regions" : [ null, null, null, null ], - "signal_time_offset" : [ null, null, null, null ], - "signal_receiving_zenith" : [ null, null, null, null ], - "signal_ray_type" : [ null, null, null, null ], - "signal_receiving_azimuth" : [ null, null, null, null ] - }, - "electric_field_parameters" : { - "ray_path_type" : [ null, null ], - "polarization_angle" : [ 1.4819734258513997, 1.5006361258682528 ], - "polarization_angle_expectation" : [ 1.3232105351636871, -1.818382118426106 ], - "signal_energy_fluence" : [ [ 0.0, 2.5899529944532245, 326.5528775660024 ], [ 0.0, 2.0442071775847332, 413.9200130611196 ] ], - "cr_spectrum_slope" : [ null, -1.7873921532547352 ], - "zenith" : [ 0.78, 0.78 ], - "azimuth" : [ 3.95, 3.95 ], - "signal_time" : [ -193.88791795047757, null ], - "nu_vertex_distance" : [ null, null ], - "nu_viewing_angle" : [ null, null ], - "max_amp_antenna" : [ null, null ], - "max_amp_antenna_envelope" : [ null, null ], - "reflection_coefficient_theta" : [ null, null ], - "reflection_coefficient_phi" : [ null, null ], - "cr_spectrum_quadratic_term" : [ null, 0.4769503271885924 ], - "energy_fluence_ratios" : [ null, null ] - } - } -} \ No newline at end of file +{"0": {"station_parameters": {"nu_zenith": null, "nu_azimuth": null, "nu_energy": null, "nu_flavor": null, "ccnc": null, "nu_vertex": null, "inelasticity": null, "triggered": null, "cr_energy": null, "cr_zenith": null, "cr_azimuth": null, "channels_max_amplitude": 0.09219726935932945, "zenith": 0.76585400390625, "azimuth": 3.94125048828125, "zenith_cr_templatefit": null, "zenith_nu_templatefit": null, "cr_xcorrelations": null, "nu_xcorrelations": null, "station_time": null, "cr_energy_em": null, "nu_inttype": null, "chi2_efield_time_direction_fit": null, "ndf_efield_time_direction_fit": null, "cr_xmax": null, "vertex_2D_fit": null, "distance_correlations": null, "shower_energy": null, "viewing_angles": null}, "sim_station_parameters": {"nu_zenith": null, "nu_azimuth": null, "nu_energy": null, "nu_flavor": null, "ccnc": null, "nu_vertex": null, "inelasticity": null, "triggered": null, "cr_energy": 1.58489319246e+18, "cr_zenith": null, "cr_azimuth": null, "channels_max_amplitude": null, "zenith": 0.7853981633974483, "azimuth": 3.957853280977215, "zenith_cr_templatefit": null, "zenith_nu_templatefit": null, "cr_xcorrelations": null, "nu_xcorrelations": null, "station_time": null, "cr_energy_em": 1.3918747974426e+18, "nu_inttype": null, "chi2_efield_time_direction_fit": null, "ndf_efield_time_direction_fit": null, "cr_xmax": 646.2024663, "vertex_2D_fit": null, "distance_correlations": null, "shower_energy": null, "viewing_angles": null}, "channel_parameters": {"zenith": [null, null, null, null], "azimuth": [null, null, null, null], "maximum_amplitude": [0.0741114148547873, 0.07822416425569524, 0.08019365868532256, 0.09219726935932945], "SNR": [{"integrated_power": 0.0, "peak_2_peak_amplitude": 0.004780092071917627, "peak_amplitude": 0.00485887816779479, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.0029781482057526206, "peak_amplitude": 0.0030899574970230446, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.002963497087732319, "peak_amplitude": 0.0029912274254951183, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.003315202825440897, "peak_amplitude": 0.0035027540130119802, "Seckel_2_noise": 5}], "maximum_amplitude_envelope": [0.0789854151249416, 0.0850534211758587, 0.08035990802734218, 0.09425391321371974], "P2P_amplitude": [0.13819712244096005, 0.15451260952068696, 0.15292399225172826, 0.17650992826729145], "cr_xcorrelations": [null, null, null, null], "nu_xcorrelations": [null, null, null, null], "signal_time": [-165.45597857145674, -169.65597857145679, -182.85597857145672, -181.05597857145676], "noise_rms": [0.009497729224153949, 0.008938569434903116, 0.008083395580028708, 0.009418626437748765], "signal_regions": [null, null, null, null], "noise_regions": [null, null, null, null], "signal_time_offset": [null, null, null, null], "signal_receiving_zenith": [null, null, null, null], "signal_ray_type": [null, null, null, null], "signal_receiving_azimuth": [null, null, null, null]}, "electric_field_parameters": {"ray_path_type": [null, null], "polarization_angle": [1.4654396095601694, 1.4895721505795514], "polarization_angle_expectation": [1.3193420263512765, -1.8222506272385168], "signal_energy_fluence": [[0.0, 0.1499228726724943, 13.406681586865194], [0.0, 0.1138089722425281, 17.17484362157034]], "cr_spectrum_slope": [null, -5.857288373436567], "zenith": [0.76585400390625, 0.76585400390625], "azimuth": [3.94125048828125, 3.94125048828125], "signal_time": [-256.0418188450112, null], "nu_vertex_distance": [null, null], "nu_viewing_angle": [null, null], "max_amp_antenna": [null, null], "max_amp_antenna_envelope": [null, null], "reflection_coefficient_theta": [null, null], "reflection_coefficient_phi": [null, null], "cr_spectrum_quadratic_term": [null, 5.585796124606195], "energy_fluence_ratios": [null, null]}}, "1": {"station_parameters": {"nu_zenith": null, "nu_azimuth": null, "nu_energy": null, "nu_flavor": null, "ccnc": null, "nu_vertex": null, "inelasticity": null, "triggered": null, "cr_energy": null, "cr_zenith": null, "cr_azimuth": null, "channels_max_amplitude": 0.34265971445728505, "zenith": 0.79, "azimuth": 3.96, "zenith_cr_templatefit": null, "zenith_nu_templatefit": null, "cr_xcorrelations": null, "nu_xcorrelations": null, "station_time": null, "cr_energy_em": null, "nu_inttype": null, "chi2_efield_time_direction_fit": null, "ndf_efield_time_direction_fit": null, "cr_xmax": null, "vertex_2D_fit": null, "distance_correlations": null, "shower_energy": null, "viewing_angles": null}, "sim_station_parameters": {"nu_zenith": null, "nu_azimuth": null, "nu_energy": null, "nu_flavor": null, "ccnc": null, "nu_vertex": null, "inelasticity": null, "triggered": null, "cr_energy": 1.58489319246e+18, "cr_zenith": null, "cr_azimuth": null, "channels_max_amplitude": null, "zenith": 0.7853981633974483, "azimuth": 3.957853280977215, "zenith_cr_templatefit": null, "zenith_nu_templatefit": null, "cr_xcorrelations": null, "nu_xcorrelations": null, "station_time": null, "cr_energy_em": 1.3918747974426e+18, "nu_inttype": null, "chi2_efield_time_direction_fit": null, "ndf_efield_time_direction_fit": null, "cr_xmax": 646.2024663, "vertex_2D_fit": null, "distance_correlations": null, "shower_energy": null, "viewing_angles": null}, "channel_parameters": {"zenith": [null, null, null, null], "azimuth": [null, null, null, null], "maximum_amplitude": [0.29720895105426465, 0.34265971445728505, 0.28631950709258547, 0.3355886336552229], "SNR": [{"integrated_power": 0.0, "peak_2_peak_amplitude": 0.006919176246584566, "peak_amplitude": 0.007147260712361315, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.0047795421977155994, "peak_amplitude": 0.00483475221965112, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.007958346991753162, "peak_amplitude": 0.00808818541204719, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.006171263679930979, "peak_amplitude": 0.00634102083568216, "Seckel_2_noise": 5}], "maximum_amplitude_envelope": [0.30136927893948445, 0.3430368419418388, 0.2883085667945778, 0.3430413736731565], "P2P_amplitude": [0.5781561921741288, 0.6591005848543372, 0.5651166372487595, 0.6691483403791307], "cr_xcorrelations": [null, null, null, null], "nu_xcorrelations": [null, null, null, null], "signal_time": [583.9440214285432, 585.5440214285434, 570.5440214285434, 575.1440214285433], "noise_rms": [0.009359269057858015, 0.009003414652873262, 0.008341757854147532, 0.009479378300635246], "signal_regions": [null, null, null, null], "noise_regions": [null, null, null, null], "signal_time_offset": [null, null, null, null], "signal_receiving_zenith": [null, null, null, null], "signal_ray_type": [null, null, null, null], "signal_receiving_azimuth": [null, null, null, null]}, "electric_field_parameters": {"ray_path_type": [null, null], "polarization_angle": [1.4164838522395478, 1.4309598632203113], "polarization_angle_expectation": [1.3259385237455839, -1.8156541298442093], "signal_energy_fluence": [[0.0, 7.388558320124647, 305.36881995591784], [0.0, 6.420446226373949, 324.0685176647595]], "cr_spectrum_slope": [null, -6.704957977720079], "zenith": [0.79, 0.79], "azimuth": [3.96, 3.96], "signal_time": [500.04549368634525, null], "nu_vertex_distance": [null, null], "nu_viewing_angle": [null, null], "max_amp_antenna": [null, null], "max_amp_antenna_envelope": [null, null], "reflection_coefficient_theta": [null, null], "reflection_coefficient_phi": [null, null], "cr_spectrum_quadratic_term": [null, -2.2958549567008015], "energy_fluence_ratios": [null, null]}}, "2": {"station_parameters": {"nu_zenith": null, "nu_azimuth": null, "nu_energy": null, "nu_flavor": null, "ccnc": null, "nu_vertex": null, "inelasticity": null, "triggered": null, "cr_energy": null, "cr_zenith": null, "cr_azimuth": null, "channels_max_amplitude": 0.29293556379400426, "zenith": 0.78, "azimuth": 3.95, "zenith_cr_templatefit": null, "zenith_nu_templatefit": null, "cr_xcorrelations": null, "nu_xcorrelations": null, "station_time": null, "cr_energy_em": null, "nu_inttype": null, "chi2_efield_time_direction_fit": null, "ndf_efield_time_direction_fit": null, "cr_xmax": null, "vertex_2D_fit": null, "distance_correlations": null, "shower_energy": null, "viewing_angles": null}, "sim_station_parameters": {"nu_zenith": null, "nu_azimuth": null, "nu_energy": null, "nu_flavor": null, "ccnc": null, "nu_vertex": null, "inelasticity": null, "triggered": null, "cr_energy": 1.58489319246e+18, "cr_zenith": null, "cr_azimuth": null, "channels_max_amplitude": null, "zenith": 0.7853981633974483, "azimuth": 3.957853280977215, "zenith_cr_templatefit": null, "zenith_nu_templatefit": null, "cr_xcorrelations": null, "nu_xcorrelations": null, "station_time": null, "cr_energy_em": 1.3918747974426e+18, "nu_inttype": null, "chi2_efield_time_direction_fit": null, "ndf_efield_time_direction_fit": null, "cr_xmax": 646.2024663, "vertex_2D_fit": null, "distance_correlations": null, "shower_energy": null, "viewing_angles": null}, "channel_parameters": {"zenith": [null, null, null, null], "azimuth": [null, null, null, null], "maximum_amplitude": [0.2565625777544711, 0.2778654985637024, 0.2672414043783895, 0.29293556379400426], "SNR": [{"integrated_power": 0.0, "peak_2_peak_amplitude": 0.007948759818769018, "peak_amplitude": 0.008209754518559166, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.002601223824751355, "peak_amplitude": 0.0026817225129317146, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.008923287811149969, "peak_amplitude": 0.009007262903406938, "Seckel_2_noise": 5}, {"integrated_power": 0.0, "peak_2_peak_amplitude": 0.0068025654072232655, "peak_amplitude": 0.007054720375806741, "Seckel_2_noise": 5}], "maximum_amplitude_envelope": [0.2618281394747253, 0.28224029243565324, 0.26779467480180347, 0.2932426286212969], "P2P_amplitude": [0.5044222411860875, 0.547608219972491, 0.5318703949237792, 0.5851999240317709], "cr_xcorrelations": [null, null, null, null], "nu_xcorrelations": [null, null, null, null], "signal_time": [-102.65597857145667, -133.45597857145674, -148.2559785714567, -147.45597857145674], "noise_rms": [0.008359098917345579, 0.00848077598024109, 0.008989153845587915, 0.008201090181413125], "signal_regions": [null, null, null, null], "noise_regions": [null, null, null, null], "signal_time_offset": [null, null, null, null], "signal_receiving_zenith": [null, null, null, null], "signal_ray_type": [null, null, null, null], "signal_receiving_azimuth": [null, null, null, null]}, "electric_field_parameters": {"ray_path_type": [null, null], "polarization_angle": [1.4786245054415663, 1.5037562254029073], "polarization_angle_expectation": [1.3232105351636871, -1.818382118426106], "signal_energy_fluence": [[0.0, 2.8633662095356187, 335.1320229092165], [0.0, 1.9443851451542769, 431.3306782776532]], "cr_spectrum_slope": [null, -1.715922037957672], "zenith": [0.78, 0.78], "azimuth": [3.95, 3.95], "signal_time": [-193.88791795047757, null], "nu_vertex_distance": [null, null], "nu_viewing_angle": [null, null], "max_amp_antenna": [null, null], "max_amp_antenna_envelope": [null, null], "reflection_coefficient_theta": [null, null], "reflection_coefficient_phi": [null, null], "cr_spectrum_quadratic_term": [null, 0.5209156977104409], "energy_fluence_ratios": [null, null]}}} \ No newline at end of file diff --git a/NuRadioReco/test/trigger_tests/compare_to_reference.py b/NuRadioReco/test/trigger_tests/compare_to_reference.py index c11d604bd..7cfcdbd5e 100644 --- a/NuRadioReco/test/trigger_tests/compare_to_reference.py +++ b/NuRadioReco/test/trigger_tests/compare_to_reference.py @@ -41,7 +41,7 @@ for prop in properties: if(prop == "trigger_time"): try: - np.testing.assert_allclose(np.array(trigger_results[trigger_name][prop], dtype=np.float64), np.array(reference[trigger_name][prop], dtype=np.float64)) + np.testing.assert_allclose(np.array(trigger_results[trigger_name][prop], dtype=float), np.array(reference[trigger_name][prop], dtype=float)) except AssertionError as e: print('Property {} of trigger {} differs from reference'.format(prop, trigger_name)) print(e) diff --git a/NuRadioReco/utilities/bandpass_filter.py b/NuRadioReco/utilities/bandpass_filter.py index d02035e8a..b91700b55 100644 --- a/NuRadioReco/utilities/bandpass_filter.py +++ b/NuRadioReco/utilities/bandpass_filter.py @@ -48,21 +48,21 @@ def get_filter_response(frequencies, passband, filter_type, order, rp=None): f[np.where(frequencies > passband[1])] = 0. return f elif (filter_type == 'butter'): - f = np.zeros_like(frequencies, dtype=np.complex) + f = np.zeros_like(frequencies, dtype=complex) mask = frequencies > 0 b, a = scipy.signal.butter(order, *scipy_args, analog=True) w, h = scipy.signal.freqs(b, a, frequencies[mask]) f[mask] = h return f elif (filter_type == 'butterabs'): - f = np.zeros_like(frequencies, dtype=np.complex) + f = np.zeros_like(frequencies, dtype=complex) mask = frequencies > 0 b, a = scipy.signal.butter(order, *scipy_args, analog=True) w, h = scipy.signal.freqs(b, a, frequencies[mask]) f[mask] = h return np.abs(f) elif(filter_type == 'cheby1'): - f = np.zeros_like(frequencies, dtype=np.complex) + f = np.zeros_like(frequencies, dtype=complex) mask = frequencies > 0 b, a = scipy.signal.cheby1(order, rp, *scipy_args, analog=True) w, h = scipy.signal.freqs(b, a, frequencies[mask]) diff --git a/NuRadioReco/utilities/noise.py b/NuRadioReco/utilities/noise.py index 2dc66b775..01648bd76 100644 --- a/NuRadioReco/utilities/noise.py +++ b/NuRadioReco/utilities/noise.py @@ -2,9 +2,11 @@ from NuRadioReco.modules import channelGenericNoiseAdder from NuRadioReco.utilities import units, fft from NuRadioReco.modules.trigger.highLowThreshold import get_high_low_triggers -from NuRadioReco.detector import detector +from NuRadioReco.detector import generic_detector as detector from scipy import constants import datetime +import scipy +import scipy.signal import copy import time @@ -31,10 +33,11 @@ def rolled_sum_roll(traces, rolling): # assume first trace always has no rolling sumtr = traces[0].copy() - for i in range(1,len(traces)): + for i in range(1, len(traces)): sumtr += np.roll(traces[i], rolling[i]) return sumtr + def rolling_indices(traces, rolling): """ pre calculates rolling index array for rolled sum via take @@ -53,6 +56,7 @@ def rolling_indices(traces, rolling): rolling_indices.append(np.roll(idx, roll)) return np.array(rolling_indices).astype(int) + def rolled_sum_take(traces, rolling_indices): """ calculates rolled sum via np.take @@ -72,10 +76,11 @@ def rolled_sum_take(traces, rolling_indices): # assume first trace always has no rolling sumtr = traces[0].copy() - for i in range(1,len(traces)): + for i in range(1, len(traces)): sumtr += np.take(traces[i], rolling_indices[i]) return sumtr + def rolled_sum_slicing(traces, rolling): """ calculates rolled sum via slicing @@ -95,7 +100,7 @@ def rolled_sum_slicing(traces, rolling): # assume first trace always has no rolling sumtr = traces[0].copy() - for i in range(1,len(traces)): + for i in range(1, len(traces)): r = rolling[i] if r > 0: sumtr[:-r] += traces[i][r:] @@ -108,8 +113,6 @@ def rolled_sum_slicing(traces, rolling): return sumtr - - class thermalNoiseGenerator(): def __init__(self, n_samples, sampling_rate, Vrms, threshold, time_coincidence, n_majority, time_coincidence_majority, @@ -228,7 +231,12 @@ class thermalNoiseGeneratorPhasedArray(): def __init__(self, detector_filename, station_id, triggered_channels, Vrms, threshold, ref_index, - noise_type="rayleigh"): + noise_type="rayleigh", log_level=logging.WARNING, + pre_trigger_time=100 * units.ns, trace_length=512 * units.ns, filt=None, + upsampling=2, window_length=16 * units.ns, step_size=8 * units.ns, + main_low_angle=np.deg2rad(-59.54968597864437), + main_high_angle=np.deg2rad(59.54968597864437), + n_beams=11, quantize=True): """ Efficient algorithms to generate thermal noise fluctuations that fulfill a phased array trigger @@ -244,27 +252,61 @@ def __init__(self, detector_filename, station_id, triggered_channels, the RMS noise threshold: float the trigger threshold (assuming a symmetric high and low threshold) - trigger_time: float - the trigger time (time when the trigger completes) - filt: array of floats - the filter that should be applied after noise generation (needs to match frequency binning) + ref_index: float + reference refractive index for calculating time delays of the beams + + Other Parameters + ---------------- noise_type: string the type of the noise, can be * "rayleigh" (default) * "noise" + log_level: logging enum, default warn + the print level for this module + pre_trigger_time: float, default 100 ns + the time in the trace before the trigger happens + trace_length: float, default 512 ns + the total trace length + filt: array of complex values, default None + the filter that should be applied after noise generation (needs to match frequency binning in upsampled domain) + if `None`, a default filter is calculated from 96 to 220 MHz + upsampling: int, default 2 + factor by which the waveforms will be upsampled before calculating time delays and beamforming + window_length: float, default 16 ns + time interval of the integration window + step_size: float, default 8 ns + duration of a stride between window calcuations + main_low_angle: float, default -59.5 deg + angle (radians) of the lowest beam + main_high_angle: float 59.5 deg + angle (radians) of the highest beam + n_beams: int, default 11 + number of beams to calculate + quantize: bool, default True + If set to true, the conversion to and from ADC will be performed to mimic digitizations """ + logger.setLevel(log_level) self.debug = False self.max_amp = 0 - self.upsampling = 2 - self.det = detector.Detector(json_filename=detector_filename) + self.upsampling = upsampling + self.det = detector.GenericDetector(json_filename=detector_filename) self.det.update(datetime.datetime(2018, 10, 1)) self.n_samples = self.det.get_number_of_samples(station_id, triggered_channels[0]) # assuming same settings for all channels self.sampling_rate = self.det.get_sampling_frequency(station_id, triggered_channels[0]) - det_channel = self.det.get_channel(station_id, triggered_channels[0]) - self.adc_n_bits = det_channel["adc_nbits"] - self.adc_noise_n_bits = det_channel["adc_noise_nbits"] + self.pre_trigger_bins = int(pre_trigger_time * self.sampling_rate) + self.n_samples_trigger = int(trace_length * self.sampling_rate) + + if self.n_samples_trigger > self.n_samples: + raise ValueError(f"Requested `trace_length` of {trace_length/units.ns:.1f}ns ({self.n_samples_trigger} bins)" + + f" is longer than the number of samples specified in the detector file ({self.n_samples} bins)") + + self.quantize = quantize + if self.quantize: + det_channel = self.det.get_channel(station_id, triggered_channels[0]) + self.adc_n_bits = det_channel["trigger_adc_nbits"] + self.adc_noise_n_bits = det_channel["trigger_adc_noise_nbits"] self.n_channels = len(triggered_channels) self.triggered_channels = triggered_channels @@ -277,12 +319,10 @@ def __init__(self, detector_filename, station_id, triggered_channels, for channel_id in triggered_channels: cable_delays[channel_id] = self.det.get_cable_delay(station_id, channel_id) - main_low_angle = np.deg2rad(-59.54968597864437) - main_high_angle = np.deg2rad(59.54968597864437) - phasing_angles_4ant = np.arcsin(np.linspace(np.sin(main_low_angle), np.sin(main_high_angle), 11)) + phasing_angles = np.arcsin(np.linspace(np.sin(main_low_angle), np.sin(main_high_angle), n_beams)) cspeed = constants.c * units.m / units.s - self.beam_time_delays = np.zeros((len(phasing_angles_4ant), self.n_channels), dtype=np.int) - for iBeam, angle in enumerate(phasing_angles_4ant): + self.beam_time_delays = np.zeros((len(phasing_angles), self.n_channels), dtype=np.int) + for iBeam, angle in enumerate(phasing_angles): delays = [] for key in self.ant_z: @@ -293,7 +333,6 @@ def __init__(self, detector_filename, station_id, triggered_channels, roll = np.array(np.round(np.array(delays) * self.sampling_rate * self.upsampling)).astype(int) self.beam_time_delays[iBeam] = roll - print(self.beam_time_delays) self.Vrms = Vrms self.threshold = threshold self.noise_type = noise_type @@ -302,21 +341,36 @@ def __init__(self, detector_filename, station_id, triggered_channels, self.max_freq = 0.5 * self.sampling_rate * self.upsampling self.dt = 1. / self.sampling_rate self.ff = np.fft.rfftfreq(self.n_samples * self.upsampling, 1. / (self.sampling_rate * self.upsampling)) - import NuRadioReco.modules.channelBandPassFilter - channelBandPassFilter = NuRadioReco.modules.channelBandPassFilter.channelBandPassFilter() - self.filt = channelBandPassFilter.get_filter(self.ff, station_id, channel_id, self.det, - passband=[96 * units.MHz, 100 * units.GHz], filter_type='cheby1', order=4, rp=0.1) - self.filt *= channelBandPassFilter.get_filter(self.ff, station_id, channel_id, self.det, - passband=[0 * units.MHz, 220 * units.MHz], filter_type='cheby1', order=7, rp=0.1) + + # Construct a default filter if one is not supplied + if filt is None: + import NuRadioReco.modules.channelBandPassFilter + channelBandPassFilter = NuRadioReco.modules.channelBandPassFilter.channelBandPassFilter() + self.filt = channelBandPassFilter.get_filter(self.ff, station_id, channel_id, self.det, + passband=[96 * units.MHz, 100 * units.GHz], filter_type='cheby1', order=4, rp=0.1) + self.filt *= channelBandPassFilter.get_filter(self.ff, station_id, channel_id, self.det, + passband=[1 * units.MHz, 220 * units.MHz], filter_type='cheby1', order=7, rp=0.1) + else: + if len(filt) != len(self.ff): + raise ValueError(f"Frequency filter supplied has {len(filt)} bins. It should match the upsampled" + + f" frequency binning of {len(self.ff)} bins from {self.ff[0] / units.MHz:.0f} to {self.ff[-1] / units.MHz:.0f} MHz") + self.filt = np.array(filt) + self.norm = np.trapz(np.abs(self.filt) ** 2, self.ff) self.amplitude = (self.max_freq - self.min_freq) ** 0.5 / self.norm ** 0.5 * self.Vrms - print(f"Vrms = {self.Vrms:.2f}, noise amplitude = {self.amplitude:.2f}, bandwidth = {self.norm/units.MHz:.0f}MHz") - print(f"frequency range {self.min_freq/units.MHz}MHz - {self.max_freq/units.MHz}MHz") + logger.info(f"Vrms = {self.Vrms:.3g}V, noise amplitude = {self.amplitude:.3g}V, bandwidth = {self.norm / units.MHz:.0f}MHz") + logger.info(f"frequency range {self.min_freq / units.MHz}MHz - {self.max_freq / units.MHz}MHz") + + if self.quantize: + self.adc_ref_voltage = self.Vrms * (2 ** (self.adc_n_bits - 1) - 1) / (2 ** (self.adc_noise_n_bits - 1) - 1) - self.adc_ref_voltage = self.Vrms * (2 ** (self.adc_n_bits - 1) - 1) / (2 ** (self.adc_noise_n_bits - 1) - 1) + self.window = int(window_length * self.sampling_rate * self.upsampling) + self.step = int(step_size * self.sampling_rate * self.upsampling) - self.window = int(16 * units.ns * self.sampling_rate * 2.0) - self.step = int(8 * units.ns * self.sampling_rate * 2.0) + if self.window >= self.pre_trigger_bins: + logger.warning(f"Pre-trigger time ({pre_trigger_time / units.ns:0.2} ns, {self.pre_trigger_bins} bins)" + + f" is within one window ({self.window} bins) of the beginning of the waveform" + + " it is recommended to choose a larger value to avoid clipping effects") self.noise = channelGenericNoiseAdder.channelGenericNoiseAdder() @@ -325,22 +379,23 @@ def __init__(self, detector_filename, station_id, triggered_channels, self.sampling_rate * self.upsampling, self.amplitude, self.noise_type) - def __generation(self): """ separated trace generation part for PA noise trigger """ for iCh in range(self.n_channels): - #spec = self.noise.bandlimited_noise(self.min_freq, self.max_freq, self.n_samples * self.upsampling, + # spec = self.noise.bandlimited_noise(self.min_freq, self.max_freq, self.n_samples * self.upsampling, # self.sampling_rate * self.upsampling, # self.amplitude, self.noise_type, time_domain=False) - + # function that does not re-calculate parameters in each simulated trace spec = self.noise.bandlimited_noise_from_precalculated_parameters(self.noise_type, time_domain=False) spec *= self.filt trace = fft.freq2time(spec, self.sampling_rate * self.upsampling) - self._traces[iCh] = perfect_floor_comparator(trace, self.adc_n_bits, self.adc_ref_voltage) - + if self.quantize: + self._traces[iCh] = perfect_floor_comparator(trace, self.adc_n_bits, self.adc_ref_voltage) + else: + self._traces[iCh] = trace def __phasing(self): """ separated phasing part for PA noise trigger """ @@ -362,43 +417,47 @@ def __triggering(self): self.max_amp = 0 # take square over entire array - coh_sum_squared = self._phased_traces**2 + coh_sum_squared = self._phased_traces ** 2 # bin the data into windows of length self.step and normalise to step length - reduced_array = np.add.reduceat(coh_sum_squared.T,np.arange(0,np.shape(coh_sum_squared)[1],self.step)).T / self.step + reduced_array = np.add.reduceat(coh_sum_squared.T, np.arange(0, np.shape(coh_sum_squared)[1], self.step)).T / self.step sliding_windows = [] # self.window can extend over multiple steps, # assuming self.window being an integer multiple of self.step the reduction sums over subsequent steps - steps_per_window = self.window//self.step + steps_per_window = self.window // self.step # better extend the array in order to also trigger on sum of last/first (matching a previous implementation) - extended_reduced_array = np.column_stack([reduced_array, reduced_array[:,0:steps_per_window]]) + extended_reduced_array = np.column_stack([reduced_array, reduced_array[:, 0:steps_per_window]]) for offset in range(steps_per_window): # sum over steps_per_window adjacent steps window_sum = np.add.reduceat(extended_reduced_array.T, np.arange(offset, np.shape(extended_reduced_array)[1], steps_per_window)).T / steps_per_window sliding_windows.append(window_sum) - #self.max_amp = max(np.array(sliding_windows).max(), self.max_amp) + # self.max_amp = max(np.array(sliding_windows).max(), self.max_amp) self.max_amp = np.array(sliding_windows).max() # check if trigger condition is fulfilled anywhere if self.max_amp > self.threshold: - # check in which beam the trigger condition was fulfilled - sliding_windows = np.concatenate(sliding_windows, axis=1) - triggered_beams = np.amax(sliding_windows, axis=1) > self.threshold - for iBeam, is_triggered in enumerate(triggered_beams): - # print out each beam that has triggered - if is_triggered: - logger.info(f"triggered at beam {iBeam}") + sliding_windows = np.array(sliding_windows) + tmp = np.argwhere(sliding_windows > self.threshold) + triggered_step, triggered_beam, triggered_bin = tmp[0] + triggered_bin *= (self.window + triggered_step * steps_per_window) if(self.debug): + # check in which beam the trigger condition was fulfilled + sliding_windows = np.concatenate(sliding_windows, axis=1) + triggered_beams = np.amax(sliding_windows, axis=1) > self.threshold + for iBeam, is_triggered in enumerate(triggered_beams): + # print out each beam that has triggered + if is_triggered: + logger.info(f"triggered at beam {iBeam}") import matplotlib.pyplot as plt - fig, ax = plt.subplots(5, 1, sharex=True) + fig, ax = plt.subplots(self.n_channels + 1, 1, sharex=True) for iCh in range(self.n_channels): ax[iCh].plot(self._traces[iCh]) logger.info(f"{self._traces[iCh].std():.2f}") - ax[4].plot(self._phased_traces[iBeam]) + ax[self.n_channels].plot(self._phased_traces[iBeam]) fig.tight_layout() plt.show() - return True - return False + return True, triggered_bin, triggered_beam + return False, None, None def __triggering_strided(self): """ separated trigger part for PA noise trigger using np.lib.stride_tricks.as_strided """ @@ -411,22 +470,22 @@ def __triggering_strided(self): coh_sum_windowed = np.lib.stride_tricks.as_strided(coh_sum_squared, (num_frames, self.window), (coh_sum_squared.strides[0] * self.step, coh_sum_squared.strides[0])) squared_mean = np.sum(coh_sum_windowed, axis=1) / self.window - #self.max_amp = max(squared_mean.max(), self.max_amp) + # self.max_amp = max(squared_mean.max(), self.max_amp) self.max_amp = max(squared_mean.max(), self.max_amp) if True in (squared_mean > self.threshold): - logger.info(f"triggered at beam {iBeam}") + triggered_bin = np.where(squared_mean > self.threshold)[0,0] + logger.debug(f"triggered at beam {iBeam}") if(self.debug): import matplotlib.pyplot as plt - fig, ax = plt.subplots(5, 1, sharex=True) + fig, ax = plt.subplots(self.n_channels+1, 1, sharex=True) for iCh in range(self.n_channels): ax[iCh].plot(self._traces[iCh]) logger.info(f"{self._traces[iCh].std():.2f}") - ax[4].plot(self._phased_traces[iBeam]) + ax[self.n_channels].plot(self._phased_traces[iBeam]) fig.tight_layout() plt.show() - return True - return False - + return True, triggered_bin, iBeam + return False, None, None def generate_noise(self, phasing_mode="slice", trigger_mode="binned_sum", debug=False): """ @@ -444,7 +503,7 @@ def generate_noise(self, phasing_mode="slice", trigger_mode="binned_sum", debug= generate debug plot Returns ------- - np.array of shape (n_channels, n_samples) + np.array of shape (n_channels, n_samples), index of triggered bin, index of triggered beam """ self.debug = debug @@ -460,7 +519,7 @@ def generate_noise(self, phasing_mode="slice", trigger_mode="binned_sum", debug= while True: counter += 1 if(counter % 1000 == 0): - logger.info(f"{counter:d}, {self.max_amp:.2f}, threshold = {self.threshold:.2f}") + logger.info(f"{counter:d}, {self.max_amp:.2g}, threshold = {self.threshold:.2g}") # some printout for profiling logger.info(f"Time consumption: GENERATION: {dt_generation:.4f}, PHASING: {dt_phasing:.4f}, TRIGGER: {dt_triggering:.4f}") tstart = time.process_time() @@ -480,15 +539,15 @@ def generate_noise(self, phasing_mode="slice", trigger_mode="binned_sum", debug= logger.error(f"Requested phasing_mode {phasing_mode}. Only 'slice' and 'roll' are allowed") raise NotImplementedError(f"Requested phasing_mode {phasing_mode}. Only 'slice' and 'roll' are allowed") - # time profiling phasing + # time profiling phasing dt_phasing += time.process_time() - tstart tstart = time.process_time() if trigger_mode == "binned_sum": - is_triggered = self.__triggering() + is_triggered, triggered_bin, triggered_beam = self.__triggering() elif trigger_mode == "stride": # more time consuming attempt to do triggering compared to taking binned sums - is_triggered = self.__triggering_strided() + is_triggered, triggered_bin, triggered_beam = self.__triggering_strided() else: logger.error(f"Requested trigger_mode {trigger_mode}. Only 'binned_sum' and 'stride' are allowed") raise NotImplementedError(f"Requested trigger_mode {trigger_mode}. Only 'binned_sum' and 'stride' are supported") @@ -497,7 +556,21 @@ def generate_noise(self, phasing_mode="slice", trigger_mode="binned_sum", debug= dt_triggering += time.process_time() - tstart if is_triggered: - return self._traces, self._phased_traces + triggered_bin = triggered_bin // self.upsampling # the trace is cut in the downsampled version. Therefore, triggered bin is factor of two smaller. + i_low = triggered_bin - self.pre_trigger_bins + i_high = i_low + self.n_samples_trigger + + # traces need to be downsampled + # resample and use axis -1 since trace might be either shape (N) for analytic trace or shape (3,N) for E-field + self._traces = scipy.signal.resample(self._traces, np.shape(self._traces)[-1] // self.upsampling, axis=-1) + + if (i_low >= 0) and (i_high < self.n_samples): # If range is directly a subset of the waveform + return self._traces[:, i_low:i_high], self._phased_traces, triggered_beam + + # Otherwise, roll the waveforms. Safe as long as noise is generated in the freq domain + self._phased_traces = np.roll(self._phased_traces, -i_low * self.upsampling, axis=-1) + self._traces = np.roll(self._traces, -i_low, axis=-1) + return self._traces[:, :self.n_samples_trigger], self._phased_traces, triggered_beam def generate_noise2(self, debug=False): """ @@ -512,7 +585,7 @@ def generate_noise2(self, debug=False): while True: counter += 1 if(counter % 1000 == 0): - print(f"{counter:d}, {max_amp:.2f}, threshold = {self.threshold:.2f}") + print(f"{counter:d}, {max_amp:.3g}, threshold = {self.threshold:.3g}") for iCh in range(self.n_channels): spec = self.noise.bandlimited_noise(self.min_freq, self.max_freq, self.n_samples * self.upsampling, self.sampling_rate * self.upsampling, @@ -520,7 +593,10 @@ def generate_noise2(self, debug=False): spec *= self.filt trace = fft.freq2time(spec, self.sampling_rate * self.upsampling) - traces[iCh] = perfect_floor_comparator(trace, self.adc_n_bits, self.adc_ref_voltage) + if self.quantize: + self._traces[iCh] = perfect_floor_comparator(trace, self.adc_n_bits, self.adc_ref_voltage) + else: + self._traces[iCh] = trace shifts = np.zeros(self.n_channels, dtype=np.int) shifted_traces = copy.copy(traces) diff --git a/NuRadioReco/utilities/trace_utilities.py b/NuRadioReco/utilities/trace_utilities.py index ca0fe352d..57bc82fb4 100644 --- a/NuRadioReco/utilities/trace_utilities.py +++ b/NuRadioReco/utilities/trace_utilities.py @@ -32,7 +32,7 @@ def get_efield_antenna_factor(station, frequencies, channels, detector, zenith, antenna_pattern_provider: AntennaPatternProvider """ n_ice = ice.get_refractive_index(-0.01, detector.get_site(station.get_id())) - efield_antenna_factor = np.zeros((len(channels), 2, len(frequencies)), dtype=np.complex) # from antenna model in e_theta, e_phi + efield_antenna_factor = np.zeros((len(channels), 2, len(frequencies)), dtype=complex) # from antenna model in e_theta, e_phi for iCh, channel_id in enumerate(channels): zenith_antenna = zenith t_theta = 1. @@ -88,12 +88,12 @@ def get_channel_voltage_from_efield(station, electric_field, channels, detector, spectrum = electric_field.get_frequency_spectrum() efield_antenna_factor = get_efield_antenna_factor(station, frequencies, channels, detector, zenith, azimuth, antenna_pattern_provider) if return_spectrum: - voltage_spectrum = np.zeros((len(channels), len(frequencies)), dtype=np.complex) + voltage_spectrum = np.zeros((len(channels), len(frequencies)), dtype=complex) for i_ch, ch in enumerate(channels): voltage_spectrum[i_ch] = np.sum(efield_antenna_factor[i_ch] * np.array([spectrum[1], spectrum[2]]), axis=0) return voltage_spectrum else: - voltage_trace = np.zeros((len(channels), 2 * (len(frequencies) - 1)), dtype=np.complex) + voltage_trace = np.zeros((len(channels), 2 * (len(frequencies) - 1)), dtype=complex) for i_ch, ch in enumerate(channels): voltage_trace[i_ch] = fft.freq2time(np.sum(efield_antenna_factor[i_ch] * np.array([spectrum[1], spectrum[2]]), axis=0), electric_field.get_sampling_rate()) return np.real(voltage_trace) @@ -218,7 +218,7 @@ def apply_butterworth(spectrum, frequencies, passband, order=8): The filtered spectrum """ - f = np.zeros_like(frequencies, dtype=np.complex) + f = np.zeros_like(frequencies, dtype=complex) mask = frequencies > 0 b, a = scipy.signal.butter(order, passband, 'bandpass', analog=True) w, h = scipy.signal.freqs(b, a, frequencies[mask]) diff --git a/README.md b/README.md index 799965a22..385697f19 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ -# NuRadioMC/NuRadioReco -A Monte Carlo simulation package for radio neutrino detectors and reconstruction framework for radio detectors of high-energy neutrinos and cosmic-rays +# NuRadioMC and NuRadioReco +NuRadioMC: A Monte Carlo simulation package for radio neutrino detectors. + +NuRadioReco: A reconstruction and detector simulation framework for radio detectors of high-energy neutrinos and cosmic-rays. (NuRadioReco is independet of NuRadioMC +and used by NuRadioMC for the detector and trigger simulation. To simplify the development and continuous integration testing, both codes are developed +in the same github repository.) The documentation can be found at https://nu-radio.github.io/NuRadioMC/main.html -If you want to keep up to date, consider signing up to the following email lists: - * user email list, will be used to announce new versions and major improvements etc. Subscribe via https://lists.uu.se/sympa/subscribe/physics-astro-nuradiomc - * developer email list, will be used to discuss the future development of NuRadioMC/Reco. Subscribe via: https://lists.uu.se/sympa/subscribe/physics-astro-nuradiomc-dev If you're using NuRadioMC for your research, please cite @@ -16,7 +17,6 @@ and for the detector simulation and event reconstruction part * C. Glaser, A. Nelles, I. Plaisier, C. Welling et al., "NuRadioReco: A reconstruction framework for radio neutrino detectors", [Eur. Phys. J. C (2019) 79: 464](https://dx.doi.org/10.1140/epjc/s10052-019-6971-5), [arXiv:1903.07023](https://arxiv.org/abs/1903.07023) - NuRadioMC is continuously improved and new features are being added. The following papers document new features (in reverse chronological order): * N. Heyer and C. Glaser, “First-principle calculation of birefringence effects for in-ice radio detection of neutrinos”, [arXiv:2205.06169](https://arxiv.org/abs/2205.15872) (adds birefringence modelling to NuRadioMC) @@ -28,18 +28,31 @@ NuRadioMC is continuously improved and new features are being added. The followi * D. García-Fernández, C. Glaser and A. Nelles, “The signatures of secondary leptons in radio-neutrino detectors in ice”, [Phys. Rev. D 102, 083011](https://dx.doi.org/10.1103/PhysRevD.102.083011), [arXiv:2003.13442](https://arxiv.org/abs/2003.13442) (addition of secondary interactions of muons and taus) - If you would like to contribute, please contact @cg-laser or @anelles for permissions to work on NuRadioMC. We work with pull requests only that can be merged after review. Also please visit https://nu-radio.github.io/NuRadioMC/Introduction/pages/contributing.html for details on our workflow and coding conventions. -NuRadioMC is used in an increasing number of studies. To get an overview for what NuRadioMC can be used for, please have a look at the following publications or see [here](https://inspirehep.net/literature?sort=mostrecent&size=25&page=1&q=refersto%3Arecid%3A1738571%20or%20refersto%3Arecid%3A1725583): +## Publications builing up on NuRadioMC/Reco +NuRadioMC is used in an increasing number of studies. To get an overview for what NuRadioMC can be used for, please have a look at the following publications or see [here](https://inspirehep.net/literature?sort=mostrecent&size=25&page=1&q=refersto%3Arecid%3A1738571%20or%20refersto%3Arecid%3A1725583): -* V. B. Valera, M. Bustamante and C. Glaser, “Near-future discovery of the diffuse flux of ultra-high-energy cosmic neutrinos”, [arXiv:2210.03756](https://arxiv.org/abs/2210.03756) -* Alfonso Garcia Soto, Diksha Garg, Mary Hall Reno, Carlos A. Argüelles, "Probing Quantum Gravity with Elastic Interactions of Ultra-High-Energy Neutrinos", [arXiv:2209.06282](https://arxiv.org/abs/2209.06282) -* Damiano F. G. Fiorillo, Mauricio Bustamante, Victor B. Valera, "Near-future discovery of point sources of ultra-high-energy neutrinos", [arXiv:2205.15985](https://arxiv.org/abs/2205.15985) +* IceCube-Gen2 collaboration, [IceCube-Gen2 Technical Design Report](https://icecube-gen2.wisc.edu/science/publications/TDR) +* ARIANNA collaboration (A. Anker et al.), "Developing New Analysis Tools for Near Surface Radio-based Neutrino Detectors", [arXiv:2307.07188](https://arxiv.org/abs/2307.07188) +* L. Pyras, C. Glaser S. Hallmann and A. Nelles, "Atmospheric muons at PeV energies in radio neutrino detectors", [arXiv:2307.04736](https://arxiv.org/abs/2307.04736) +* I. Plaisier, S. Bouma, A. Nelles, "Reconstructing the arrival direction of neutrinos in deep in-ice radio detectors", [arXiv:2302.00054](https://arxiv.org/abs/2302.00054) +* S. Bouma, A. Nelles for the IceCube-Gen2 collaboration, "Direction reconstruction performance for IceCube-Gen2 Radio", [PoS(ICRC2023)1045](https://pos.sissa.it/444/1045/pdf) +* F. Schlüter and S. Toscano for the IceCube-Gen2 collaboration, "Estimating the coincidence rate between the optical and radio array of IceCube-Gen2", [PoS(ICRC2023)1022](https://pos.sissa.it/444/1022/pdf) +* C. Glaser, A. Coleman and T. Glusenkamp, "NuRadioOpt: Optimization of Radio Detectors of Ultra-High Energy Neutrinos through Deep Learning and Differential Programming", [PoS(ICRC2023)1114](https://pos.sissa.it/444/1114/pdf) +* A. Coleman and C. Glaser for the RNO-G collaboration, "Enhancing the Sensitivity of RNO-G Using a Machine-learning Based Trigger", [PoS(ICRC2023)1100](https://pos.sissa.it/444/1100/pdf) +* N. Heyer, C. Glaser and T. Glusenkamp for the IceCube-Gen2 collaboration, "Deep Learning Based Event Reconstruction for the IceCube-Gen2 Radio Detector" [PoS(ICRC2023)1102](https://pos.sissa.it/444/1102/pdf) +* N. Heyer and C. Glaser, "Impact of Birefringence on In-Ice Radio Detectors of ultra-high-energy Neutrinos", [PoS(ICRC2023)1101](https://pos.sissa.it/444/1101/pdf) +* J. Henrichs, A. Nelles for the RNO-G Collaboration, "Searching for cosmic-ray air showers with RNO-G", [PoS(ICRC2023)259](https://pos.sissa.it/444/259/pdf) +* B. Oeyen for the RNO-G Collaboration, "The interplay of ice-firn model and station calibration in RNO-G", [PoS(ICRC2023)1042](https://pos.sissa.it/444/1042/pdf) +* P. Windischhofer, C. Welling and C. Deaconu, "Eisvogel: Exact and efficient calculations of radio emissions from in-ice neutrino showers", [PoS(ICRC2023)1157](https://pos.sissa.it/444/1157/) +* V. B. Valera, M. Bustamante and C. Glaser, “Near-future discovery of the diffuse flux of ultra-high-energy cosmic neutrinos”, Phys. Rev. D 107, 043019 [arXiv:2210.03756](https://arxiv.org/abs/2210.03756) +* Alfonso Garcia Soto, Diksha Garg, Mary Hall Reno, Carlos A. Argüelles, "Probing Quantum Gravity with Elastic Interactions of Ultra-High-Energy Neutrinos", Phys. Rev. D 107, 033009 (2023) [arXiv:2209.06282](https://arxiv.org/abs/2209.06282) +* Damiano F. G. Fiorillo, Mauricio Bustamante, Victor B. Valera, "Near-future discovery of point sources of ultra-high-energy neutrinos", JCAP 03 (2023) 026 [arXiv:2205.15985](https://arxiv.org/abs/2205.15985) * C. Glaser, S. McAleer, S. Stjärnholm, P. Baldi, S. W. Barwick, “Deep learning reconstruction of the neutrino direction and energy from in-ice radio detector data”, Astroparticle Physics 145, (2023) 102781, [doi:10.1016/j.astropartphys.2022.102781](https://doi.org/10.1016/j.astropartphys.2022.102781), [arXiv:2205.15872](https://arxiv.org/abs/2205.15872) -* J. Beise and C. Glaser, “In-situ calibration system for the measurement of the snow accumulation and the index-of-refraction profile for radio neutrino detectors”, [arXiv:2205.00726](https://arxiv.org/abs/2205.00726) +* J. Beise and C. Glaser, “In-situ calibration system for the measurement of the snow accumulation and the index-of-refraction profile for radio neutrino detectors”, Journal of Instrumentation 18 P01036 (2023), [arXiv:2205.00726](https://arxiv.org/abs/2205.00726) * V. B. Valera, M. Bustamante and C. Glaser, “The ultra-high-energy neutrino-nucleon cross section: measurement forecasts for an era of cosmic EeV-neutrino discovery”, Journal of High Energy Physics 06 (2022) 105, [doi:10.1007/JHEP06(2022)105](https://doi.org/10.1007/JHEP06(2022%29105), [arXiv:2204.04237](https://arxiv.org/abs/2204.04237) * ARIANNA collaboration (A. Anker et al.), “Measuring the Polarization Reconstruction Resolution of the ARIANNA Neutrino Detector with Cosmic Rays”, Journal of Cosmology and Astroparticle Physics 04(2022)022, [doi:10.1088/1475-7516/2022/04/022](https://doi.org/10.1088/1475-7516/2022/04/022), [arXiv:2112.01501](https://arxiv.org/abs/2112.01501) * ARIANNA collaboration (A. Anker et al.), “Improving sensitivity of the ARIANNA detector by rejecting thermal noise with deep learning”, Journal of Instrumentation 17 P03007 (2022), [doi:10.1088/1748-0221/17/03/P03007](https://doi.org/10.1088/1748-0221/17/03/P03007), [arXiv:2112.01031](https://arxiv.org/abs/2112.01031) @@ -60,3 +73,8 @@ NuRadioMC is used in an increasing number of studies. To get an overview for wha * C. Glaser, S. Barwick, "An improved trigger for Askaryan radio detectors", [JINST 16 (2021) 05, T05001](https://doi.org/10.1088/1748-0221/16/05/T05001), [arXiv:2011.12997](https://arxiv.org/abs/2011.12997) * RNO-G collaboration, "Design and Sensitivity of the Radio Neutrino Observatory in Greenland (RNO-G)", [JINST 16 (2021) 03, P03025](https://doi.org/10.1088/1748-0221/16/03/P03025) [arXiv:2010.12279](https://arxiv.org/abs/2010.12279) * ARIANNA collaboration, "Probing the angular and polarization reconstruction of the ARIANNA detector at the South Pole", [JINST 15 (2020) 09, P09039](https://doi.org/10.1088/1748-0221/15/09/P09039), [arXiv:2006.03027](https://arxiv.org/abs/2006.03027) + + +If you want to keep up to date, consider signing up to the following email lists: + * user email list, will be used to announce new versions and major improvements etc. Subscribe via https://lists.uu.se/sympa/subscribe/physics-astro-nuradiomc + * developer email list, will be used to discuss the future development of NuRadioMC/Reco. Subscribe via: https://lists.uu.se/sympa/subscribe/physics-astro-nuradiomc-dev diff --git a/changelog.txt b/changelog.txt index 8d6735957..a63f64f40 100644 --- a/changelog.txt +++ b/changelog.txt @@ -4,15 +4,28 @@ please update the categories "new features" and "bugfixes" before a pull request version 2.2.0-dev new features: +- expand values stored in SimplePhasedTrigger +- added getting to query ARZ charge-excess profiles - upgrade to proposal v 7.5.0 with new NuRadioProposal interface and improved LPM treatment - add script to generate / download pre-calculated proposal tables for known detector configurations - adding default RadioPropa ice model object to medium with the feature to set a personlised object as alternative - add positions array functionality to simple ice model in average and gradient functions - analytic ray tracing solutions are now sorted consistently from lowest to highest ray - added ability to generate high-low-triggered noise on a narrow band but return full-band waveforms +- phased array noise generation utility class: calculate trigger time and cut trace accordingly +- use Philox noise generator in noise adder module (this changes the default random sequence) +- added 'block offset' removal/simulation module for RNO-G + bugfixes: - fixed/improved C++ raytracer not finding solutions for some near-horizontal or near-shadowzone vertices - fixed wrong number in Feldman-Cousins upper limit +- in antennapattern.py, fixed line 1398; was masking frequencies >= 0 instead of frequencies > 0, causing NaN errors +- Fixed issue where saving empty traces (channels or Efields) created unreadable .nur files + + +version 2.1.8 +bugfixes: +- replace deprecated np.float with float version 2.1.7 new features: @@ -25,16 +38,18 @@ new features: bugfixes: - fix bug in NuRadioProposal which pervented muons from tau decays to be propagated - update proposal version to 6.1.8 to avoid problems with pypi +- updated reference channel check in GenericDetector to look into the reference station when the channel +is not found in the current station version 2.1.6 bugfixes: -- the n_interaction parameter was accidentally overridden. n_interaction counts the number of interactions of taus and +- the n_interaction parameter was accidentally overridden. n_interaction counts the number of interactions of taus and muons generated in the corresponding CC interactions. The bug resulted in the following behaviour: The n_interaction value of the primary neutrino interaction was overridden with the n_interaction value of the last shower of the previous event group. If the previous event group was a CC interaction, the n_interaction value of the primary interaction was set to a value larger than 1. If that happened, and if the primary -shower did not trigger the detector, this shower is not added to the output hdf5 file. This has consquences for analyzing hdf5 files for -secondary interactions. With this bug, the initial interaction needs to be idenfified with np.abs(flavor) == 14 or 16. -This bug does not effect any effective volume calculation. +shower did not trigger the detector, this shower is not added to the output hdf5 file. This has consquences for analyzing hdf5 files for +secondary interactions. With this bug, the initial interaction needs to be idenfified with np.abs(flavor) == 14 or 16. +This bug does not effect any effective volume calculation. new features: - add hvsp1, update idl1 model for emitter simulation (from KU lab measurements) and remove hvsp2(no longer in use for experiment) @@ -44,7 +59,7 @@ version 2.1.5 bugfixes: - the phased array trigger module did not calculate the trigger time. - The trigger time was always set to the trace start time. This got fixed in this version. + The trigger time was always set to the trace start time. This got fixed in this version. version 2.1.4 @@ -57,7 +72,7 @@ bugfixes: version 2.1.2 bugfixes: -- Fixes that the generic detector crashes for certain detector files. +- Fixes that the generic detector crashes for certain detector files. version 2.1.1 new features: @@ -74,7 +89,7 @@ new features: - add a basic uproot data reader for RNO-G data - add option to simulate emitters - added helper function for cosmic ray flux models -- speed improvements of ARZ model through use of numba +- speed improvements of ARZ model through use of numba bugfixes: - correct volume extension depending on zenith angle range when running with @@ -93,26 +108,26 @@ bugfixes: version 2.0.0 - NuRadioReco is merged into the NuRadioMC repository. No new features were added and everything works (e.g. the imports) -as before but the NuRadioReco code is now part of the NuRadioMC github repository to simplify code development where NuRadioMC -changes depend on changes in NuRadioReco. +as before but the NuRadioReco code is now part of the NuRadioMC github repository to simplify code development where NuRadioMC +changes depend on changes in NuRadioReco. version 1.2.0 new features: -- major change in internal looping. Now, the radio signal from each shower is calculated and signal arrival times +- major change in internal looping. Now, the radio signal from each shower is calculated and signal arrival times (from different shower, ray racing solutions and receiver positions) that cluster together are simualted as one event - merge hdf5 utility has multithreading option (and was refactored -> function names changed) - distance cut can sum up all sourounding shower energies to properly account for possible interference - noise temperature can be specified per station and channel - trigger thresholds can be specified per channel -- bandwidth can be specified per station and channel +- bandwidth can be specified per station and channel - specifying the detector simulation became easier (check out the updated examples) - memory consumption was optimized to stay <4GB per core - random realization of showers are saved so that triggered events can be resimulated using the same random realization - added option for noiseless channels in a "with noise" simulation - add option to generate events on the fly and pass them directly to the simulation part (no need to save input hdf5 files anymore) - added uncertainties to CTW cross sections -- +- bugfixes: - Fixed issue with merge hdf5 utility so that "event_group_ids" are properly unique @@ -120,7 +135,7 @@ bugfixes: -version 1.1.2 - +version 1.1.2 - new features: - Veff utility can now handle extended bins - New tutorial and example for the webinar @@ -137,10 +152,10 @@ bugfixes: version 1.1.1 - 2020/03/23 new features -- New version for the ARZ model available (ARZ2020) -- a list with event ids can be passed to the main simulation class. All events not in this list will not be simulated. - This is useful for a quick resimulation of certain events. -- Alvarez???? Askaryan models now place the trace into the center of the trace (instead of 50ns into the trace) +- New version for the ARZ model available (ARZ2020) +- a list with event ids can be passed to the main simulation class. All events not in this list will not be simulated. + This is useful for a quick resimulation of certain events. +- Alvarez???? Askaryan models now place the trace into the center of the trace (instead of 50ns into the trace) - New data set array 'vertex_times' contains the time elapsed from the first interaction to the current interaction - new utility to split output hdf5 files into smaller chucks (to be able to resimulate events on a cluster) - Greenland added to proposal config @@ -148,11 +163,11 @@ new features - improved Earth model (PREM), path from interaction vertex through Earth is calculated (before interaction was assumed to happen at the surface) - detector description is saved to nur output files -- new handling of random numbers. Each module has its own random generator instance. Global seed can be controlled via +- new handling of random numbers. Each module has its own random generator instance. Global seed can be controlled via config file setting. bugfixes: -- ARZxxxx and Alvarez2009 Askaryan modules now use the same (random) shower per event. +- ARZxxxx and Alvarez2009 Askaryan modules now use the same (random) shower per event. - fixes zenith distribution of event generator back to cos(zenith) - ray tracing precision was increased to 1e-9 - saveguard against too many ray tracing solutions added @@ -176,9 +191,9 @@ bugfixes: - fixed that Veff utility script didn't calculate effective volumes for individual triggers - fixed that the multiple_triggers key was not created for multiple stations - Angular distribution now accounts for projected area - - - - + - + + version 1.0.0 - 2019/08/30 - first python 3 release diff --git a/documentation/Makefile b/documentation/Makefile index 22f3d5479..4a3fdd9d1 100644 --- a/documentation/Makefile +++ b/documentation/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -w sphinx-debug.log SPHINXBUILD = sphinx-build SPHINXPROJ = NuRadio SOURCEDIR = source diff --git a/documentation/make_docs.py b/documentation/make_docs.py index f529dce94..b7909c628 100644 --- a/documentation/make_docs.py +++ b/documentation/make_docs.py @@ -113,11 +113,12 @@ subprocess.check_output(['make', 'clean']) sphinx_log = subprocess.run(['make', 'html'], stderr=subprocess.PIPE, stdout=pipe_stdout) - # errs = sphinx_log.stderr.decode().split('\n') - errs = re.split('\\x1b\[[0-9;]+m', sphinx_log.stderr.decode()) # split the errors - # output = sphinx_log.stdout.decode().split('\n') - - for err in errs: + # we write out all the sphinx errors to sphinx-debug.log, and parse these + with open('sphinx-debug.log') as f: + errs_raw = f.read() + errs_sphinx = re.split('\\x1b\[[0-9;]+m', errs_raw) # split the errors + + for err in errs_sphinx: if not err.split(): # whitespace only continue for key in error_dict.keys(): @@ -131,20 +132,26 @@ continue fixable_errors += len(error_dict[key]['matches']) - print(2*'\n'+78*'-') + # stderr includes non-sphinx errors/warnings raised during the build process + # we record these for debugging but don't fail on them + errs_other = re.split('\\x1b\[[0-9;]+m', sphinx_log.stderr.decode()) + errs_other = [err for err in errs_other if not err in errs_sphinx] + error_dict['other']['matches'] += errs_other + + errors_string = '\n'.join([ + f'[{key}]\n' + '\n'.join(value['matches']) + for key, value in error_dict.items() if len(value['matches']) + ]) + + print(2*'\n'+78*'-', flush=True) if fixable_errors: - logger.warning("The documentation was not built without errors. Please fix the following errors!") - for key, item in error_dict.items(): - if len(item['matches']): - print(f'[{key}]') - print('\n'.join(item['matches'])) + logger.error(f"The documentation was not built without errors. Please fix the following errors!\n{errors_string}") elif len(error_dict['other']['matches']): logger.warning(( "make_docs found some errors but doesn't know what to do with them.\n" - "The documentation may not be rejected, but consider fixing the following anyway:" + f"The documentation may not be rejected, but consider fixing the following anyway:\n{errors_string}" )) - print('\n'.join(error_dict['other']['matches'])) - print(78*'-'+2*'\n') + print(78*'-'+2*'\n', flush=True) if sphinx_log.returncode: logger.error("The documentation failed to build, make_docs will raise an error.") diff --git a/documentation/source/Introduction/pages/installation.rst b/documentation/source/Introduction/pages/installation.rst index 402faee2e..62fd29dda 100644 --- a/documentation/source/Introduction/pages/installation.rst +++ b/documentation/source/Introduction/pages/installation.rst @@ -145,7 +145,7 @@ These packages are recommended to be able to use all of NuRadioMC/NuRadioReco's .. code-block:: bash - pip install uproot==4.1.1 + pip install uproot==4.1.1 'awkward<2' - To access some detector databases: @@ -189,7 +189,7 @@ These packages are recommended to be able to use all of NuRadioMC/NuRadioReco's .. code-block:: bash - pip install proposal==7.5.1 + pip install proposal==7.6.2 Note that the pip installation for this version of proposal may not work on all systems, in particular: diff --git a/documentation/source/NuRadioMC/pages/HDF5_structure.rst b/documentation/source/NuRadioMC/pages/HDF5_structure.rst index 66ad8022e..20abb7a84 100644 --- a/documentation/source/NuRadioMC/pages/HDF5_structure.rst +++ b/documentation/source/NuRadioMC/pages/HDF5_structure.rst @@ -114,8 +114,12 @@ is the number of showers (which may be larger than the number of events), and `` Station data ____________ In addition, the HDF5 file contains a key for each station in the simulation. -The station contains more detailed information for each event that triggered it: -``m_events`` and ``m_showers`` refer to the number of events and showers that triggered the station. +The station contains more detailed information for each station. Some parameters are per event and +some parameters are per shower. See https://doi.org/10.22323/1.395.1231 for a description of how showers relate to events. +``m_events`` and ``m_showers`` refer to the number of events and showers that triggered the station. NOTE: The simple table +structure of hdf5 files can not capture the complex relation between events and showers in all cases. Some fields can be ambiguous +(e.g. `trigger_times` that only lists the last trigger that a shower generated). +For more advanced analyses, please use the ``*.nur`` files. The ``event_group_id`` is the same as in the global dictionary. Therefore you can check for one event with an ``event_group_id`` which stations contain the same ``event_group_id`` and retrieve the information, which station triggered, with which amplitude, etc. The same approach works for ``shower_id``. @@ -138,7 +142,7 @@ station triggered, with which amplitude, etc. The same approach works for ``show ``maximum_amplitudes_envelope`` | (``m_events``, ``n_channels``) | Maximum amplitude of the hilbert envelope for each event and channel ``multiple_triggers`` | (``m_showers``, ``n_triggers``) | A boolean array that specifies if a shower contributed to an event that fulfills a certain trigger. The index of the trigger can be translated to the trigger name via the attribute ``trigger_names``. ``multiple_triggers_per_event`` | (``m_events``, ``n_triggers``) | A boolean array that specifies if each event fulfilled a certain trigger. The index of the trigger can be translated to the trigger name via the attribute ``trigger_names``. - ``polarization`` | (``m_showers``, ``n_channels``, ``n_ray_tracing_solutions``, ``3``) | 3D (Cartesian) coordinates of the polarization vector + ``polarization`` | (``m_showers``, ``n_channels``, ``n_ray_tracing_solutions``, ``3``) | 3D coordinates of the polarization vector at the antenna in cartesian coordinates. (The receive vector (which is opposite to the propagation direction) was used to rotate from spherical/on-sky coordinates to cartesian coordinates). The polarization vector does not include any propagation effects that could change the polarization, such as different reflectivities at the surface for the p and s polarization component. ``ray_tracing_C0`` | (``m_showers``, ``n_channels``, ``n_ray_tracing_solutions``) | One of two parameters specifying the **analytic** ray tracing solution. Can be used to retrieve the solutions without having to re-run the ray tracer. ``ray_tracing_C1`` | (``m_showers``, ``n_channels``, ``n_ray_tracing_solutions``) | One of two parameters specifying the **analytic** ray tracing solution. Can be used to retrieve the solutions without having to re-run the ray tracer. ``ray_tracing_reflection`` | (``m_showers``, ``n_channels``, ``n_ray_tracing_solutions``) | @@ -151,4 +155,5 @@ station triggered, with which amplitude, etc. The same approach works for ``show ``travel_times`` | (``m_showers``, ``n_channels``, ``n_ray_tracing_solutions``) | The time travelled by each ray tracing solution to a specific channel ``triggered`` | (``m_showers``) | Whether each shower contributed to an event that satisfied any trigger condition ``triggered_per_event`` | (``m_events``) | Whether each event fulfilled any trigger condition. - ``trigger_times`` | (``m_showers``, ``n_triggers``) | The trigger times for each shower and trigger. + ``trigger_times`` | (``m_showers``, ``n_triggers``) | The trigger times for each shower and trigger. IMPORTANT: A shower can potentially generate multiple events. Then this field is ambiguous, as only a single trigger time per shower can be saved. In that case, the latest trigger time is saved into this field. + ``trigger_times_per_event`` | (``m_events``, ``n_triggers``) | The trigger times per event. diff --git a/documentation/source/conf.py b/documentation/source/conf.py index d24163148..b5242e833 100644 --- a/documentation/source/conf.py +++ b/documentation/source/conf.py @@ -239,7 +239,7 @@ autodoc_mock_imports = [ 'ROOT', 'mysql-python', 'pygdsm', 'MySQLdb', 'healpy', 'scripts', 'uproot', 'radiopropa', 'plotly', 'past', - 'nifty5' + 'nifty5', 'mattak' ] # Raise warnings if any cross-references are broken nitpicky = True @@ -247,6 +247,7 @@ # this ignores some cross-reference errors inside docstrings # that we don't care about nitpick_ignore_regex = [ + ("py:class", "aenum._enum.Enum"), ("py:class", "aenum.Enum"), ("py:class", "tinydb_serialization.Serializer"), ("py:class", "radiopropa.ScalarField"), diff --git a/pyproject.toml b/pyproject.toml index b3b89eac1..b525276d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ tinydb = ">=4.1.1" tinydb-serialization = ">=2.1" aenum = "*" astropy = "*" -radiotools = ">=0.2" +radiotools = ">=0.2.1" cython = "*" dash = ">=2.0" future = "*" @@ -33,7 +33,7 @@ h5py = "*" peakutils = "*" pymongo = "*" pyyaml = "*" -awkward = "*" +awkward = "<2" python = "^3.6" matplotlib = "*" requests = "*" @@ -45,14 +45,21 @@ numba = "*" [tool.poetry.dev-dependencies] Sphinx = "*" sphinx-rtd-theme = "*" -numpydoc = "1.1.0" -proposal = "7.5.1" +numpydoc = "*" +proposal = "7.6.2" pygdsm = {git = "https://github.com/telegraphic/pygdsm"} nifty5 = {git = "https://gitlab.mpcdf.mpg.de/ift/nifty.git", branch="NIFTy_5"} pypocketfft = {git = "https://gitlab.mpcdf.mpg.de/mtr/pypocketfft"} +MCEq = "*" +crflux = "*" +pandas = "*" +mattak = {git = "https://github.com/RNO-G/mattak"} +runtable = {git = "ssh://git@github.com/RNO-G/rnog-runtable.git"} [tool.poetry.extras] documentation = ["Sphinx", "sphinx-rtd-theme", "numpydoc"] proposal = ["proposal"] galacticnoise = ['pygdsm'] ift_reco = ['nifty5', 'pypocketfft'] +muon_flux_calc = ['MCEq', 'crflux'] +RNO_G_DATA = ["mattak", "runtable", "pandas"]