Commit 59355eb6 authored by Oliver Kirsebom's avatar Oliver Kirsebom
Browse files

Merge branch 'FAV_filter' into 'master'

Fav filter

See merge request data_analytics_dal/projects/boat_detector!1
parents 8d175762 222e535b
......@@ -16,14 +16,15 @@ from ketos.data_handling.parsing import str2bool
ureg = UnitRegistry()
PeakFindingConfig = namedtuple('PeakFindingConfig', 'separation size multiplicity')
PeakFindingConfig.__new__.__defaults__ = (60, 3.0, 1)
PeakFindingConfig = namedtuple('PeakFindingConfig', 'separation size multiplicity height')
PeakFindingConfig.__new__.__defaults__ = (60, 3.0, 1, 0)
PeakFindingConfig.__doc__ = '''\
Configuration of peak finding algorithm
separation - Minimum temporal separation between neighboring peaks in seconds
size - Minimum peak height relative to baseline given in multiples of the signal standard devitation (float)
multiplicity - Minimum number of data series in which peak occurs'''
multiplicity - Minimum number of data series in which peak occurs
height - minimum absolute height of peak'''
SVMConfig = namedtuple('SVMConfig', 'nu kernel gamma degree training_data')
SVMConfig.__new__.__defaults__ = (0.01, "poly", 0.001, 2, "None")
......@@ -91,7 +92,9 @@ def parse_peak_finding_config(data):
size = float(d['prominence'])
if d['multiplicity'] is not None:
multiplicity = int(d['multiplicity'])
res = PeakFindingConfig(separation=separation, size=size, multiplicity=multiplicity)
if d['height'] is not None:
height = float(d['height'])
res = PeakFindingConfig(separation=separation, size=size, multiplicity=multiplicity, height=height)
return s, res
def parse_svm_config(data):
......@@ -199,7 +202,7 @@ def main():
cfg = configs['peak_finding_config']
time_res = extract_time_res(X)
dist = max(1, int(cfg.separation / time_res))
df['Peak Finding'] = detect_peaks(X, distance=dist, prominence=cfg.size, multiplicity=cfg.multiplicity)
df['Peak Finding'] = detect_peaks(X, distance=dist, prominence=cfg.size, multiplicity=cfg.multiplicity, height=cfg.height)
#====================
......@@ -270,7 +273,6 @@ def main():
if time_table != None:
df_tt = pd.read_csv(time_table)
df_tt = df_tt.sort_index(ascending=False, axis=0)
print(df_tt)
fnames = list()
times = list()
for t in df_out['time']:
......@@ -304,4 +306,4 @@ def main():
if __name__ == '__main__':
main()
\ No newline at end of file
main()
{
"data_series": ["num_peaks"],
"detectors": ["PEAK_FINDING"],
"median_subtraction": "False",
"anomaly_separation": "2 minute",
"peak_finding_config": {
"separation": "2 minutes",
"prominence": "2.0",
"multiplicity": "1",
"height": "2"
}
}
{
"date_time_format": "HMS_%H_%M_%S__DMY_%d_%m_%y",
"batch_size": "5E6",
"filters": ["FREQUENCY", "MEDIAN", "MEDIAN_SUBTRACTION"],
"frequency_config": [
{
"name": "31.2Hz",
"range": ["22.1Hz", "44.2Hz"]
},
{
"name": "62.5Hz",
"range": ["44.2Hz", "88.4Hz"]
},
{
"name": "125Hz",
"range": ["88.4Hz", "176.8Hz"]
},
{
"name": "250Hz",
"range": ["176.8Hz", "353.6Hz"]
},
{
"name": "500Hz",
"range": ["353.6Hz", "707.1Hz"]
},
{
"name": "1kHz",
"range": ["707.1Hz", "1414.2Hz"]
},
{
"name": "2kHz",
"range": ["1414.2Hz", "2828.4Hz"]
}
],
"median_config": {
"window_size": "1 min",
"step_size": "1 min"
},
"median_subtraction_config": {
"window_size": "1 hour"
},
"spectrogram": {
"rate": "4000 Hz",
"window_size": "0.1 s",
"step_size": "0.025 s",
"window_function": "HAMMING"
}
}
......@@ -16,7 +16,7 @@ import ketos.data_handling.parsing as pa
from ketos.audio_processing.audio import AudioSignal, TimeStampedAudioSignal
from ketos.audio_processing.spectrogram import MagSpectrogram
from ketos.data_handling.data_handling import AudioSequenceReader
from ketos.audio_processing.spectrogram_filters import FrequencyFilter, WindowFilter, WindowSubtractionFilter, CroppingFilter, HarmonicFilter
from ketos.audio_processing.spectrogram_filters import FrequencyFilter, WindowFilter, WindowSubtractionFilter, CroppingFilter, HarmonicFilter, FAVFilter, FAVThresholdFilter
import time
from collections import namedtuple
from pint import UnitRegistry # SI units
......@@ -78,6 +78,17 @@ def parse_config(path):
elif x == 'HARMONIC':
f = HarmonicFilter()
filters.append(f)
elif x == 'FAV':
f = FAVFilter()
filters.append(f)
elif x == 'FAV_THRESHOLD':
threshold = 3.0
if data.get('fav_config') is not None:
threshold = float(data['fav_config'].get('threshold'))
f = FAVThresholdFilter(threshold=threshold)
filters.append(f)
else:
print('Warning: Unknown filter {0} will be ignored'.format(x))
# spectrogram
spectr_config = pa.parse_spectrogram_configuration(data['spectrogram'])
......@@ -270,12 +281,20 @@ def main(config_file=None, input_dir=None, output_file=None, show_graph=None, sa
elapsed_time = time.time() - start_time
print(time.strftime(" Elapsed time: %H:%M:%S", time.gmtime(elapsed_time)))
# number of columns
N = len(filtered_data.columns)
# plot
if save_graph or show_graph:
ax = plt.gca()
ax.set_xlabel('Time')
ax.set_ylabel('Filtered signal')
filtered_data.plot(y=filtered_data.columns, ax=ax)
if N < 10:
filtered_data.plot(y=filtered_data.columns, ax=ax)
else:
img = np.log10(filtered_data.to_numpy())
img_plot = ax.imshow(img.T, aspect='auto', origin='lower', extent=(0, 1, 0, 1))
if save_graph:
fname = output_file[:output_file.rfind('.')] + '.png'
plt.savefig(fname)
......
from setuptools import setup, find_packages
setup(name='boat_detector',
version='0.0.2',
version='0.0.3',
description="Python scripts for detecting noise made by boats in broadband hydrophone data",
url='https://gitlab.meridian.cs.dal.ca/data_analytics_dal/packages/boat_detector',
author='Oliver Kirsebom, Fabio Frazao',
......@@ -9,7 +9,7 @@ setup(name='boat_detector',
license='GNU General Public License v3.0',
packages=find_packages(),
install_requires=[
'ketos==1.0.1',
'ketos==1.0.3',
],
entry_points = {"console_scripts": ["audio-filterer=bin.audio_filterer:main", "anomaly-detector=bin.anomaly_detector:main"]},
include_package_data=True,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment