Commit 84c89bad authored by matt24smith's avatar matt24smith
Browse files

add ship_type when generating track vectors and update docstrings

parent 019b80b9
Pipeline #4979 passed with stages
in 2 minutes and 34 seconds
......@@ -59,21 +59,15 @@ def append_file(picklefile, batch):
# skip empty rows
if len(rows) == 0:
continue
# skip duplicate epoch-minute timestamps for each mmsi
#skipidx = np.nonzero([x['mmsi']==y['mmsi'] and x['epoch']==y['epoch'] for x,y in zip(rows[1:], rows[:-1])])[0]-1
skipidx = np.nonzero([
x['mmsi'] == y['mmsi'] and x['type'] == y['type']
and x['epoch'] == y['epoch'] for x, y in zip(rows[1:], rows[:-1])
])[0]
# write to disk
with open(f'{picklefile}_{key}', 'ab') as f:
pickle.dump(rows[~skipidx], f)
pickle.dump(rows, f)
# def decode_raw_pyais(fpath, tmpdir):
def decode_raw_pyais(fpath, tmp_dir=tmp_dir):
''' parallel process worker function. see decode_msgs() for usage
''' parallel process worker function. see decode_msgs() for usage.
if Rust is installed, this function will be ignored.
arg:
fpath: (string)
......@@ -226,7 +220,9 @@ def insert_serialized(dbpath, delete=True):
def decode_msgs(filepaths, dbpath, processes=12, delete=True):
''' decode NMEA binary message format and store in an SQLite database
''' Decode NMEA format AIS messages and store in an SQLite database.
To speed up decoding, create the database on a different hard drive
from where the raw data is stored.
args:
filepaths (list)
......@@ -292,9 +288,8 @@ def decode_msgs(filepaths, dbpath, processes=12, delete=True):
proc = partial(decode_raw_pyais)
# parallelize decoding step
print(
f'decoding messages... results will be placed temporarily in {tmp_dir} until database insert'
)
print('decoding messages... results will be placed temporarily '
f'in {tmp_dir} until database insert')
if processes:
with Pool(processes) as p:
list(p.imap_unordered(proc, filepaths))
......
......@@ -64,8 +64,7 @@ def TrackGen(
'dim_stern',
'dim_port',
'dim_star',
# 'ship_type',
# 'ship_type_txt',
'ship_type',
],
deduplicate_timestamps: bool = False,
) -> dict:
......@@ -120,7 +119,7 @@ def TrackGen(
staticcols = set(colnames) & set([
'mmsi',
'vessel_name',
# 'ship_type',
'ship_type',
# 'ship_type_txt',
'dim_bow',
'dim_stern',
......
__version__ = "1.0.21"
__version__ = "1.0.22"
......@@ -2,6 +2,12 @@
Changelog
=========
v1.0.22
-------
add ship_type when generating track vectors and update docstrings
v1.0.21
-------
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment