From cd66adaa800db7f78245e46997e78b13fd909e91 Mon Sep 17 00:00:00 2001 From: Federico Lolli <federico.lolli@skywarder.eu> Date: Mon, 30 Sep 2024 15:57:02 +0200 Subject: [PATCH] [tools] add log-converter --- tools/log-converter/log_converter.py | 134 +++++++++++++++++++++++++++ tools/log-converter/pyproject.toml | 10 ++ tools/log-converter/uv.lock | 51 ++++++++++ 3 files changed, 195 insertions(+) create mode 100644 tools/log-converter/log_converter.py create mode 100644 tools/log-converter/pyproject.toml create mode 100644 tools/log-converter/uv.lock diff --git a/tools/log-converter/log_converter.py b/tools/log-converter/log_converter.py new file mode 100644 index 0000000..9e24f77 --- /dev/null +++ b/tools/log-converter/log_converter.py @@ -0,0 +1,134 @@ +import os +from pathlib import Path + +import click +import polars as pl + +NAS_CALIBRATE_STATE = 1 +FLYING_FMM_STATE = 10 +SECONDS_BEFORE_FLYING = 15 + + +@click.command() +@click.argument("path", type=click.Path(exists=True)) +@click.option("--output", type=click.Path(), default=".") +def main(path: Path, output: Path): + """ + Convert the logs found in the given path to a format compliant with ARPIST. + """ + # now walk in the directory pointed by path the files: main_Boardcore_EventData.csv, main_Boardcore_NASState.csv, main_Boardcore_ReferenceValues.csv and save their path to variables + nas_controller_status_path = None + nas_state_path = None + reference_values_path = None + fmm_status_path = None + + for p, _, fn in os.walk(path): + for f in fn: + if f == "main_Main_NASControllerStatus.csv": + nas_controller_status_path = os.path.join(p, f) + elif f == "main_Main_FlightModeManagerStatus.csv": + fmm_status_path = os.path.join(p, f) + elif f == "main_Boardcore_NASState.csv": + nas_state_path = os.path.join(p, f) + elif f == "main_Boardcore_ReferenceValues.csv": + reference_values_path = os.path.join(p, f) + + if not all( + [ + nas_controller_status_path, + nas_state_path, + reference_values_path, + fmm_status_path, + ] + ): + raise ValueError("Not all files were found in the given path.") + + nas_controller_status = pl.read_csv(nas_controller_status_path) + nas_state = pl.read_csv(nas_state_path) + reference_values = pl.read_csv(reference_values_path) + fmm_status = pl.read_csv(fmm_status_path) + + # sort by timestamp and extract the timestamp associated to the calibrate event and topic + nas_controller_status = nas_controller_status.sort("timestamp") + calibrate_tms = nas_controller_status.filter( + pl.col("state") == NAS_CALIBRATE_STATE + ).select("timestamp") + + # add the calibrate timestamp to the reference_values as a new column + reference_values = reference_values.with_columns(calibrate_tms.select("timestamp")) + + # select cols + reference_values = reference_values.select( + pl.from_epoch(pl.col("timestamp"), time_unit="us"), + pl.col("refLatitude").alias("latitude"), + pl.col("refLongitude").alias("longitude"), + pl.col("refAltitude").alias("altitude"), + ) + nas_state = nas_state.select( + pl.from_epoch(pl.col("timestamp"), time_unit="us"), + "n", + "e", + "d", + "vn", + "ve", + "vd", + ) + fmm_status = fmm_status.select( + pl.from_epoch(pl.col("timestamp"), time_unit="us"), "state" + ) + + # find the min and max timestamp + # min_ts = min( + # reference_values.select("timestamp").min().item(0, 0), + # (nas_state.select("timestamp").min().item(0, 0)), + # ) + max_ts = max( + reference_values.select("timestamp").max().item(0, 0), + (nas_state.select("timestamp").max().item(0, 0)), + ) + + # upsample and downsample the dataframes + last_row = reference_values.tail(1) + last_row[0, "timestamp"] = max_ts + reference_values = pl.concat([reference_values, last_row], how="vertical") + reference_values = ( + reference_values.group_by_dynamic(pl.col("timestamp"), every="500ms") + .agg(pl.all().last()) + .upsample(time_column="timestamp", every="500ms") + .fill_null(strategy="forward") + ) + nas_state = ( + nas_state.group_by_dynamic(pl.col("timestamp"), every="250ms") + .agg(pl.all().last()) + .upsample(time_column="timestamp", every="250ms") + .fill_null(strategy="forward") + ) + + # filter from 15 seconds before flying + start_ts = fmm_status.filter(pl.col("state") == FLYING_FMM_STATE).select( + "timestamp" + )[0, 0] - pl.duration(seconds=SECONDS_BEFORE_FLYING) + reference_values = reference_values.filter(pl.col("timestamp") >= start_ts) + nas_state = nas_state.filter(pl.col("timestamp") >= start_ts) + + # save the dataframes to csv + output = Path(output) + reference_values.select( + pl.col("timestamp").dt.timestamp(time_unit="us"), + "latitude", + "longitude", + "altitude", + ).write_csv(output / "low_rate.csv") + nas_state.select( + pl.col("timestamp").dt.timestamp(time_unit="us"), + "n", + "e", + "d", + "vn", + "ve", + "vd", + ).write_csv(output / "high_rate.csv") + + +if __name__ == "__main__": + main() diff --git a/tools/log-converter/pyproject.toml b/tools/log-converter/pyproject.toml new file mode 100644 index 0000000..8d4bc7c --- /dev/null +++ b/tools/log-converter/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "log-converter" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = ["click>=8.1.7", "polars>=1.8.2"] + +[project.scripts] +log_converter = "log_converter:log_converter" diff --git a/tools/log-converter/uv.lock b/tools/log-converter/uv.lock new file mode 100644 index 0000000..6a88285 --- /dev/null +++ b/tools/log-converter/uv.lock @@ -0,0 +1,51 @@ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "click" +version = "8.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "log-converter" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "click" }, + { name = "polars" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.1.7" }, + { name = "polars", specifier = ">=1.8.2" }, +] + +[[package]] +name = "polars" +version = "1.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/75/2196c26fe049ecce55a0fa87b22ab3d9477bc9bab38116ed04854fc65ecb/polars-1.8.2.tar.gz", hash = "sha256:42f69277d5be2833b0b826af5e75dcf430222d65c9633872856e176a0bed27a0", size = 4010537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/8b/6829e22a0f4c6e754c2e2b5d81025ab14d7b214018119762f52bad7325aa/polars-1.8.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:114be1ebfb051b794fb9e1f15999430c79cc0824595e237d3f45632be3e56d73", size = 31165933 }, + { url = "https://files.pythonhosted.org/packages/8f/cd/5d6b837f42c1b6d87012beca940a075e450a352ab717a649000c2ec57d71/polars-1.8.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e4fc36cfe48972d4c5be21a7cb119d6378fb7af0bb3eeb61456b66a1f43228e3", size = 27488552 }, + { url = "https://files.pythonhosted.org/packages/a7/f3/c317b1bc6759d1ec343c25d5ebd376a07a2e1fd2bd04fdc07ce6b2a855c4/polars-1.8.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c1e448d6e38697650b22dd359f13c40b567c0b66686c8602e4367400e87801", size = 32548666 }, + { url = "https://files.pythonhosted.org/packages/1d/df/5ccf44218728caecda9f555879b40fe4ab34ff629c81b9117a1107437fdc/polars-1.8.2-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:570ee86b033dc5a6dbe2cb0df48522301642f304dda3da48f53d7488899a2206", size = 29187225 }, + { url = "https://files.pythonhosted.org/packages/9c/45/77e4fda23368907c06bf70fc722de28d442c5087bbc8a60c29b8396750ea/polars-1.8.2-cp38-abi3-win_amd64.whl", hash = "sha256:ce1a1c1e2150ffcc44a5f1c461d738e1dcd95abbd0f210af0271c7ac0c9f7ef9", size = 32394690 }, +] -- GitLab