From cd66adaa800db7f78245e46997e78b13fd909e91 Mon Sep 17 00:00:00 2001
From: Federico Lolli <federico.lolli@skywarder.eu>
Date: Mon, 30 Sep 2024 15:57:02 +0200
Subject: [PATCH] [tools] add log-converter

---
 tools/log-converter/log_converter.py | 134 +++++++++++++++++++++++++++
 tools/log-converter/pyproject.toml   |  10 ++
 tools/log-converter/uv.lock          |  51 ++++++++++
 3 files changed, 195 insertions(+)
 create mode 100644 tools/log-converter/log_converter.py
 create mode 100644 tools/log-converter/pyproject.toml
 create mode 100644 tools/log-converter/uv.lock

diff --git a/tools/log-converter/log_converter.py b/tools/log-converter/log_converter.py
new file mode 100644
index 0000000..9e24f77
--- /dev/null
+++ b/tools/log-converter/log_converter.py
@@ -0,0 +1,134 @@
+import os
+from pathlib import Path
+
+import click
+import polars as pl
+
+NAS_CALIBRATE_STATE = 1
+FLYING_FMM_STATE = 10
+SECONDS_BEFORE_FLYING = 15
+
+
+@click.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.option("--output", type=click.Path(), default=".")
+def main(path: Path, output: Path):
+    """
+    Convert the logs found in the given path to a format compliant with ARPIST.
+    """
+    # now walk in the directory pointed by path the files: main_Boardcore_EventData.csv, main_Boardcore_NASState.csv, main_Boardcore_ReferenceValues.csv and save their path to variables
+    nas_controller_status_path = None
+    nas_state_path = None
+    reference_values_path = None
+    fmm_status_path = None
+
+    for p, _, fn in os.walk(path):
+        for f in fn:
+            if f == "main_Main_NASControllerStatus.csv":
+                nas_controller_status_path = os.path.join(p, f)
+            elif f == "main_Main_FlightModeManagerStatus.csv":
+                fmm_status_path = os.path.join(p, f)
+            elif f == "main_Boardcore_NASState.csv":
+                nas_state_path = os.path.join(p, f)
+            elif f == "main_Boardcore_ReferenceValues.csv":
+                reference_values_path = os.path.join(p, f)
+
+    if not all(
+        [
+            nas_controller_status_path,
+            nas_state_path,
+            reference_values_path,
+            fmm_status_path,
+        ]
+    ):
+        raise ValueError("Not all files were found in the given path.")
+
+    nas_controller_status = pl.read_csv(nas_controller_status_path)
+    nas_state = pl.read_csv(nas_state_path)
+    reference_values = pl.read_csv(reference_values_path)
+    fmm_status = pl.read_csv(fmm_status_path)
+
+    # sort by timestamp and extract the timestamp associated to the calibrate event and topic
+    nas_controller_status = nas_controller_status.sort("timestamp")
+    calibrate_tms = nas_controller_status.filter(
+        pl.col("state") == NAS_CALIBRATE_STATE
+    ).select("timestamp")
+
+    # add the calibrate timestamp to the reference_values as a new column
+    reference_values = reference_values.with_columns(calibrate_tms.select("timestamp"))
+
+    # select cols
+    reference_values = reference_values.select(
+        pl.from_epoch(pl.col("timestamp"), time_unit="us"),
+        pl.col("refLatitude").alias("latitude"),
+        pl.col("refLongitude").alias("longitude"),
+        pl.col("refAltitude").alias("altitude"),
+    )
+    nas_state = nas_state.select(
+        pl.from_epoch(pl.col("timestamp"), time_unit="us"),
+        "n",
+        "e",
+        "d",
+        "vn",
+        "ve",
+        "vd",
+    )
+    fmm_status = fmm_status.select(
+        pl.from_epoch(pl.col("timestamp"), time_unit="us"), "state"
+    )
+
+    # find the min and max timestamp
+    # min_ts = min(
+    #     reference_values.select("timestamp").min().item(0, 0),
+    #     (nas_state.select("timestamp").min().item(0, 0)),
+    # )
+    max_ts = max(
+        reference_values.select("timestamp").max().item(0, 0),
+        (nas_state.select("timestamp").max().item(0, 0)),
+    )
+
+    # upsample and downsample the dataframes
+    last_row = reference_values.tail(1)
+    last_row[0, "timestamp"] = max_ts
+    reference_values = pl.concat([reference_values, last_row], how="vertical")
+    reference_values = (
+        reference_values.group_by_dynamic(pl.col("timestamp"), every="500ms")
+        .agg(pl.all().last())
+        .upsample(time_column="timestamp", every="500ms")
+        .fill_null(strategy="forward")
+    )
+    nas_state = (
+        nas_state.group_by_dynamic(pl.col("timestamp"), every="250ms")
+        .agg(pl.all().last())
+        .upsample(time_column="timestamp", every="250ms")
+        .fill_null(strategy="forward")
+    )
+
+    # filter from 15 seconds before flying
+    start_ts = fmm_status.filter(pl.col("state") == FLYING_FMM_STATE).select(
+        "timestamp"
+    )[0, 0] - pl.duration(seconds=SECONDS_BEFORE_FLYING)
+    reference_values = reference_values.filter(pl.col("timestamp") >= start_ts)
+    nas_state = nas_state.filter(pl.col("timestamp") >= start_ts)
+
+    # save the dataframes to csv
+    output = Path(output)
+    reference_values.select(
+        pl.col("timestamp").dt.timestamp(time_unit="us"),
+        "latitude",
+        "longitude",
+        "altitude",
+    ).write_csv(output / "low_rate.csv")
+    nas_state.select(
+        pl.col("timestamp").dt.timestamp(time_unit="us"),
+        "n",
+        "e",
+        "d",
+        "vn",
+        "ve",
+        "vd",
+    ).write_csv(output / "high_rate.csv")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/log-converter/pyproject.toml b/tools/log-converter/pyproject.toml
new file mode 100644
index 0000000..8d4bc7c
--- /dev/null
+++ b/tools/log-converter/pyproject.toml
@@ -0,0 +1,10 @@
+[project]
+name = "log-converter"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = ["click>=8.1.7", "polars>=1.8.2"]
+
+[project.scripts]
+log_converter = "log_converter:log_converter"
diff --git a/tools/log-converter/uv.lock b/tools/log-converter/uv.lock
new file mode 100644
index 0000000..6a88285
--- /dev/null
+++ b/tools/log-converter/uv.lock
@@ -0,0 +1,51 @@
+version = 1
+requires-python = ">=3.12"
+
+[[package]]
+name = "click"
+version = "8.1.7"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "platform_system == 'Windows'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 },
+]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 },
+]
+
+[[package]]
+name = "log-converter"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "click" },
+    { name = "polars" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "click", specifier = ">=8.1.7" },
+    { name = "polars", specifier = ">=1.8.2" },
+]
+
+[[package]]
+name = "polars"
+version = "1.8.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3d/75/2196c26fe049ecce55a0fa87b22ab3d9477bc9bab38116ed04854fc65ecb/polars-1.8.2.tar.gz", hash = "sha256:42f69277d5be2833b0b826af5e75dcf430222d65c9633872856e176a0bed27a0", size = 4010537 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b5/8b/6829e22a0f4c6e754c2e2b5d81025ab14d7b214018119762f52bad7325aa/polars-1.8.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:114be1ebfb051b794fb9e1f15999430c79cc0824595e237d3f45632be3e56d73", size = 31165933 },
+    { url = "https://files.pythonhosted.org/packages/8f/cd/5d6b837f42c1b6d87012beca940a075e450a352ab717a649000c2ec57d71/polars-1.8.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e4fc36cfe48972d4c5be21a7cb119d6378fb7af0bb3eeb61456b66a1f43228e3", size = 27488552 },
+    { url = "https://files.pythonhosted.org/packages/a7/f3/c317b1bc6759d1ec343c25d5ebd376a07a2e1fd2bd04fdc07ce6b2a855c4/polars-1.8.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c1e448d6e38697650b22dd359f13c40b567c0b66686c8602e4367400e87801", size = 32548666 },
+    { url = "https://files.pythonhosted.org/packages/1d/df/5ccf44218728caecda9f555879b40fe4ab34ff629c81b9117a1107437fdc/polars-1.8.2-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:570ee86b033dc5a6dbe2cb0df48522301642f304dda3da48f53d7488899a2206", size = 29187225 },
+    { url = "https://files.pythonhosted.org/packages/9c/45/77e4fda23368907c06bf70fc722de28d442c5087bbc8a60c29b8396750ea/polars-1.8.2-cp38-abi3-win_amd64.whl", hash = "sha256:ce1a1c1e2150ffcc44a5f1c461d738e1dcd95abbd0f210af0271c7ac0c9f7ef9", size = 32394690 },
+]
-- 
GitLab