From 63d602dd7a68b78d89b15727eb253bde2e3fd5c7 Mon Sep 17 00:00:00 2001
From: Oleg Kalachev <okalachev@gmail.com>
Date: Sun, 2 Jun 2024 01:45:05 +0300
Subject: [PATCH] Add C++ tool for conversion csv logs to ulog

---
 .github/workflows/tools.yml       | 21 +++++++++
 tools/csv_to_ulog/CMakeLists.txt  | 23 ++++++++++
 tools/csv_to_ulog/README.md       | 20 +++++++++
 tools/csv_to_ulog/csv_to_ulog.cpp | 72 +++++++++++++++++++++++++++++++
 4 files changed, 136 insertions(+)
 create mode 100644 .github/workflows/tools.yml
 create mode 100644 tools/csv_to_ulog/CMakeLists.txt
 create mode 100644 tools/csv_to_ulog/README.md
 create mode 100644 tools/csv_to_ulog/csv_to_ulog.cpp

diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml
new file mode 100644
index 0000000..01d3035
--- /dev/null
+++ b/.github/workflows/tools.yml
@@ -0,0 +1,21 @@
+name: Build tools
+
+on:
+  push:
+    branches: [ '*' ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  csv_to_ulog:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build csv_to_ulog
+      run: cd tools/csv_to_ulog && mkdir build && cd build && cmake .. && make
+    - name: Test csv_to_ulog
+      run: |
+        cd tools/csv_to_ulog/build
+        echo -e "t,x,y,z\n0,1,2,3\n1,4,5,6" > log.csv
+        ./csv_to_ulog log.csv
+        test $(stat -c %s log.ulg) -eq 196
diff --git a/tools/csv_to_ulog/CMakeLists.txt b/tools/csv_to_ulog/CMakeLists.txt
new file mode 100644
index 0000000..3e094b3
--- /dev/null
+++ b/tools/csv_to_ulog/CMakeLists.txt
@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 3.15)
+project(csv_to_ulog)
+include(FetchContent)
+set(CMAKE_CXX_STANDARD 17)
+
+FetchContent_Declare(
+  ulog_cpp
+  GIT_REPOSITORY https://github.com/PX4/ulog_cpp.git
+  GIT_TAG cf24ec6
+)
+
+FetchContent_Declare(
+  rapidcsv
+  GIT_REPOSITORY https://github.com/d99kris/rapidcsv.git
+  GIT_TAG v8.82
+)
+
+FetchContent_MakeAvailable(ulog_cpp)
+FetchContent_MakeAvailable(rapidcsv)
+
+add_executable(csv_to_ulog csv_to_ulog.cpp)
+target_link_libraries(csv_to_ulog PUBLIC ulog_cpp::ulog_cpp)
+target_include_directories(csv_to_ulog PUBLIC ${rapidcsv_SOURCE_DIR}/src)
diff --git a/tools/csv_to_ulog/README.md b/tools/csv_to_ulog/README.md
new file mode 100644
index 0000000..9c0a2d5
--- /dev/null
+++ b/tools/csv_to_ulog/README.md
@@ -0,0 +1,20 @@
+# csv_to_ulog
+
+Tool for converting CSV flight logs to ULog format so they can be analyzed using [FlightPlot](https://github.com/PX4/FlightPlot) software.
+
+To build, go to the `<flix>/tools/csv_to_ulog` directory and run:
+
+```bash
+mkdir build
+cd build
+cmake ..
+make
+```
+
+Convert a CSV file to ULog:
+
+```bash
+./csv_to_ulog log_file.csv
+```
+
+ULog file will be saved in the same directory.
diff --git a/tools/csv_to_ulog/csv_to_ulog.cpp b/tools/csv_to_ulog/csv_to_ulog.cpp
new file mode 100644
index 0000000..1affddb
--- /dev/null
+++ b/tools/csv_to_ulog/csv_to_ulog.cpp
@@ -0,0 +1,72 @@
+// Copyright (c) 2023 Oleg Kalachev <okalachev@gmail.com>
+// Repository: https://github.com/okalachev/flix
+
+// Tool for conversion CSV log file to ULog format
+
+#include <ulog_cpp/simple_writer.hpp>
+#include <vector>
+#include <string>
+#include <filesystem>
+#include "rapidcsv.h"
+
+using std::vector;
+using std::string;
+
+struct Data {
+	uint64_t timestamp;
+	float values[30];
+};
+
+int main(int argc, char** argv)
+{
+	if (argc < 2) {
+		printf("Usage: %s file.csv [file.ulg]\n", argv[0]);
+		return -1;
+	}
+
+	// check input file exists
+	if (!std::filesystem::exists(argv[1])) {
+		printf("Input file \"%s\" does not exist\n", argv[1]);
+		return -1;
+	}
+
+	// open csv file
+	rapidcsv::Document csv(argv[1]);
+	auto columns = csv.GetColumnNames();
+
+
+	// open ulog file
+	string ulog_file;
+	if (argc < 3) {
+		ulog_file = std::filesystem::path(argv[1]).replace_extension(".ulg").string();
+	} else {
+		ulog_file = argv[2];
+	}
+	ulog_cpp::SimpleWriter writer(ulog_file.c_str(), 0);
+	writer.writeInfo("sys_name", "flix");
+
+	vector<ulog_cpp::Field> fields;
+	fields.push_back(ulog_cpp::Field("uint64_t", "timestamp"));
+	columns.erase(columns.begin()); // remove timestamp column
+	for (auto& column : columns) {
+		// Valid field name for ULog: [a-z0-9_]+
+		std::replace(column.begin(), column.end(), '.', '_'); // replace dots with underscores
+		std::transform(column.begin(), column.end(), column.begin(), [](unsigned char c) { return std::tolower(c); }); // lowercase column name
+		fields.push_back(ulog_cpp::Field("float", column));
+	}
+
+	const char* msg_name = "state";
+	writer.writeMessageFormat(msg_name, fields);
+	writer.headerComplete();
+
+	const uint16_t msg_id = writer.writeAddLoggedMessage(msg_name);
+
+	for (size_t i = 0; i < csv.GetRowCount(); i++) {
+		Data data;
+		data.timestamp = csv.GetCell<float>(0, i) * 1000000.0;
+		for (size_t j = 1; j <= columns.size(); j++) {
+			data.values[j - 1] = csv.GetCell<float>(j, i);
+		}
+		writer.writeData(msg_id, data);
+	}
+}