Skip to content

Commit 7631772

Browse files
BvngeeBvngee
committed
various fixes; can load any recording now!!
1 parent caa4429 commit 7631772

File tree

8 files changed

+72
-99
lines changed

8 files changed

+72
-99
lines changed

‎backend/src/http_server.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,8 @@ def check_subdir(d):
1717
elif f.endswith(".pq") or f.endswith(".parquet"):
1818
parquet_files.add(os.path.relpath(path, data_dir))
1919

20-
2120
check_subdir(data_dir)
2221

23-
2422
routes = web.RouteTableDef()
2523

2624

‎frontend/app/components/Navbar.tsx

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { availableRecordings, getDBCForRecording } from "../data-processing/http
2222
import { useDataMethods } from "../data-processing/DataMethodsProvider";
2323
import AutocompleteSearchbar from "./Autocomplete";
2424
import { File } from "@phosphor-icons/react/dist/ssr";
25+
import { log } from "console";
2526

2627
//takes in the list of files as an array and outputs it in mantine tree format
2728
function createFileTree(paths: string[] | undefined) {
@@ -62,7 +63,7 @@ function createFileTree(paths: string[] | undefined) {
6263
currentLevel = existingNode.children;
6364
}
6465
}
65-
return tree.children[0];
66+
return tree.children;
6667
}
6768

6869
export default function Navbar() {
@@ -74,7 +75,7 @@ export default function Navbar() {
7475
const [isProduction, setIsProduction] = useState<boolean>(true);
7576
const displayedName = fileName.split("/")[fileName.split("/").length - 1];
7677

77-
const { switchToRecording, switchToLiveData, subscribeNumRows } = useDataMethods();
78+
const { switchToRecording, switchToLiveData, subscribeNumRows, reset } = useDataMethods();
7879
const myRowsPRef = useRef<HTMLParagraphElement | null>(null);
7980
useEffect(() => {
8081
return subscribeNumRows((numRows: number) => {
@@ -125,22 +126,20 @@ export default function Navbar() {
125126
function DataSourcePicker() {
126127
function onPickerChanged(value: string) {
127128
if (value === "recording") {
128-
//TODO Make an actual function to stop the live view
129-
switchToRecording("");
129+
reset();
130130
setLive(false);
131131
} else {
132132
setLive(true);
133133
switchToLiveData();
134134
}
135-
// setLive(value === "live");
136135
}
137136

138137
function FileTree() {
139138
if (recordings == null) {
140139
return <span>Loading...</span>;
141140
}
142141

143-
const data = [createFileTree(recordings)];
142+
const data = createFileTree(recordings);
144143

145144
return (
146145
<>
@@ -164,10 +163,10 @@ export default function Navbar() {
164163

165164
<span
166165
onClick={async () => {
167-
if (tree.hoveredNode?.includes("/")) {
166+
if (!hasChildren && tree.hoveredNode) {
168167
setFileName(tree.hoveredNode);
169168
setLoading(true);
170-
switchToRecording(tree.hoveredNode);
169+
switchToRecording(tree.hoveredNode, isProduction);
171170
setLoading(false);
172171
close();
173172

@@ -233,7 +232,7 @@ export default function Navbar() {
233232
const file = files[0];
234233
setFileName(file.name);
235234
setLoading(true);
236-
switchToRecording(file);
235+
switchToRecording(file, isProduction);
237236
setLoading(false);
238237
close();
239238
}}

‎frontend/app/components/TimelineBar.tsx

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { Box, Button, RangeSlider, RangeSliderValue } from "@mantine/core";
2-
import { ArrowFatLinesRight, CaretDoubleRight, Pause, Play } from "@phosphor-icons/react";
1+
import { Box, RangeSlider, RangeSliderValue } from "@mantine/core";
2+
import { CaretDoubleRight, Pause, Play } from "@phosphor-icons/react";
33
import { useEffect, useMemo, useRef, useState } from "react";
44
import { useDataMethods } from "../data-processing/DataMethodsProvider";
55
import DataSourceType from "@/models/DataSourceType";
@@ -8,7 +8,6 @@ import { timeColumnName } from "../data-processing/datatypes";
88

99
export default function TimelineBar() {
1010
const [paused, setPaused] = useState(true);
11-
const { switchToRecording } = useDataMethods();
1211

1312
return (
1413
<>
@@ -25,11 +24,6 @@ export default function TimelineBar() {
2524
<MainSlider />
2625
</div>
2726
<SyncButton />
28-
<Button
29-
onClick={() =>
30-
switchToRecording("fs-data/FS-2/2025-03-06-BrakingTests1.parquet")
31-
}
32-
/>
3327
</div>
3428
</>
3529
);

‎frontend/app/components/visualizations/GPS/GPSInternal.tsx

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -183,11 +183,14 @@ export default function GPSInternal({
183183
});
184184
});
185185
const unsub2 = subscribeViewInterval(([left, right]) => {
186-
console.log("1", performance.now());
186+
// console.log("1", performance.now());
187+
187188
// We reset and regenerate the fillStyle lookupValues each time
188189
// viewInterval changes. This seems to be a fairly fast operation
189190
visibleSeries.fill({ lookupValue: 0 });
190-
console.log("2", performance.now());
191+
192+
// console.log("2", performance.now());
193+
191194
// We need +1 as left and right from dataProvider are both inclusive
192195
const length = right - left + 1;
193196
// Generates a list of values of length `length` that are evenly
@@ -196,14 +199,17 @@ export default function GPSInternal({
196199
{ length },
197200
(_, i) => 0.3 + (i / (length - 1)) * (1 - 0.3),
198201
);
199-
console.log("3", performance.now());
202+
203+
// console.log("3", performance.now());
204+
200205
// Insert the above gradient array into the lookupValues dataset
201206
// channel starting at the correct index
202207
visibleSeries.alterSamples(left, {
203208
// todo: this doesn't work with data cleaning
204209
lookupValues: gradient,
205210
});
206-
console.log("4", performance.now());
211+
212+
// console.log("4", performance.now());
207213
});
208214
const unsub3 = subscribeCursorRow((cursorRow) => {
209215
// Always update carSeries to point to where the cursor is hovering,
@@ -230,6 +236,7 @@ export default function GPSInternal({
230236
const unsub4 = subscribeReset(() => {
231237
bgSeries.clear();
232238
visibleSeries.clear();
239+
carSeries.clear();
233240
});
234241

235242
return () => {

‎frontend/app/components/visualizations/LapCounter.tsx

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import { useContext, useEffect, useId, useRef } from "react";
44
import { LightningChartsContext } from "./lightning-charts/GlobalContext";
55
import globalTheme from "./lightning-charts/GlobalTheme";
6-
import { VisualizationProps } from "../FlexLayoutComponent";
76
import { useDataMethods } from "@/app/data-processing/DataMethodsProvider";
87

98
// data is represented in row, column (y, x)
@@ -34,7 +33,6 @@ export function LapCounter() {
3433

3534
if (!lapArray || !lapTimeArray) return;
3635

37-
console.log("hi");
3836
lapTimes = [];
3937

4038
let previous = lapArray[0];
@@ -72,8 +70,6 @@ export function LapCounter() {
7270
dataGrid.setRowContent(i + 1, [i + 1, lapTimes[i]]);
7371
}
7472

75-
console.log(lapTime);
76-
7773
dataGrid.setRowContent(dataGrid.getRowMax(), [dataGrid.getRowMax(), lapTime]);
7874
});
7975

‎frontend/app/data-processing/DataMethodsProvider.tsx

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ type DataControllers = {
6767
reset: () => void;
6868

6969
switchToLiveData: (setIsConnected?: Dispatch<SetStateAction<boolean>>) => void;
70-
switchToRecording: (filename: string | File) => void;
70+
switchToRecording: (filename: string | File, isProduction: boolean) => void;
7171
};
7272
type DataRefs = {
7373
dataArraysRef: RefObject<DataArraysTyped>;
@@ -206,6 +206,7 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
206206
closeWebSocketConnection();
207207

208208
subscriptionsReset.current.forEach((s) => s());
209+
subscriptionsDataSource.current.forEach((s) => s(DataSourceType.NONE));
209210
};
210211

211212
const switchToLiveData = (setIsConnected?: Dispatch<SetStateAction<boolean>>) => {
@@ -288,29 +289,40 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
288289
subscriptionsDataSource.current.forEach((s) => s(DataSourceType.LIVE));
289290
setIsTimelineSynced(true, "dataProvider");
290291
};
291-
const switchToRecording = async (file: string | File) => {
292+
const switchToRecording = async (file: string | File, isProduction: boolean) => {
292293
reset();
293-
const table = await getRecording(file);
294-
if (!table) return;
294+
const table = await getRecording(file, isProduction);
295+
if (!table) {
296+
console.log("WARN: couldn't load recording!");
297+
return;
298+
};
299+
295300
let arraysTyped = {} as DataArraysTyped;
296301
// TODO: Make this actually use the schema of the incoming
297-
// websocket stream!!! This assumes all columns are present
302+
// parquet recording!!! This assumes all columns are present
298303
// (non-present should be null)
299304
for (const key of columnNames) {
300305
const vector = table.getChild(key);
301306
if (vector) {
302-
const arr = vector.toArray();
307+
let arr = vector.toArray();
308+
309+
// Hack since lcjs/chartjs don't support 64bit ints
310+
if (arr instanceof BigInt64Array || arr instanceof BigUint64Array) {
311+
arr = convertToInt32Array(arr);
312+
}
313+
314+
// Store extracted TypedArray in final dictionary
303315
arraysTyped[key] = arr;
304316
}
305317
}
306318
// some basic manual data processing. TODO: make somthing extremely
307319
// generic / extensible for arbitrary data processing
308-
arraysTyped.VDM_GPS_Latitude = arraysTyped.VDM_GPS_Latitude!.map((n) =>
320+
arraysTyped.VDM_GPS_Latitude = arraysTyped.VDM_GPS_Latitude?.map((n) =>
309321
n == 0.0 ? NaN : n,
310-
);
311-
arraysTyped.VDM_GPS_Longitude = arraysTyped.VDM_GPS_Longitude!.map((n) =>
322+
) ?? null;
323+
arraysTyped.VDM_GPS_Longitude = arraysTyped.VDM_GPS_Longitude?.map((n) =>
312324
n == 0.0 ? NaN : n,
313-
);
325+
) ?? null;
314326

315327
numRowsRef.current = table.numRows;
316328
subscriptionsNumRows.current.forEach((s) => s(numRowsRef.current));
@@ -362,11 +374,6 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
362374
};
363375
}, []);
364376

365-
// const methods: DataMethods = useMemo(
366-
// () => (),
367-
// [],
368-
// );
369-
370377
return (
371378
<DataMethodsContext.Provider value={methods}>{children}</DataMethodsContext.Provider>
372379
);
@@ -379,3 +386,17 @@ export function useDataMethods() {
379386
}
380387
return context;
381388
}
389+
390+
391+
// Used for recordings that have some i64 columns (Thanks claude)
392+
function convertToInt32Array(bigIntArray: BigInt64Array | BigUint64Array) {
393+
const int32Array = new Int32Array(bigIntArray.length);
394+
const INT32_MIN = -2147483648;
395+
const INT32_MAX = 2147483647;
396+
397+
for (let i = 0; i < bigIntArray.length; i++) {
398+
const num = Number(bigIntArray[i]);
399+
int32Array[i] = Math.max(INT32_MIN, Math.min(INT32_MAX, num));
400+
}
401+
return int32Array;
402+
}

‎frontend/app/data-processing/http.ts

Lines changed: 12 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ import wasmInit, { readParquet, wasmMemory } from "parquet-wasm/esm";
33
// Enables zero-copy transer of Arrow data from wasm memory to JS
44
import * as arrowJSFFI from "arrow-js-ffi";
55

6-
import { DataRow, schema } from "./datatypes";
7-
6+
// Tries to call one of the HTTP endpoints of the server, either local or remote
7+
// depending on isProduction. Returns the Response, with some error handling.
8+
// Internal; should be used by the public-facing functions below
89
async function tryFetch(path: string, isProduction: boolean) {
910
const url = isProduction ? "https://telemetry.formulaslug.com" : "http://localhost:9000";
1011
try {
@@ -19,6 +20,8 @@ async function tryFetch(path: string, isProduction: boolean) {
1920
}
2021
}
2122

23+
// Fetch the DBC data associated with a given Recording (filepath). Contains a
24+
// map of ColumnName to information like units, range, etc
2225
export async function getDBCForRecording(
2326
filepath: string,
2427
isProduction: boolean,
@@ -28,16 +31,19 @@ export async function getDBCForRecording(
2831
);
2932
}
3033

31-
// ???
32-
export async function getRecording(file: string | File) {
34+
// Fetch the recording data of a filepath (by querying the server) or from a
35+
// directly uploaded file (drag-n-drop). Returned data is in Arrow Table format.
36+
export async function getRecording(file: string | File, isProduction: boolean) {
3337
// Initializes WebAssembly memory for parquet-wasm, and gets a reference to it
3438
await wasmInit();
3539
const WASM_MEMORY = wasmMemory();
3640

3741
let arrayBuffer: ArrayBuffer | undefined;
3842

43+
// If `file` is a string, fetch it from the server. If it's a File, extract
44+
// the arraybuffer data from that directly
3945
if (typeof file == "string") { // string (pathname)
40-
const resp = await tryFetch(`/api/get-recording/${file}`, false)
46+
const resp = await tryFetch(`/api/get-recording/${file}`, isProduction)
4147
arrayBuffer = await resp?.arrayBuffer();
4248
} else { // File object
4349
arrayBuffer = await file.arrayBuffer();
@@ -49,7 +55,7 @@ export async function getRecording(file: string | File) {
4955
}
5056

5157
const arrowTableWasm = readParquet(new Uint8Array(arrayBuffer)).intoFFI();
52-
const arrowTable = arrowJSFFI.parseTable<DataRow>(
58+
const arrowTable = arrowJSFFI.parseTable(
5359
WASM_MEMORY.buffer,
5460
arrowTableWasm.arrayAddrs(),
5561
arrowTableWasm.schemaAddr(),
@@ -72,53 +78,3 @@ export async function availableRecordings(isProduction: boolean) {
7278
const resp = await tryFetch(`/api/available-recordings`, isProduction);
7379
return resp?.json();
7480
}
75-
76-
// export async function initRecordingSource(
77-
// filepath: string,
78-
// data: RefObject<DataArrays>,
79-
// dataTrimmed: RefObject<DataArrays>,
80-
// setNumRows: Dispatch<SetStateAction<number>>,
81-
// viewLength: number,
82-
// ) {
83-
// const arrowTable = (await getRecording(filepath))!;
84-
//
85-
// // Completely reset data, so keys unused by the recording are left as null
86-
// data.current = nullDataArrays();
87-
// dataTrimmed.current = nullDataArrays();
88-
//
89-
// for (const [i, key] of arrowTable.schema.names.entries()) {
90-
// // Get Arrow vector from table
91-
// const vec = arrowTable.getChildAt(i);
92-
// // Extract JS array from Arrow vector
93-
// let dataArr = vec?.toArray();
94-
// let dataArrTrimmed = vec
95-
// ?.slice(Math.max(0, arrowTable.numRows - viewLength - 1))
96-
// .toArray();
97-
//
98-
// // Chartjs doesn't support bigints ootb. The easiest solution for now is
99-
// // just to replace them with supported TypedArrays
100-
// if (dataArr instanceof BigInt64Array || dataArr instanceof BigUint64Array) {
101-
// dataArr = convertToInt32Array(dataArr);
102-
// dataArrTrimmed = convertToInt32Array(dataArrTrimmed);
103-
// }
104-
//
105-
// // Set the data arrays directly
106-
// data.current[key] = dataArr;
107-
// dataTrimmed.current[key] = dataArrTrimmed;
108-
// }
109-
//
110-
// setNumRows(arrowTable.numRows);
111-
// }
112-
113-
// Thanks claude
114-
function convertToInt32Array(bigIntArray: BigInt64Array | BigUint64Array) {
115-
const int32Array = new Int32Array(bigIntArray.length);
116-
const INT32_MIN = -2147483648;
117-
const INT32_MAX = 2147483647;
118-
119-
for (let i = 0; i < bigIntArray.length; i++) {
120-
const num = Number(bigIntArray[i]);
121-
int32Array[i] = Math.max(INT32_MIN, Math.min(INT32_MAX, num));
122-
}
123-
return int32Array;
124-
}

‎frontend/app/page.tsx

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@ import TimelineBar from "./components/TimelineBar";
77
import { useEffect } from "react";
88

99
export default function Page() {
10-
useEffect(() => window.addEventListener(`contextmenu`, (e) => e.preventDefault()), []);
10+
// This is pretty stupid, but it prevents annoying right-click context menus
11+
// on lcjs charts (by disabling right-click menus altogether)
12+
useEffect(() => window.addEventListener("contextmenu", (e) => e.preventDefault()), []);
1113
return (
1214
<div className="w-[100vw] h-[100vh] flex flex-col">
1315
<Navbar />

0 commit comments

Comments
 (0)