diff --git a/api/pkgs/@duckdb/node-api/README.md b/api/pkgs/@duckdb/node-api/README.md index 907ad9f2..f9b793be 100644 --- a/api/pkgs/@duckdb/node-api/README.md +++ b/api/pkgs/@duckdb/node-api/README.md @@ -16,8 +16,6 @@ This is a high-level API meant for applications. It depends on low-level binding ### Roadmap Some features are not yet complete: -- Friendlier APIs for convering results to common JS data structures. -- Friendlier APIs for converting values of specialized and complex DuckDB types to common JS types. - Appending and binding advanced data types. (Additional DuckDB C API support needed.) - Writing to data chunk vectors. (Directly writing to binary buffers is challenging to support using the Node Addon API.) - User-defined types & functions. (Support for this was added to the DuckDB C API in v1.1.0.) @@ -94,22 +92,21 @@ const result = await prepared.run(); Get column names and types: ```ts -const columnNames = []; -const columnTypes = []; -const columnCount = result.columnCount; -for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { - const columnName = result.columnName(columnIndex); - const columnType = result.columnType(columnIndex); - columnNames.push(columnName); - columnTypes.push(columnType); -} +const columnNames = result.columnNames(); +const columnTypes = result.columnTypes(); +``` + +Fetch all chunks: +```ts +const chunks = await result.fetchAllChunks(); ``` -Fetch data chunks: +Fetch one chunk at a time: ```ts const chunks = []; while (true) { const chunk = await result.fetchChunk(); + // Last chunk will have zero rows. if (chunk.rowCount === 0) { break; } @@ -117,13 +114,23 @@ while (true) { } ``` -Read column data: +Read chunk data (column-major): +```ts +const columns = chunk.getColumns(); // array of columns, each as an array of values +``` + +Read chunk data (row-major): +```ts +const columns = chunk.getRows(); // array of rows, each as an array of values +``` + +Read chunk data (one value at a time) ```ts const columns = []; -const columnCount = result.columnCount; +const columnCount = chunk.columnCount; for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { const columnValues = []; - const columnVector = chunk.getColumn(columnIndex); + const columnVector = chunk.getColumnVector(columnIndex); const itemCount = columnVector.itemCount; for (let itemIndex = 0; itemIndex < itemCount; itemIndex++) { const value = columnVector.getItem(itemIndex); @@ -207,13 +214,15 @@ if (columnType.typeId === DuckDBTypeId.BLOB) { if (columnType.typeId === DuckDBTypeId.DATE) { const dateDays = columnValue.days; const dateString = columnValue.toString(); + const { year, month, day } = columnValue.toParts(); } if (columnType.typeId === DuckDBTypeId.DECIMAL) { const decimalWidth = columnValue.width; const decimalScale = columnValue.scale; - const decimalValue = columnValue.value; // bigint (raw fixed-point integer; `scale` indicates number of fractional digits) + const decimalValue = columnValue.value; // bigint (Scaled-up value. Represented number is value/(10^scale).) const decimalString = columnValue.toString(); + const decimalDouble = columnValue.toDouble(); } if (columnType.typeId === DuckDBTypeId.INTERVAL) { @@ -256,22 +265,26 @@ if (columnType.typeId === DuckDBTypeId.TIMESTAMP_S) { if (columnType.typeId === DuckDBTypeId.TIMESTAMP_TZ) { const timestampTZMicros = columnValue.micros; // bigint const timestampTZString = columnValue.toString(); + const { date: { year, month, day }, time: { hour, min, sec, micros } } = columnValue.toParts(); } if (columnType.typeId === DuckDBTypeId.TIMESTAMP) { const timestampMicros = columnValue.micros; // bigint const timestampString = columnValue.toString(); + const { date: { year, month, day }, time: { hour, min, sec, micros } } = columnValue.toParts(); } if (columnType.typeId === DuckDBTypeId.TIME_TZ) { const timeTZMicros = columnValue.micros; // bigint const timeTZOffset = columnValue.offset; const timeTZString = columnValue.toString(); + const { time: { hour, min, sec, micros }, offset } = columnValue.toParts(); } if (columnType.typeId === DuckDBTypeId.TIME) { const timeMicros = columnValue.micros; // bigint const timeString = columnValue.toString(); + const { hour, min, sec, micros } = columnValue.toParts(); } if (columnType.typeId === DuckDBTypeId.UNION) { @@ -333,7 +346,7 @@ for (let statementIndex = 0; statementIndex < statementCount; statementIndex++) } ``` -### Control Evaluation +### Control Evaluation of Tasks ```ts import { DuckDBPendingResultState } from '@duckdb/node-api'; diff --git a/api/src/DuckDBDataChunk.ts b/api/src/DuckDBDataChunk.ts index 31fb1b18..2e0e6c93 100644 --- a/api/src/DuckDBDataChunk.ts +++ b/api/src/DuckDBDataChunk.ts @@ -1,5 +1,6 @@ import duckdb from '@duckdb/node-bindings'; import { DuckDBVector } from './DuckDBVector'; +import { DuckDBValue } from './values'; export class DuckDBDataChunk { public readonly chunk: duckdb.DataChunk; @@ -15,13 +16,41 @@ export class DuckDBDataChunk { public get columnCount(): number { return duckdb.data_chunk_get_column_count(this.chunk); } - public getColumn(columnIndex: number): DuckDBVector { + public getColumnVector(columnIndex: number): DuckDBVector { // TODO: cache vectors? return DuckDBVector.create( duckdb.data_chunk_get_vector(this.chunk, columnIndex), this.rowCount ); } + public getColumnValues(columnIndex: number): DuckDBValue[] { + return this.getColumnVector(columnIndex).toArray(); + } + public getColumns(): DuckDBValue[][] { + const columns: DuckDBValue[][] = []; + const columnCount = this.columnCount; + for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { + columns.push(this.getColumnValues(columnIndex)); + } + return columns; + } + public getRows(): DuckDBValue[][] { + const rows: DuckDBValue[][] = []; + const vectors: DuckDBVector[] = []; + const columnCount = this.columnCount; + for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { + vectors.push(this.getColumnVector(columnIndex)); + } + const rowCount = this.rowCount; + for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) { + const row: DuckDBValue[] = []; + for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { + row.push(vectors[columnIndex].getItem(rowIndex)); + } + rows.push(row); + } + return rows; + } public get rowCount(): number { return duckdb.data_chunk_get_size(this.chunk); } diff --git a/api/src/DuckDBResult.ts b/api/src/DuckDBResult.ts index cd44621d..72a73dcf 100644 --- a/api/src/DuckDBResult.ts +++ b/api/src/DuckDBResult.ts @@ -22,6 +22,14 @@ export class DuckDBResult { public columnName(columnIndex: number): string { return duckdb.column_name(this.result, columnIndex); } + public columnNames(): string[] { + const columnNames: string[] = []; + const columnCount = this.columnCount; + for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { + columnNames.push(this.columnName(columnIndex)); + } + return columnNames; + } public columnTypeId(columnIndex: number): DuckDBTypeId { return duckdb.column_type( this.result, @@ -38,10 +46,28 @@ export class DuckDBResult { duckdb.column_logical_type(this.result, columnIndex) ).asType(); } + public columnTypes(): DuckDBType[] { + const columnTypes: DuckDBType[] = []; + const columnCount = this.columnCount; + for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { + columnTypes.push(this.columnType(columnIndex)); + } + return columnTypes; + } public get rowsChanged(): number { return duckdb.rows_changed(this.result); } public async fetchChunk(): Promise { return new DuckDBDataChunk(await duckdb.fetch_chunk(this.result)); } + public async fetchAllChunks(): Promise { + const chunks: DuckDBDataChunk[] = []; + while (true) { + const chunk = await this.fetchChunk(); + if (chunk.rowCount === 0) { + return chunks; + } + chunks.push(chunk); + } + } } diff --git a/api/test/api.test.ts b/api/test/api.test.ts index 88aeed97..599ac101 100644 --- a/api/test/api.test.ts +++ b/api/test/api.test.ts @@ -163,11 +163,11 @@ function getColumnVector TVector ): TVector { - const column = chunk.getColumn(columnIndex); - if (!isVectorType(column, vectorType)) { + const columnVector = chunk.getColumnVector(columnIndex); + if (!isVectorType(columnVector, vectorType)) { assert.fail(`expected column ${columnIndex} to be a ${vectorType}`); } - return column; + return columnVector; } function assertVectorValues( @@ -945,4 +945,16 @@ describe('api', () => { assert.deepEqual(DuckDBDecimalValue.fromDouble(3.14159, 6, 5), decimalValue(314159n, 6, 5)); assert.deepEqual(decimalValue(314159n, 6, 5).toDouble(), 3.14159); }); + test('result inspection conveniences', async () => { + await withConnection(async (connection) => { + const result = await connection.run('select i::int as a, i::int + 10 as b from range(3) t(i)'); + assert.deepEqual(result.columnNames(), ['a', 'b']); + assert.deepEqual(result.columnTypes(), [DuckDBIntegerType.instance, DuckDBIntegerType.instance]); + const chunks = await result.fetchAllChunks(); + const chunkColumns = chunks.map(chunk => chunk.getColumns()); + assert.deepEqual(chunkColumns, [[[0, 1, 2], [10, 11, 12]]]); + const chunkRows = chunks.map(chunk => chunk.getRows()); + assert.deepEqual(chunkRows, [[[0, 10], [1, 11], [2, 12]]]); + }); + }); }); diff --git a/api/test/bench/util/runSql.ts b/api/test/bench/util/runSql.ts index 73f6888f..02be4add 100644 --- a/api/test/bench/util/runSql.ts +++ b/api/test/bench/util/runSql.ts @@ -6,7 +6,7 @@ export async function runSql(connection: DuckDBConnection, sql: string): Promise let nullCount = 0; let chunk = await result.fetchChunk(); while (chunk.rowCount > 0) { - const col0 = chunk.getColumn(0); + const col0 = chunk.getColumnVector(0); for (let i = 0; i < col0.itemCount; i++) { if (col0.getItem(i) === null) { nullCount++;