Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(Google BigQuery Node): Better error messages, transform timestamps #9255

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@ export class GoogleBigQuery extends VersionedNodeType {
group: ['input'],
subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
description: 'Consume Google BigQuery API',
defaultVersion: 2,
defaultVersion: 2.1,
};

const nodeVersions: IVersionedNodeType['nodeVersions'] = {
1: new GoogleBigQueryV1(baseDescription),
2: new GoogleBigQueryV2(baseDescription),
2.1: new GoogleBigQueryV2(baseDescription),
};

super(nodeVersions, baseDescription);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import type { IDataObject, IExecuteFunctions } from 'n8n-workflow';
import type { IDataObject, IExecuteFunctions, INode } from 'n8n-workflow';
import { constructExecutionMetaData } from 'n8n-core';
import { mock } from 'jest-mock-extended';
import { prepareOutput } from '../../../v2/helpers/utils';

describe('Google BigQuery v2 Utils', () => {
it('should prepareOutput', () => {
const thisArg = mock<IExecuteFunctions>({ helpers: mock({ constructExecutionMetaData }) });
const thisArg = mock<IExecuteFunctions>({
getNode: () => ({ typeVersion: 2.1 }) as INode,
helpers: mock({ constructExecutionMetaData }),
});
const response: IDataObject = {
kind: 'bigquery#getQueryResultsResponse',
etag: 'e_tag',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -249,10 +249,16 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
returnData.push(...executionErrorData);
continue;
}
if ((error.message as string).includes('location')) {
if ((error.message as string).includes('location') || error.httpCode === '404') {
error.description =
"Are you sure your table is in that region? You can specify the region using the 'Location' parameter from options.";
}

if (error.httpCode === '403' && error.message.includes('Drive')) {
error.description =
'If your table(s) pull from a document in Google Drive, make sure that document is shared with your user';
}

throw new NodeOperationError(this.getNode(), error as Error, {
itemIndex: i,
description: error.description,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export const versionDescription: INodeTypeDescription = {
name: 'googleBigQuery',
icon: 'file:googleBigQuery.svg',
group: ['input'],
version: 2,
version: [2, 2.1],
subtitle: '={{$parameter["operation"]}}',
description: 'Consume Google BigQuery API',
defaults: {
Expand Down
29 changes: 24 additions & 5 deletions packages/nodes-base/nodes/Google/BigQuery/v2/helpers/utils.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { DateTime } from 'luxon';
import type { IDataObject, IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { jsonParse, NodeOperationError } from 'n8n-workflow';
import type { SchemaField, TableRawData, TableSchema } from './interfaces';

function getFieldValue(schemaField: SchemaField, field: IDataObject) {
function getFieldValue(schemaField: SchemaField, field: IDataObject, parseTimestamps = false) {
if (schemaField.type === 'RECORD') {
// eslint-disable-next-line @typescript-eslint/no-use-before-define
return simplify([field.v as TableRawData], schemaField.fields as unknown as SchemaField[]);
Expand All @@ -12,6 +13,9 @@ function getFieldValue(schemaField: SchemaField, field: IDataObject) {
try {
value = jsonParse(value as string);
} catch (error) {}
} else if (schemaField.type === 'TIMESTAMP' && parseTimestamps) {
const dt = DateTime.fromSeconds(Number(value));
value = dt.isValid ? dt.toISO() : value;
}
return value;
}
Expand All @@ -26,18 +30,27 @@ export function wrapData(data: IDataObject | IDataObject[]): INodeExecutionData[
}));
}

export function simplify(data: TableRawData[], schema: SchemaField[], includeSchema = false) {
export function simplify(
data: TableRawData[],
schema: SchemaField[],
includeSchema = false,
parseTimestamps = false,
) {
const returnData: IDataObject[] = [];
for (const entry of data) {
const record: IDataObject = {};

for (const [index, field] of entry.f.entries()) {
if (schema[index].mode !== 'REPEATED') {
record[schema[index].name] = getFieldValue(schema[index], field);
record[schema[index].name] = getFieldValue(schema[index], field, parseTimestamps);
} else {
record[schema[index].name] = (field.v as unknown as IDataObject[]).flatMap(
(repeatedField) => {
return getFieldValue(schema[index], repeatedField as unknown as IDataObject);
return getFieldValue(
schema[index],
repeatedField as unknown as IDataObject,
parseTimestamps,
);
},
);
}
Expand Down Expand Up @@ -68,12 +81,18 @@ export function prepareOutput(
responseData = response;
} else {
const { rows, schema } = response;
const parseTimestamps = this.getNode().typeVersion >= 2.1;

if (rows !== undefined && schema !== undefined) {
const fields = (schema as TableSchema).fields;
responseData = rows;

responseData = simplify(responseData as TableRawData[], fields, includeSchema);
responseData = simplify(
responseData as TableRawData[],
fields,
includeSchema,
parseTimestamps,
);
} else if (schema && includeSchema) {
responseData = { success: true, _schema: schema };
} else {
Expand Down
Loading