diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 3057394364a..f9b5d933a5c 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -1099,6 +1099,11 @@ export class BigQuery extends Service { }; }), }; + } else if ((providedType as string).toUpperCase() === 'TIMESTAMP(12)') { + return { + type: 'TIMESTAMP', + timestampPrecision: '12', + }; } providedType = (providedType as string).toUpperCase(); @@ -2249,11 +2254,30 @@ export class BigQuery extends Service { if (res && res.jobComplete) { let rows: any = []; if (res.schema && res.rows) { - rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { - wrapIntegers: options.wrapIntegers || false, - parseJSON: options.parseJSON, - }); - delete res.rows; + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const listParams = { + 'formatOptions.timestampOutputFormat': + queryReq.formatOptions?.timestampOutputFormat, + 'formatOptions.useInt64Timestamp': + queryReq.formatOptions?.useInt64Timestamp, + }; + rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { + wrapIntegers: options.wrapIntegers || false, + parseJSON: options.parseJSON, + listParams, + }); + delete res.rows; + } catch (e) { + (callback as SimpleQueryRowsCallback)(e as Error, null, job); + return; + } } this.trace_('[runJobsQuery] job complete'); options._cachedRows = rows; @@ -2334,6 +2358,18 @@ export class BigQuery extends Service { if (options.job) { return undefined; } + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + const defaultOpts = hasAnyFormatOpts + ? {} + : { + timestampOutputFormat: 'ISO8601_STRING', + }; + const formatOptions = extend(defaultOpts, { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }); const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, labels: queryObj.labels, @@ -2342,9 +2378,7 @@ export class BigQuery extends Service { maximumBytesBilled: queryObj.maximumBytesBilled, timeoutMs: options.timeoutMs, location: queryObj.location || options.location, - formatOptions: { - useInt64Timestamp: true, - }, + formatOptions, maxResults: queryObj.maxResults || options.maxResults, query: queryObj.query, useLegacySql: false, @@ -2588,6 +2622,7 @@ function convertSchemaFieldValue( value = BigQueryRange.fromSchemaValue_( value, schemaField.rangeElementType!.type!, + options.listParams, // Required to convert TIMESTAMP values ); break; } @@ -2666,6 +2701,11 @@ export class BigQueryRange { } private static fromStringValue_(value: string): [start: string, end: string] { + /* + This method returns start and end values for RANGE typed values returned from + the server. It decodes the server RANGE value into start and end values so + they can be used to construct a BigQueryRange. + */ let cleanedValue = value; if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { cleanedValue = cleanedValue.substring(1); @@ -2684,7 +2724,19 @@ export class BigQueryRange { return [start, end]; } - static fromSchemaValue_(value: string, elementType: string): BigQueryRange { + static fromSchemaValue_( + value: string, + elementType: string, + listParams?: + | bigquery.tabledata.IListParams + | bigquery.jobs.IGetQueryResultsParams, + ): BigQueryRange { + /* + This method is only used by convertSchemaFieldValue and only when range + values are passed into convertSchemaFieldValue. It produces a value that is + delivered to the user for read calls and it needs to pass along listParams + to ensure TIMESTAMP types are converted properly. + */ const [start, end] = BigQueryRange.fromStringValue_(value); const convertRangeSchemaValue = (value: string) => { if (value === 'UNBOUNDED' || value === 'NULL') { @@ -2692,6 +2744,7 @@ export class BigQueryRange { } return convertSchemaFieldValue({type: elementType}, value, { wrapIntegers: false, + listParams, }); }; return BigQuery.range( diff --git a/handwritten/bigquery/src/job.ts b/handwritten/bigquery/src/job.ts index d39f950b7b7..ddf7497e1cb 100644 --- a/handwritten/bigquery/src/job.ts +++ b/handwritten/bigquery/src/job.ts @@ -595,10 +595,21 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { - wrapIntegers, - parseJSON, - }); + try { + /* + Without this try/catch block, calls to /query endpoint will hang + indefinitely if a call to mergeSchemaWithRows_ fails because the + error never makes it to the callback. Instead, pass the error to the + callback the user provides so that the user can see the error. + */ + rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { + wrapIntegers, + parseJSON, + }); + } catch (e) { + callback!(e as Error, null, null, resp); + return; + } } let nextQuery: QueryResultsOptions | null = null; diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index 5bbb6f4483a..96c3a33a53a 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1472,9 +1472,14 @@ describe('BigQuery', () => { ], }, (err, rows) => { - assert.ifError(err); - assert.strictEqual(rows!.length, 1); - done(); + try { + // Without this try block the test runner silently fails + assert.ifError(err); + assert.strictEqual(rows!.length, 1); + done(); + } catch (e) { + done(e); + } }, ); }); @@ -1498,6 +1503,159 @@ describe('BigQuery', () => { }, ); }); + describe.only('High Precision Query System Tests', () => { + let bigquery: BigQuery; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = + '2023-01-01T12:00:00.123456789123Z'; + const expectedErrorMessage = + 'Cannot specify both timestamp_as_int and timestamp_output_format.'; + + before(() => { + bigquery = new BigQuery(); + }); + + const testCases = [ + { + name: 'TOF: FLOAT64, UI64: true (error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: true, + expectedTsValue: undefined, + expectedError: expectedErrorMessage, + }, + { + name: 'TOF: omitted, UI64: omitted (default INT64)', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + { + name: 'TOF: omitted, UI64: true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, async () => { + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. Queries using TIMESTAMP_ADD are another example. + */ + const query = { + query: 'SELECT ? as ts', + params: [ + bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), + ], + types: ['TIMESTAMP(12)'], + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].ts.value !== undefined); + assert.strictEqual( + rows[0].ts.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + it(`should handle nested ${testCase.name}`, async () => { + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. + */ + const query = { + query: 'SELECT ? obj', + params: [ + { + nested: { + a: bigquery.timestamp( + '2023-01-01T12:00:00.123456789123Z', + ), + }, + }, + ], + types: [ + { + nested: { + a: 'TIMESTAMP(12)', + }, + }, + ], + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].obj.nested.a.value !== undefined); + assert.strictEqual( + rows[0].obj.nested.a.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + }); + }); }); describe('named', () => { diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 96ede116075..0fe388e1e3b 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -34,8 +34,8 @@ describe('Timestamp Output Format System Tests', () => { const dataset = bigquery.dataset(datasetId); const table = dataset.table(tableId); const insertedTsValue = '2023-01-01T12:00:00.123456789123Z'; - const expectedTsValueMicroseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456789123Z'; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; before(async () => { await dataset.create(); @@ -59,13 +59,13 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=true', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=false', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with FLOAT64 and useInt64Timestamp=true (expect error)', @@ -78,19 +78,19 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with FLOAT64 and useInt64Timestamp=false', timestampOutputFormat: 'FLOAT64', useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp=true', timestampOutputFormat: 'INT64', useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp=false', timestampOutputFormat: 'INT64', useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=true (expect error)', @@ -103,50 +103,50 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=false', timestampOutputFormat: 'ISO8601_STRING', useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValuePicoseconds, }, // Additional test cases for undefined combinations { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp undefined', timestampOutputFormat: undefined, useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValuePicoseconds, }, { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=true', timestampOutputFormat: undefined, useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=false', timestampOutputFormat: undefined, useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp undefined', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with FLOAT64 and useInt64Timestamp undefined (expect error)', timestampOutputFormat: 'FLOAT64', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp undefined', timestampOutputFormat: 'INT64', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp undefined (expect error)', timestampOutputFormat: 'ISO8601_STRING', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValuePicoseconds, }, ]; diff --git a/handwritten/bigquery/test/bigquery.ts b/handwritten/bigquery/test/bigquery.ts index b53a2b8942b..d814b0e35f0 100644 --- a/handwritten/bigquery/test/bigquery.ts +++ b/handwritten/bigquery/test/bigquery.ts @@ -3397,12 +3397,141 @@ describe('BigQuery', () => { }, jobCreationMode: 'JOB_CREATION_REQUIRED', formatOptions: { - useInt64Timestamp: true, + timestampOutputFormat: 'ISO8601_STRING', }, }; assert.deepStrictEqual(req, expectedReq); }); + describe('format options', () => { + const testCases = [ + { + name: 'TOF: TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED, UI64: true', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: true, + }, + { + name: 'TOF: TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED, UI64: false (default ISO8601_STRING)', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: false, + }, + { + name: 'TOF: FLOAT64, UI64: false', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: false, + }, + { + name: 'TOF: INT64, UI64: true', + timestampOutputFormat: 'INT64', + useInt64Timestamp: true, + }, + { + name: 'TOF: INT64, UI64: false (error)', + timestampOutputFormat: 'INT64', + useInt64Timestamp: false, + }, + { + name: 'TOF: ISO8601_STRING, UI64: true (error)', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: true, + }, + { + name: 'TOF: ISO8601_STRING, UI64: false', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: false, + }, + { + name: 'TOF: omitted, UI64: omitted (default INT64)', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + }, + { + name: 'TOF: omitted, UI64: true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + }, + { + name: 'TOF: omitted, UI64: false (default ISO8601_STRING)', + timestampOutputFormat: undefined, + useInt64Timestamp: false, + }, + { + name: 'TOF: TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED, UI64: omitted (default INT64)', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: undefined, + }, + { + name: 'TOF: FLOAT64, UI64: omitted (error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: undefined, + }, + { + name: 'TOF: INT64, UI64: omitted', + timestampOutputFormat: 'INT64', + useInt64Timestamp: undefined, + }, + { + name: 'TOF: ISO8601_STRING, UI64: omitted (error)', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: undefined, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, () => { + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + const req = bq.buildQueryRequest_(QUERY_STRING, options); + for (const key in req) { + if (req[key] === undefined) { + delete req[key]; + } + } + if (req.formatOptions) { + for (const key in req.formatOptions) { + if (req.formatOptions[key] === undefined) { + delete req.formatOptions[key]; + } + } + } + + const expectedFormatOptions: any = {}; + if ( + testCase.timestampOutputFormat === undefined && + testCase.useInt64Timestamp === undefined + ) { + expectedFormatOptions.timestampOutputFormat = 'ISO8601_STRING'; + } else { + if (testCase.timestampOutputFormat !== undefined) { + expectedFormatOptions.timestampOutputFormat = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + expectedFormatOptions.useInt64Timestamp = + testCase.useInt64Timestamp; + } + } + + const expectedReq = { + query: QUERY_STRING, + useLegacySql: false, + requestId: req.requestId, + jobCreationMode: 'JOB_CREATION_OPTIONAL', + formatOptions: expectedFormatOptions, + }; + assert.deepStrictEqual(req, expectedReq); + }); + }); + }); + it('should create a QueryRequest from a SQL string', () => { const req = bq.buildQueryRequest_(QUERY_STRING, {}); for (const key in req) { @@ -3416,7 +3545,7 @@ describe('BigQuery', () => { requestId: req.requestId, jobCreationMode: 'JOB_CREATION_OPTIONAL', formatOptions: { - useInt64Timestamp: true, + timestampOutputFormat: 'ISO8601_STRING', }, }; assert.deepStrictEqual(req, expectedReq);