diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index d52d07343b5..23f1ec680bd 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -1101,10 +1101,12 @@ export class BigQuery extends Service { }), }; } else if ((providedType as string).toUpperCase() === 'TIMESTAMP(12)') { - return { - type: 'TIMESTAMP', - timestampPrecision: '12', - }; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + return { + type: 'TIMESTAMP', + timestampPrecision: '12', + }; + } } providedType = (providedType as string).toUpperCase(); @@ -2364,18 +2366,31 @@ export class BigQuery extends Service { if (options.job) { return undefined; } - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts - ? {} - : { - timestampOutputFormat: 'ISO8601_STRING', - }; - const formatOptions = extend(defaultOpts, { - timestampOutputFormat: options['formatOptions.timestampOutputFormat'], - useInt64Timestamp: options['formatOptions.useInt64Timestamp'], - }); + let formatOptions; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + const defaultOpts = hasAnyFormatOpts + ? {} + : { + timestampOutputFormat: 'ISO8601_STRING', + }; + formatOptions = extend(defaultOpts, { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }); + } else { + formatOptions = extend( + { + useInt64Timestamp: true, + }, + { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }, + ); + } const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, labels: queryObj.labels, @@ -2576,39 +2591,48 @@ function convertSchemaFieldValue( break; } case 'TIMESTAMP': { - /* - At this point, 'value' will equal the timestamp value returned from the - server. We need to parse this value differently depending on its format. - For example, value could be any of the following: - 1672574400123456 - 1672574400.123456 - 2023-01-01T12:00:00.123456789123Z - */ - const listParams = options.listParams; - const timestampOutputFormat = listParams - ? listParams['formatOptions.timestampOutputFormat'] - : undefined; - const useInt64Timestamp = listParams - ? listParams['formatOptions.useInt64Timestamp'] - : undefined; - if (timestampOutputFormat === 'ISO8601_STRING') { - // value is ISO string, create BigQueryTimestamp wrapping the string - value = BigQuery.timestamp(value); - } else if ( - useInt64Timestamp !== true && - timestampOutputFormat !== 'INT64' && - (useInt64Timestamp !== undefined || timestampOutputFormat !== undefined) - ) { - // NOTE: The additional - // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) - // check is to ensure that calls to the /query endpoint remain - // unaffected as they will not be providing any listParams. - // - // If the program reaches this point in time then - // value is float seconds so convert to BigQueryTimestamp - value = BigQuery.timestamp(Number(value)); + // High precision timestamp behaviour + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + /* + At this point, 'value' will equal the timestamp value returned from the + server. We need to parse this value differently depending on its format. + For example, value could be any of the following: + 1672574400123456 + 1672574400.123456 + 2023-01-01T12:00:00.123456789123Z + */ + const listParams = options.listParams; + const timestampOutputFormat = listParams + ? listParams['formatOptions.timestampOutputFormat'] + : undefined; + const useInt64Timestamp = listParams + ? listParams['formatOptions.useInt64Timestamp'] + : undefined; + if (timestampOutputFormat === 'ISO8601_STRING') { + // value is ISO string, create BigQueryTimestamp wrapping the string + value = BigQuery.timestamp(value); + } else if ( + useInt64Timestamp !== true && + timestampOutputFormat !== 'INT64' && + (useInt64Timestamp !== undefined || + timestampOutputFormat !== undefined) + ) { + // NOTE: The additional + // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) + // check is to ensure that calls to the /query endpoint remain + // unaffected as they will not be providing any listParams. + // + // If the program reaches this point in time then + // value is float seconds so convert to BigQueryTimestamp + value = BigQuery.timestamp(Number(value)); + } else { + // Expect int64 micros (default or explicit INT64) + const pd = new PreciseDate(); + pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); + value = BigQuery.timestamp(pd); + } } else { - // Expect int64 micros (default or explicit INT64) + // Old behaviour const pd = new PreciseDate(); pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); value = BigQuery.timestamp(pd); @@ -2834,16 +2858,6 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); - if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { - /* - TODO: - When https://github.com/googleapis/nodejs-precise-date/pull/302 - is released and we have full support for picoseconds in PreciseData - then we can remove this if block. - */ - this.value = value; - return; - } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index e92c6a6791d..f425a21caef 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -55,7 +55,6 @@ import {JobMetadata, JobOptions} from './job'; import bigquery from './types'; import {IntegerTypeCastOptions} from './bigquery'; import {RowQueue} from './rowQueue'; -import IDataFormatOptions = bigquery.IDataFormatOptions; // This is supposed to be a @google-cloud/storage `File` type. The storage npm // module includes these types, but is current installed as a devDependency. @@ -1867,13 +1866,35 @@ class Table extends ServiceObject { callback!(err, null, null, resp); return; } - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + // High precision timestamp behaviour + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + if (options.skipParsing) { + rows = rows || []; + } else { + rows = BigQuery.mergeSchemaWithRows_( + this.metadata.schema, + rows || [], + { + wrapIntegers, + selectedFields, + parseJSON, + listParams: qs, + }, + ); + } + } catch (err) { + callback!(err as Error | null, null, null, resp); + return; + } + } else { + // Old behaviour if (options.skipParsing) { rows = rows || []; } else { @@ -1884,25 +1905,33 @@ class Table extends ServiceObject { wrapIntegers, selectedFields, parseJSON, - listParams: qs, }, ); } - } catch (err) { - callback!(err as Error | null, null, null, resp); - return; } callback!(null, rows, nextQuery, resp); }; - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts - ? {} - : { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', - }; - const qs = extend(defaultOpts, options); + + let qs: any; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + const defaultOpts = hasAnyFormatOpts + ? {} + : { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + }; + qs = extend(defaultOpts, options); + } else { + qs = extend( + { + 'formatOptions.useInt64Timestamp': true, + }, + options, + ); + } + this.request( { uri: '/data', diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index 92550709de1..8a104d9d9db 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1042,25 +1042,27 @@ describe('BigQuery', () => { }); it('should create a table with timestampPrecision', async () => { - const table = dataset.table(generateName('timestamp-precision-table')); - const schema = { - fields: [ - { - name: 'ts_field', - type: 'TIMESTAMP', - timestampPrecision: 12, - }, - ], - }; - try { - await table.create({schema}); - const [metadata] = await table.getMetadata(); - assert.deepStrictEqual( - metadata.schema.fields[0].timestampPrecision, - '12', - ); - } catch (e) { - assert.ifError(e); + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const table = dataset.table(generateName('timestamp-precision-table')); + const schema = { + fields: [ + { + name: 'ts_field', + type: 'TIMESTAMP', + timestampPrecision: 12, + }, + ], + }; + try { + await table.create({schema}); + const [metadata] = await table.getMetadata(); + assert.deepStrictEqual( + metadata.schema.fields[0].timestampPrecision, + '12', + ); + } catch (e) { + assert.ifError(e); + } } }); @@ -1562,6 +1564,11 @@ describe('BigQuery', () => { testCases.forEach(testCase => { it(`should handle ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } /* The users use the new TIMESTAMP(12) type to indicate they want to opt in to using timestampPrecision=12. The reason is that some queries @@ -1614,6 +1621,11 @@ describe('BigQuery', () => { } }); it(`should handle nested ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } /* The users use the new TIMESTAMP(12) type to indicate they want to opt in to using timestampPrecision=12. The reason is that some queries @@ -2009,6 +2021,11 @@ describe('BigQuery', () => { testCases.forEach(testCase => { it(`should handle ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } /* The users use the new TIMESTAMP(12) type to indicate they want to opt in to using timestampPrecision=12. The reason is that some queries @@ -2063,6 +2080,11 @@ describe('BigQuery', () => { } }); it(`should handle nested ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } /* The users use the new TIMESTAMP(12) type to indicate they want to opt in to using timestampPrecision=12. The reason is that some queries diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 0fe388e1e3b..bd622b9dfec 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -38,6 +38,9 @@ describe('Timestamp Output Format System Tests', () => { const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; before(async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } await dataset.create(); await table.create({ schema: [{name: 'ts', type: 'TIMESTAMP', timestampPrecision: '12'}], @@ -159,6 +162,9 @@ describe('Timestamp Output Format System Tests', () => { expectedTsValue, }) => { it(name, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } const options: {[key: string]: any} = {}; if (timestampOutputFormat !== undefined) { options['formatOptions.timestampOutputFormat'] = @@ -185,6 +191,10 @@ describe('Timestamp Output Format System Tests', () => { ); it('should make a request with ISO8601_STRING when no format options are being used', done => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + done(); + return; + } void (async () => { const originalRequest = table.request; const requestPromise: Promise = new Promise( diff --git a/handwritten/bigquery/test/bigquery.ts b/handwritten/bigquery/test/bigquery.ts index 74fbbedf1c7..e5bbf0e222a 100644 --- a/handwritten/bigquery/test/bigquery.ts +++ b/handwritten/bigquery/test/bigquery.ts @@ -3439,6 +3439,14 @@ describe('BigQuery', () => { delete req[key]; } } + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + timestampOutputFormat: 'ISO8601_STRING', + } + : { + useInt64Timestamp: true, + }; const expectedReq = { query: QUERY_STRING, useLegacySql: false, @@ -3465,9 +3473,7 @@ describe('BigQuery', () => { key: 'value', }, jobCreationMode: 'JOB_CREATION_REQUIRED', - formatOptions: { - timestampOutputFormat: 'ISO8601_STRING', - }, + formatOptions, }; assert.deepStrictEqual(req, expectedReq); }); @@ -3508,6 +3514,9 @@ describe('BigQuery', () => { testCases.forEach(testCase => { it(`should handle ${testCase.name}`, () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } const req = bq.buildQueryRequest_(QUERY_STRING, testCase.opts); const expectedReq = { @@ -3535,6 +3544,7 @@ describe('BigQuery', () => { }); }); }); + it('should create a QueryRequest from a SQL string', () => { const req = bq.buildQueryRequest_(QUERY_STRING, {}); for (const key in req) { @@ -3542,14 +3552,20 @@ describe('BigQuery', () => { delete req[key]; } } + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + timestampOutputFormat: 'ISO8601_STRING', + } + : { + useInt64Timestamp: true, + }; const expectedReq = { query: QUERY_STRING, useLegacySql: false, requestId: req.requestId, jobCreationMode: 'JOB_CREATION_OPTIONAL', - formatOptions: { - timestampOutputFormat: 'ISO8601_STRING', - }, + formatOptions, }; assert.deepStrictEqual(req, expectedReq); }); diff --git a/handwritten/bigquery/test/table.ts b/handwritten/bigquery/test/table.ts index 2e13f1572d6..f3e59d7f3d9 100644 --- a/handwritten/bigquery/test/table.ts +++ b/handwritten/bigquery/test/table.ts @@ -2041,12 +2041,20 @@ describe('BigQuery/Table', () => { it('should make correct API request', done => { const options = {a: 'b', c: 'd'}; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.strictEqual(reqOpts.uri, '/data'); assert.deepStrictEqual(reqOpts.qs, { ...options, - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + ...formatOptions, }); callback(null, {}); }; @@ -2201,13 +2209,16 @@ describe('BigQuery/Table', () => { sandbox.restore(); const mergeStub = sandbox.stub(BigQuery, 'mergeSchemaWithRows_'); - table.getRows({skipParsing: true}, (err: Error, rows_: {}[], nextQuery: {}, apiResponse: any) => { - assert.ifError(err); - assert.strictEqual(rows_, rows); - assert.strictEqual(mergeStub.called, false); - assert.deepStrictEqual(apiResponse.rows, rows); - done(); - }); + table.getRows( + {skipParsing: true}, + (err: Error, rows_: {}[], nextQuery: {}, apiResponse: any) => { + assert.ifError(err); + assert.strictEqual(rows_, rows); + assert.strictEqual(mergeStub.called, false); + assert.deepStrictEqual(apiResponse.rows, rows); + done(); + }, + ); }); it('should pass nextQuery if pageToken is returned', done => { @@ -2217,18 +2228,33 @@ describe('BigQuery/Table', () => { // Set a schema so it doesn't try to refresh the metadata. table.metadata = {schema: {}}; + const callbackResponse = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.useInt64Timestamp': true, + pageToken, + } + : { + pageToken, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { - callback(null, { - pageToken, - }); + callback(null, callbackResponse); }; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.getRows(options, (err: Error, rows: {}, nextQuery: {}) => { assert.ifError(err); assert.deepStrictEqual(nextQuery, { a: 'b', c: 'd', - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + ...formatOptions, pageToken, }); // Original object isn't affected. @@ -2442,10 +2468,18 @@ describe('BigQuery/Table', () => { const wrapIntegers = {integerTypeCastFunction: sinon.stub()}; const options = {wrapIntegers}; const merged = [{name: 'stephen'}]; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + ...formatOptions, }); callback(null, {}); }; @@ -2466,10 +2500,18 @@ describe('BigQuery/Table', () => { parseJSON: true, }; const merged = [{name: 'stephen'}]; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + ...formatOptions, }); callback(null, {}); };