Skip to content
This repository was archived by the owner on Mar 5, 2026. It is now read-only.
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions protos/google/cloud/bigquery/storage/v1/table.proto
Original file line number Diff line number Diff line change
Expand Up @@ -186,4 +186,8 @@ message TableFieldSchema {
// * TIMESTAMP
FieldElementType range_element_type = 11
[(google.api.field_behavior) = OPTIONAL];

// Optional. The precision of the TIMESTAMP field. A value of 6 denotes
// microsecond precision, while 12 denotes picosecond precision.
int64 timestamp_precision = 12 [(google.api.field_behavior) = OPTIONAL];
}
8 changes: 7 additions & 1 deletion protos/protos.d.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

39 changes: 38 additions & 1 deletion protos/protos.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 15 additions & 1 deletion protos/protos.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 7 additions & 1 deletion src/adapt/proto.ts
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,13 @@ function convertTableFieldSchemaToFieldDescriptorProto(
label: label,
});
} else {
const pType = bqTypeToFieldTypeMap[type];
let pType = bqTypeToFieldTypeMap[type];
if (
type === TableFieldSchema.Type.TIMESTAMP &&
Number(field.timestampPrecision) === 12
) {
pType = FieldDescriptorProto.Type.TYPE_STRING;
}
if (pType === null) {
throw Error(`table field type ${type} not supported`);
}
Expand Down
9 changes: 9 additions & 0 deletions src/adapt/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ type ITableFieldSchema = {
*/
type?: string;

/**
* [Optional] The precision for TIMESTAMP fields. 6 for microsecond, 12 for picosecond.
*/
timestampPrecision?: number | string;

/**
* Represents the type of a field element.
*/
Expand Down Expand Up @@ -101,6 +106,10 @@ function bqFieldToStorageField(field: ITableFieldSchema): StorageTableField {
out.description = field.description;
}

if (field.timestampPrecision) {
out.timestampPrecision = field.timestampPrecision;
}

if (!field.type) {
throw Error(
`could not convert field (${field.name}) due to unknown type value: ${field.type}`,
Expand Down
149 changes: 149 additions & 0 deletions test/picosecond_timestamp.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import * as assert from 'assert';
import {describe, it} from 'mocha';
import * as protobuf from 'protobufjs';
import * as adapt from '../src/adapt';
import {protos} from '../src';
import {JSONEncoder} from '../src/managedwriter/encoder';

const TableFieldSchema =
protos.google.cloud.bigquery.storage.v1.TableFieldSchema;
const {Type} = protobuf;

describe('Picosecond Timestamp Support', () => {
describe('Schema conversion', () => {
it('should carry over timestampPrecision to StorageTableField', () => {
const schema = {
fields: [
{
name: 'ts_pico',
type: 'TIMESTAMP',
timestampPrecision: 12,
},
{
name: 'ts_micro',
type: 'TIMESTAMP',
timestampPrecision: 6,
},
{
name: 'ts_default',
type: 'TIMESTAMP',
},
],
};
const storageSchema =
adapt.convertBigQuerySchemaToStorageTableSchema(schema);
assert.strictEqual(
Number(storageSchema.fields![0].timestampPrecision),
12

Check failure on line 51 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
assert.strictEqual(
Number(storageSchema.fields![1].timestampPrecision),
6

Check failure on line 55 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
assert.strictEqual(
storageSchema.fields![2].timestampPrecision,
undefined

Check failure on line 59 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
});
});

describe('Dynamic Proto generation', () => {
it('should map TIMESTAMP with precision 12 to TYPE_STRING', () => {
const storageSchema = {
fields: [
{
name: 'ts_pico',
type: TableFieldSchema.Type.TIMESTAMP,
timestampPrecision: 12,
},
{
name: 'ts_micro',
type: TableFieldSchema.Type.TIMESTAMP,
timestampPrecision: 6,
},
],
};
const protoDescriptor = adapt.convertStorageSchemaToProto2Descriptor(
storageSchema,
'TestPico'

Check failure on line 82 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
assert.strictEqual(protoDescriptor.field![0].name, 'ts_pico');
assert.strictEqual(
protoDescriptor.field![0].type,
protos.google.protobuf.FieldDescriptorProto.Type.TYPE_STRING

Check failure on line 87 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
assert.strictEqual(protoDescriptor.field![1].name, 'ts_micro');
assert.strictEqual(
protoDescriptor.field![1].type,
protos.google.protobuf.FieldDescriptorProto.Type.TYPE_INT64

Check failure on line 92 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
});
});

describe('JSONEncoder', () => {
it('should preserve picosecond timestamp strings', () => {
const storageSchema = {
fields: [
{
name: 'ts_pico',
type: TableFieldSchema.Type.TIMESTAMP,
timestampPrecision: 12,
},
],
};
const protoDescriptor = adapt.convertStorageSchemaToProto2Descriptor(
storageSchema,
'TestEncoder'

Check failure on line 110 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
const encoder = new JSONEncoder({protoDescriptor});
const picoStr = '2025-05-07T20:17:05.123456789012Z';
const rows = [{ts_pico: picoStr}];
const encodedRows = encoder.encodeRows(rows);

const TestProto = Type.fromDescriptor(protoDescriptor);
const decoded = TestProto.decode(encodedRows[0]).toJSON();
assert.strictEqual(decoded.ts_pico, picoStr);
});

it('should format Date objects for picosecond TIMESTAMP fields', () => {
const storageSchema = {
fields: [
{
name: 'ts_pico',
type: TableFieldSchema.Type.TIMESTAMP,
timestampPrecision: 12,
},
],
};
const protoDescriptor = adapt.convertStorageSchemaToProto2Descriptor(
storageSchema,
'TestEncoderDate'

Check failure on line 134 in test/picosecond_timestamp.ts

View workflow job for this annotation

GitHub Actions / lint

Insert `,`
);
const encoder = new JSONEncoder({protoDescriptor});
const date = new Date('2025-05-07T20:17:05.123Z');
const rows = [{ts_pico: date}];
const encodedRows = encoder.encodeRows(rows);

const TestProto = Type.fromDescriptor(protoDescriptor);
const decoded = TestProto.decode(encodedRows[0]).toJSON();
// Currently it formats as DATETIME string: "YYYY-MM-DD HH:MM:SS.SSS"
// Wait, let's see what value.toJSON().replace(/^(.*)T(.*)Z$/, '$1 $2') does.
// 2025-05-07T20:17:05.123Z -> 2025-05-07 20:17:05.123
assert.strictEqual(decoded.ts_pico, '2025-05-07 20:17:05.123');
});
});
});
Loading