Skip to content

fix(bigquery): propagate DataFormatOptions to listTableData/getQueryResults and FieldValue #14651

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lqiu96 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(bigquery): propagate DataFormatOptions to listTableData/getQueryResults and FieldValue #14651

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e7fccb2
fix(bigquery): propagate DataFormatOptions to listTableData/getQueryR…
lqiu96 Oct 13, 2026
c322dbe
chore: Fix lint issues
lqiu96 Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 7 additions & 13 deletions ...-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2207,18 +2207,17 @@ public TableDataList call() throws IOException {
new PageImpl<>(
new TableDataPageFetcher(tableId, schema, serviceOptions, cursor, pageOptionMap),
cursor,
transformTableData(
result.getRows(),
schema,
serviceOptions.getDataFormatOptions().useInt64Timestamp())),
transformTableData(result.getRows(), schema, serviceOptions.getDataFormatOptions())),
result.getTotalRows());
} catch (BigQueryRetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
}

private static Iterable<FieldValueList> transformTableData(
Iterable<TableRow> tableDataPb, final Schema schema, boolean useInt64Timestamps) {
Iterable<TableRow> tableDataPb,
final Schema schema,
final DataFormatOptions dataFormatOptions) {
return ImmutableList.copyOf(
Iterables.transform(
tableDataPb != null ? tableDataPb : ImmutableList.<TableRow>of(),
Expand All @@ -2227,7 +2226,7 @@ private static Iterable<FieldValueList> transformTableData(

@Override
public FieldValueList apply(TableRow rowPb) {
return FieldValueList.fromPb(rowPb.getF(), fields, useInt64Timestamps);
return FieldValueList.fromPb(rowPb.getF(), fields, dataFormatOptions);
}
}));
}
Expand Down Expand Up @@ -2485,9 +2484,7 @@ public com.google.api.services.bigquery.model.QueryResponse call()
new QueryPageFetcher(jobId, schema, getOptions(), cursor, optionMap(options)),
cursor,
transformTableData(
results.getRows(),
schema,
getOptions().getDataFormatOptions().useInt64Timestamp())))
results.getRows(), schema, getOptions().getDataFormatOptions())))
.setJobId(jobId)
.setRowsInPage(results.getRows() != null ? (long) results.getRows().size() : 0L)
.build();
Expand All @@ -2499,10 +2496,7 @@ public com.google.api.services.bigquery.model.QueryResponse call()
new PageImpl<>(
new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)),
null,
transformTableData(
results.getRows(),
schema,
getOptions().getDataFormatOptions().useInt64Timestamp())))
transformTableData(results.getRows(), schema, getOptions().getDataFormatOptions())))
// Return the JobID of the successful job
.setJobId(
results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null)
Expand Down
3 changes: 0 additions & 3 deletions ...gquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ public class BigQueryOptions extends ServiceOptions<BigQuery, BigQueryOptions> {
private final String location;
// set the option ThrowNotFound when you want to throw the exception when the value not found
private boolean setThrowNotFound;
private boolean useInt64Timestamps;
private DataFormatOptions dataFormatOptions;
private JobCreationMode defaultJobCreationMode = JobCreationMode.JOB_CREATION_MODE_UNSPECIFIED;
private boolean enableOpenTelemetryTracing;
Expand Down Expand Up @@ -172,7 +171,6 @@ public BigQueryOptions build() {
private BigQueryOptions(Builder builder) {
super(BigQueryFactory.class, BigQueryRpcFactory.class, builder, new BigQueryDefaults());
this.location = builder.location;
this.useInt64Timestamps = builder.useInt64Timestamps;
this.enableOpenTelemetryTracing = builder.enableOpenTelemetryTracing;
this.openTelemetryTracer = builder.openTelemetryTracer;
if (builder.resultRetryAlgorithm != null) {
Expand Down Expand Up @@ -249,7 +247,6 @@ public void setThrowNotFound(boolean setThrowNotFound) {
*/
@ObsoleteApi("Use Builder#setDataFormatOptions(DataFormatOptions) instead")
public void setUseInt64Timestamps(boolean useInt64Timestamps) {
this.useInt64Timestamps = useInt64Timestamps;
// Because this setter exists outside the Builder, DataFormatOptions needs be rebuilt to
// account for this setting.
this.dataFormatOptions =
Expand Down
46 changes: 29 additions & 17 deletions java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import com.google.common.io.BaseEncoding;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
import java.time.Duration;
import java.time.Instant;
Expand Down Expand Up @@ -76,13 +75,13 @@ public enum Attribute {
}

private FieldValue(Attribute attribute, Object value) {
this(attribute, value, false);
this(attribute, value, DataFormatOptions.newBuilder().build());
}

private FieldValue(Attribute attribute, Object value, Boolean useInt64Timestamps) {
private FieldValue(Attribute attribute, Object value, DataFormatOptions dataFormatOptions) {
this.attribute = checkNotNull(attribute);
this.value = value;
this.useInt64Timestamps = useInt64Timestamps;
this.useInt64Timestamps = dataFormatOptions != null && dataFormatOptions.useInt64Timestamp();
}

/**
Expand Down Expand Up @@ -219,10 +218,14 @@ public boolean getBooleanValue() {
@SuppressWarnings("unchecked")
public long getTimestampValue() {
if (useInt64Timestamps) {
return new BigInteger(getStringValue()).longValue();
// When useInt64Timestamp is enabled, the backend returns timestamps as signed 64-bit integer
// (INT64) microseconds since epoch (e.g. "1408452095220000"), which always fits within a
// primitive long. Parse directly with Long.parseLong to avoid allocating a BigInteger per
// cell.
return Long.parseLong(getStringValue());
}
// timestamps are encoded in the format 1408452095.22 where the integer part is seconds since
// epoch (e.g. 1408452095.22 == 2014-08-19 07:41:35.220 -05:00)
// By default (useInt64Timestamp = false), timestamps are encoded as floating-point seconds
// since epoch (e.g. 1408452095.22 == 2014-08-19 07:41:35.220 -05:00).
BigDecimal secondsWithMicro = new BigDecimal(getStringValue());
// Rounding the BigDecimal to the nearest whole number before setting the longValue in order to
// address TimeStamp rounding issue described in
Expand Down Expand Up @@ -370,22 +373,31 @@ public final boolean equals(Object obj) {
*/
@BetaApi
public static FieldValue of(Attribute attribute, Object value) {
return of(attribute, value, false);
return of(attribute, value, DataFormatOptions.newBuilder().build());
}

@BetaApi
public static FieldValue of(Attribute attribute, Object value, Boolean useInt64Timestamps) {
return new FieldValue(attribute, value, useInt64Timestamps);
return of(
attribute,
value,
DataFormatOptions.newBuilder()
.useInt64Timestamp(Boolean.TRUE.equals(useInt64Timestamps))
.build());
}

static FieldValue of(Attribute attribute, Object value, DataFormatOptions dataFormatOptions) {
return new FieldValue(attribute, value, dataFormatOptions);
}

static FieldValue fromPb(Object cellPb) {
return fromPb(cellPb, null, false);
return fromPb(cellPb, null, DataFormatOptions.newBuilder().build());
}

@SuppressWarnings("unchecked")
static FieldValue fromPb(Object cellPb, Field recordSchema, Boolean useInt64Timestamps) {
static FieldValue fromPb(Object cellPb, Field recordSchema, DataFormatOptions dataFormatOptions) {
if (Data.isNull(cellPb)) {
return FieldValue.of(Attribute.PRIMITIVE, null, useInt64Timestamps);
return FieldValue.of(Attribute.PRIMITIVE, null, dataFormatOptions);
}
if (cellPb instanceof String) {
if ((recordSchema != null)
Expand All @@ -394,14 +406,14 @@ static FieldValue fromPb(Object cellPb, Field recordSchema, Boolean useInt64Time
return FieldValue.of(
Attribute.RANGE,
Range.of((String) cellPb, recordSchema.getRangeElementType()),
useInt64Timestamps);
dataFormatOptions);
}
return FieldValue.of(Attribute.PRIMITIVE, cellPb, useInt64Timestamps);
return FieldValue.of(Attribute.PRIMITIVE, cellPb, dataFormatOptions);
}
if (cellPb instanceof List) {
return FieldValue.of(
Attribute.REPEATED,
FieldValueList.fromPb((List<Object>) cellPb, null, useInt64Timestamps));
FieldValueList.fromPb((List<Object>) cellPb, null, dataFormatOptions));
}
if (cellPb instanceof Map) {
Map<String, Object> cellMapPb = (Map<String, Object>) cellPb;
Expand All @@ -410,12 +422,12 @@ static FieldValue fromPb(Object cellPb, Field recordSchema, Boolean useInt64Time
return FieldValue.of(
Attribute.RECORD,
FieldValueList.fromPb(
(List<Object>) cellMapPb.get("f"), subFieldsSchema, useInt64Timestamps));
(List<Object>) cellMapPb.get("f"), subFieldsSchema, dataFormatOptions));
}
// This should never be the case when we are processing a first level table field (i.e. a
// row's field, not a record sub-field)
if (cellMapPb.containsKey("v")) {
return FieldValue.fromPb(cellMapPb.get("v"), recordSchema, useInt64Timestamps);
return FieldValue.fromPb(cellMapPb.get("v"), recordSchema, dataFormatOptions);
}
}
throw new IllegalArgumentException("Unexpected table cell format");
Expand Down
9 changes: 5 additions & 4 deletions ...igquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValueList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,11 @@ FieldValueList withSchema(FieldList schema) {
}

static FieldValueList fromPb(List<?> rowPb, FieldList schema) {
return fromPb(rowPb, schema, false);
return fromPb(rowPb, schema, DataFormatOptions.newBuilder().build());
}

static FieldValueList fromPb(List<?> rowPb, FieldList schema, Boolean useInt64Timestamps) {
static FieldValueList fromPb(
List<?> rowPb, FieldList schema, DataFormatOptions dataFormatOptions) {
List<FieldValue> row = new ArrayList<>(rowPb.size());
if (schema != null) {
if (schema.size() != rowPb.size()) {
Expand All @@ -124,11 +125,11 @@ static FieldValueList fromPb(List<?> rowPb, FieldList schema, Boolean useInt64Ti
Iterator<Field> schemaIter = schema.iterator();
Iterator<?> rowPbIter = rowPb.iterator();
while (rowPbIter.hasNext() && schemaIter.hasNext()) {
row.add(FieldValue.fromPb(rowPbIter.next(), schemaIter.next(), useInt64Timestamps));
row.add(FieldValue.fromPb(rowPbIter.next(), schemaIter.next(), dataFormatOptions));
}
} else {
for (Object cellPb : rowPb) {
row.add(FieldValue.fromPb(cellPb, null, useInt64Timestamps));
row.add(FieldValue.fromPb(cellPb, null, dataFormatOptions));
}
}

Expand Down
16 changes: 16 additions & 0 deletions ...google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1424,6 +1424,14 @@ public TableDataList listTableDataSkipExceptionTranslation(
.tabledata()
.list(projectId, datasetId, tableId)
.setPrettyPrint(false)
.setFormatOptionsUseInt64Timestamp(
this.options.getDataFormatOptions() != null
? this.options.getDataFormatOptions().useInt64Timestamp()
: null)
.setFormatOptionsTimestampOutputFormat(
this.options.getDataFormatOptions() != null
? this.options.getDataFormatOptions().timestampFormatOptions().toString()
: null)
.setMaxResults(Option.MAX_RESULTS.getLong(options))
.setPageToken(Option.PAGE_TOKEN.getString(options))
.setStartIndex(
Expand Down Expand Up @@ -1827,6 +1835,14 @@ public GetQueryResultsResponse getQueryResultsSkipExceptionTranslation(
.jobs()
.getQueryResults(projectId, jobId)
.setPrettyPrint(false)
.setFormatOptionsUseInt64Timestamp(
this.options.getDataFormatOptions() != null
? this.options.getDataFormatOptions().useInt64Timestamp()
: null)
.setFormatOptionsTimestampOutputFormat(
this.options.getDataFormatOptions() != null
? this.options.getDataFormatOptions().timestampFormatOptions().toString()
: null)
.setLocation(location)
.setMaxResults(Option.MAX_RESULTS.getLong(options))
.setPageToken(Option.PAGE_TOKEN.getString(options))
Expand Down
13 changes: 8 additions & 5 deletions ...ery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,16 +75,19 @@ class FieldValueListTest {
ImmutableMap.of(
"v", "99999999999999999999999999999999999999.99999999999999999999999999999999999999");

private final DataFormatOptions losslessOptions =
DataFormatOptions.newBuilder().useInt64Timestamp(true).build();
private final FieldValue booleanFv = FieldValue.of(Attribute.PRIMITIVE, "false");
private final FieldValue booleanLosslessTimestampFv =
FieldValue.of(Attribute.PRIMITIVE, "false", true);
FieldValue.of(Attribute.PRIMITIVE, "false", losslessOptions);
private final FieldValue integerFv = FieldValue.of(Attribute.PRIMITIVE, "1");
private final FieldValue integerLosslessTimestampFv =
FieldValue.of(Attribute.PRIMITIVE, "1", true);
FieldValue.of(Attribute.PRIMITIVE, "1", losslessOptions);
private final FieldValue floatFv = FieldValue.of(Attribute.PRIMITIVE, "1.5");
private final FieldValue stringFv = FieldValue.of(Attribute.PRIMITIVE, "string");
private final FieldValue timestampFv = FieldValue.of(Attribute.PRIMITIVE, "42");
private final FieldValue losslessTimestampFv = FieldValue.of(Attribute.PRIMITIVE, "42", true);
private final FieldValue losslessTimestampFv =
FieldValue.of(Attribute.PRIMITIVE, "42", losslessOptions);
private final FieldValue bytesFv = FieldValue.of(Attribute.PRIMITIVE, BYTES_BASE64);
private final FieldValue nullFv = FieldValue.of(Attribute.PRIMITIVE, null);
private final FieldValue repeatedFv =
Expand Down Expand Up @@ -144,11 +147,11 @@ void testFromPb() {
// Schema does not influence values equality
assertEquals(fieldValues, FieldValueList.fromPb(fieldValuesPb, null));

assertNotEquals(fieldValues, FieldValueList.fromPb(fieldValuesPb, null, true));
assertNotEquals(fieldValues, FieldValueList.fromPb(fieldValuesPb, null, losslessOptions));

assertEquals(
fieldValuesLosslessTimestamp,
FieldValueList.fromPb(fieldValuesLosslessTimestampPb, null, true));
FieldValueList.fromPb(fieldValuesLosslessTimestampPb, null, losslessOptions));
}

@Test
Expand Down
10 changes: 7 additions & 3 deletions ...igquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,18 +134,20 @@ public void testTimestamp() {

@Test
public void testInt64Timestamp() {
DataFormatOptions losslessOptions =
DataFormatOptions.newBuilder().useInt64Timestamp(true).build();
FieldValue lossyFieldValue =
FieldValue.of(FieldValue.Attribute.PRIMITIVE, "1.9954383398377106E10");
long lossy = lossyFieldValue.getTimestampValue();

FieldValue losslessFieldValue =
FieldValue.of(FieldValue.Attribute.PRIMITIVE, "19954383398377106", true);
FieldValue.of(FieldValue.Attribute.PRIMITIVE, "19954383398377106", losslessOptions);
long lossless = losslessFieldValue.getTimestampValue();

assertEquals(lossy, lossless);

FieldValue fieldValue =
FieldValue.of(FieldValue.Attribute.PRIMITIVE, "19954383398377106", true);
FieldValue.of(FieldValue.Attribute.PRIMITIVE, "19954383398377106", losslessOptions);
long received = fieldValue.getTimestampValue();
long expected = 19954383398377106L;
assertEquals(expected, received);
Expand All @@ -155,9 +157,11 @@ public void testInt64Timestamp() {
public void testLosslessMaxTimestamp() {
// Test the lossless behavior (useInt64Timestamps = true)
// The backend returns a 64-bit integer string for microseconds when this option is enabled
DataFormatOptions losslessOptions =
DataFormatOptions.newBuilder().useInt64Timestamp(true).build();
String losslessTimestampString = "253402300799999999";
FieldValue losslessValue =
FieldValue.of(FieldValue.Attribute.PRIMITIVE, losslessTimestampString, true);
FieldValue.of(FieldValue.Attribute.PRIMITIVE, losslessTimestampString, losslessOptions);

// Exactly matches the microsecond equivalent of 9999-12-31 23:59:59.999999
assertEquals(253402300799999999L, losslessValue.getTimestampValue());
Expand Down
35 changes: 35 additions & 0 deletions ...le-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import com.google.api.services.bigquery.model.TableReference;
import com.google.cloud.NoCredentials;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.DataFormatOptions;
import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer;
import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer;
import io.opentelemetry.api.common.AttributeKey;
Expand Down Expand Up @@ -1697,6 +1698,40 @@ public void testGetQueryResultsWithRowLimitNoTelemetry() throws Exception {
verifyNoSpans();
}

@Test
public void testDataFormatOptionsPropagatedToListTableDataAndGetQueryResults()
throws Exception {
BigQueryOptions int64Options =
BigQueryOptions.newBuilder()
.setProjectId(PROJECT_ID)
.setCredentials(NoCredentials.getInstance())
.setEnableOpenTelemetryTracing(false)
.setDataFormatOptions(
DataFormatOptions.newBuilder()
.useInt64Timestamp(true)
.timestampFormatOptions(
DataFormatOptions.TimestampFormatOptions.ISO8601_STRING)
.build())
.setTransportOptions(
BigQueryOptions.getDefaultHttpTransportOptions().toBuilder()
.setHttpTransportFactory(() -> mockTransport)
.build())
.build();
HttpBigQueryRpc int64Rpc = new HttpBigQueryRpc(int64Options);

setMockResponse("{\"kind\":\"bigquery#tableDataList\",\"rows\":[]}");
int64Rpc.listTableDataSkipExceptionTranslation(
PROJECT_ID, DATASET_ID, TABLE_ID, new HashMap<>());
verifyRequest("GET", "formatOptions.useInt64Timestamp=true");
verifyRequest("GET", "formatOptions.timestampOutputFormat=ISO8601_STRING");

setMockResponse("{\"kind\":\"bigquery#getQueryResultsResponse\"}");
int64Rpc.getQueryResultsSkipExceptionTranslation(
PROJECT_ID, JOB_ID, LOCATION, new HashMap<>());
verifyRequest("GET", "formatOptions.useInt64Timestamp=true");
verifyRequest("GET", "formatOptions.timestampOutputFormat=ISO8601_STRING");
}

@Test
public void testQueryRpcNoTelemetry() throws Exception {
setMockResponse("{\"kind\":\"bigquery#queryResponse\"}");
Expand Down
Loading