From 0557e69149c08c1adcb91249e51cd6cda10b5643 Mon Sep 17 00:00:00 2001 From: Neenu Shaji Date: Fri, 7 Aug 2026 13:47:16 -0400 Subject: [PATCH 1/5] feat(bigquery-jdbc): implement TypeRegistry and TypeDescriptor (#13947) --- .../jdbc/BigQueryTemporalUtility.java | 115 +++++ .../bigquery/jdbc/BigQueryTypeRegistry.java | 434 ++++++++++++++++++ .../cloud/bigquery/jdbc/TypeDescriptor.java | 78 ++++ 3 files changed, 627 insertions(+) create mode 100644 java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTemporalUtility.java create mode 100644 java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java create mode 100644 java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/TypeDescriptor.java diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTemporalUtility.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTemporalUtility.java new file mode 100644 index 000000000000..b26cf78bac0a --- /dev/null +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTemporalUtility.java @@ -0,0 +1,115 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc; + +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.util.Calendar; + +/** + * A highly optimized utility for bridging BigQuery's civil time and absolute time semantics to + * legacy JDBC Date/Time/Timestamp classes using JSR-310 timezone anchoring. + */ +final class BigQueryTemporalUtility { + + private BigQueryTemporalUtility() {} + + /** + * Converts a BigQuery civil DATETIME string into an absolute Timestamp by anchoring it to the + * provided timezone (or JVM default if null). + */ + public static Timestamp boxDateTime(String val, ZoneId zoneId) { + ZoneId targetZone = zoneId != null ? zoneId : ZoneId.systemDefault(); + String isoString = val.replace(' ', 'T'); + return Timestamp.from(LocalDateTime.parse(isoString).atZone(targetZone).toInstant()); + } + + /** + * Converts a BigQuery civil DATE string into an absolute Date by anchoring it to midnight of the + * provided timezone (or JVM default if null). + */ + public static Date boxDate(String val, ZoneId zoneId) { + ZoneId targetZone = zoneId != null ? zoneId : ZoneId.systemDefault(); + return new Date(LocalDate.parse(val).atStartOfDay(targetZone).toInstant().toEpochMilli()); + } + + /** + * Converts a BigQuery civil TIME string into an absolute Time. If a ZoneId is provided (e.g. from + * the legacy JDBC 3.0 getTime(col, Calendar) API), this uses legacy Calendar manipulation to + * strictly mimic older JVM historical DST quirks for 1970. If no ZoneId is provided (e.g. modern + * JDBC 4.2 getObject(col, LocalTime.class)), this uses pure JSR-310 math which guarantees + * perfectly accurate modern conversions. + */ + public static Time boxTime(String val, ZoneId zoneId) { + LocalTime localTime = LocalTime.parse(val); + + if (zoneId == null) { + // JDBC 4.2 Modern API (no Calendar provided): + // Use pure JSR-310 math for perfectly accurate modern conversions without Calendar quirks. + return new Time( + localTime + .atDate(LocalDate.of(1970, 1, 1)) + .atZone(ZoneId.systemDefault()) + .toInstant() + .toEpochMilli()); + } + + // Legacy JDBC 3.0 API (Calendar provided): + // Use legacy Calendar manipulation to intentionally replicate old JVM historical DST quirks + // for January 1, 1970, ensuring strict backwards compatibility for legacy ORMs. + Calendar targetCal = Calendar.getInstance(java.util.TimeZone.getTimeZone(zoneId)); + targetCal.set(Calendar.YEAR, 1970); + targetCal.set(Calendar.MONTH, Calendar.JANUARY); + targetCal.set(Calendar.DAY_OF_MONTH, 1); + targetCal.set(Calendar.HOUR_OF_DAY, localTime.getHour()); + targetCal.set(Calendar.MINUTE, localTime.getMinute()); + targetCal.set(Calendar.SECOND, localTime.getSecond()); + targetCal.set(Calendar.MILLISECOND, localTime.getNano() / 1_000_000); + + return new Time(targetCal.getTimeInMillis()); + } + + /** + * Converts a BigQuery absolute TIMESTAMP string into a legacy Timestamp. Because it is absolute, + * the Calendar timezone is explicitly ignored per JDBC 4.2 spec. + */ + public static Timestamp boxTimestamp(String val) { + String iso = val; + // Handle the " UTC" suffix format + if (iso.endsWith(" UTC")) { + iso = iso.substring(0, iso.length() - 4) + "Z"; + } + // Replace the date-time space separator with 'T' (e.g. 2023-10-01 12:00:00 -> + // 2023-10-01T12:00:00) + if (iso.length() > 10 && iso.charAt(10) == ' ') { + iso = iso.substring(0, 10) + 'T' + iso.substring(11); + } + + try { + return Timestamp.from(Instant.parse(iso)); + } catch (java.time.format.DateTimeParseException e) { + // Fallback for non-standard formats + return Timestamp.valueOf(val); + } + } +} diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java new file mode 100644 index 000000000000..c2142072db5d --- /dev/null +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java @@ -0,0 +1,434 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc; + +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import java.math.BigDecimal; +import java.sql.Array; +import java.sql.Date; +import java.sql.Struct; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A central, bidirectional engine for resolving and coercing types between JDBC, Java, and + * BigQuery. + */ +final class BigQueryTypeRegistry { + + private static final TypeDescriptor[] DESCRIPTORS_BY_ORDINAL; + private static final Map, TypeDescriptor> DESCRIPTORS_BY_CLASS; + private static final Map> DESCRIPTORS_BY_JDBC_TYPE; + + static { + DESCRIPTORS_BY_ORDINAL = new TypeDescriptor[StandardSQLTypeName.values().length]; + DESCRIPTORS_BY_CLASS = new ConcurrentHashMap<>(); + DESCRIPTORS_BY_JDBC_TYPE = new ConcurrentHashMap<>(); + + register(createBoolDescriptor()); + register(createStringDescriptor()); + register(createInt64Descriptor()); + register(createFloat64Descriptor()); + register(createNumericDescriptor()); + register(createDateDescriptor()); + register(createDatetimeDescriptor()); + register(createTimestampDescriptor()); + register(createTimeDescriptor()); + register(createBytesDescriptor()); + register(createArrayDescriptor()); + register(createStructDescriptor()); + register(createJsonDescriptor()); + register(createBignumericDescriptor()); + register(createGeographyDescriptor()); + register(createIntervalDescriptor()); + register(createRangeDescriptor()); + } + + static TypeDescriptor createBoolDescriptor() { + return new TypeDescriptor<>( + Types.BOOLEAN, + Boolean.class, + StandardSQLTypeName.BOOL, + Arrays.asList(Boolean.class), + (val, targetClass, zone) -> { + if (val instanceof Boolean) return val; + if (val instanceof String) return Boolean.parseBoolean((String) val); + throw new BigQueryJdbcException("Cannot convert to BOOL: " + val); + }); + } + + static TypeDescriptor createStringDescriptor() { + return new TypeDescriptor<>( + Types.NVARCHAR, + String.class, + StandardSQLTypeName.STRING, + Arrays.asList(String.class), + (val, targetClass, zone) -> String.valueOf(val)); + } + + static TypeDescriptor createInt64Descriptor() { + return new TypeDescriptor<>( + Types.BIGINT, + Long.class, + StandardSQLTypeName.INT64, + Arrays.asList(Long.class, Integer.class, Short.class, Byte.class), + (val, targetClass, zone) -> { + long longVal; + if (val instanceof Number) longVal = ((Number) val).longValue(); + else if (val instanceof String) longVal = Long.parseLong((String) val); + else throw new BigQueryJdbcException("Cannot convert to INT64: " + val); + + if (targetClass == Integer.class) return (int) longVal; + if (targetClass == Short.class) return (short) longVal; + if (targetClass == Byte.class) return (byte) longVal; + return longVal; + }); + } + + static TypeDescriptor createFloat64Descriptor() { + return new TypeDescriptor<>( + Types.DOUBLE, + Double.class, + StandardSQLTypeName.FLOAT64, + Arrays.asList(Double.class, Float.class), + (val, targetClass, zone) -> { + double doubleVal; + if (val instanceof Number) doubleVal = ((Number) val).doubleValue(); + else if (val instanceof String) doubleVal = Double.parseDouble((String) val); + else throw new BigQueryJdbcException("Cannot convert to FLOAT64: " + val); + + if (targetClass == Float.class) return (float) doubleVal; + return doubleVal; + }); + } + + static TypeDescriptor createNumericDescriptor() { + return new TypeDescriptor<>( + Types.NUMERIC, + BigDecimal.class, + StandardSQLTypeName.NUMERIC, + Arrays.asList(BigDecimal.class), + (val, targetClass, zone) -> { + if (val instanceof BigDecimal) return val; + if (val instanceof Number) return new BigDecimal(val.toString()); + if (val instanceof String) return new BigDecimal((String) val); + throw new BigQueryJdbcException("Cannot convert to NUMERIC: " + val); + }); + } + + static TypeDescriptor createDateDescriptor() { + return new TypeDescriptor<>( + Types.DATE, + Date.class, + StandardSQLTypeName.DATE, + Arrays.asList(Date.class, LocalDate.class), + (val, targetClass, zone) -> { + // TODO(Phase 3): Add native JSR-310 fast-path to bypass boxing for LocalDate + Date sqlDate; + if (val instanceof Date) sqlDate = (Date) val; + else if (val instanceof java.util.Date) + sqlDate = new Date(((java.util.Date) val).getTime()); + else if (val instanceof LocalDate) sqlDate = Date.valueOf((LocalDate) val); + else if (val instanceof String) + sqlDate = BigQueryTemporalUtility.boxDate((String) val, zone); + else throw new BigQueryJdbcException("Cannot convert to DATE: " + val); + + if (targetClass == LocalDate.class) return sqlDate.toLocalDate(); + return sqlDate; + }); + } + + static TypeDescriptor createDatetimeDescriptor() { + return new TypeDescriptor<>( + Types.TIMESTAMP, + Timestamp.class, + StandardSQLTypeName.DATETIME, + Arrays.asList(Timestamp.class, LocalDateTime.class), + (val, targetClass, zone) -> { + // TODO(Phase 3): Add native JSR-310 fast-path to bypass boxing for LocalDateTime + Timestamp ts; + if (val instanceof Timestamp) ts = (Timestamp) val; + else if (val instanceof java.util.Date) + ts = new Timestamp(((java.util.Date) val).getTime()); + else if (val instanceof LocalDateTime) ts = Timestamp.valueOf((LocalDateTime) val); + else if (val instanceof String) + ts = BigQueryTemporalUtility.boxDateTime((String) val, zone); + else throw new BigQueryJdbcException("Cannot convert to DATETIME: " + val); + + if (targetClass == LocalDateTime.class) return ts.toLocalDateTime(); + return ts; + }); + } + + static TypeDescriptor createTimestampDescriptor() { + return new TypeDescriptor<>( + Types.TIMESTAMP, + Timestamp.class, + StandardSQLTypeName.TIMESTAMP, + Arrays.asList(Timestamp.class, OffsetDateTime.class, Instant.class, ZonedDateTime.class), + (val, targetClass, zone) -> { + // TODO(Phase 3): Add native JSR-310 fast-path to bypass boxing for Instant, etc. + Timestamp ts; + if (val instanceof Timestamp) ts = (Timestamp) val; + else if (val instanceof java.util.Date) + ts = new Timestamp(((java.util.Date) val).getTime()); + else if (val instanceof Instant) ts = Timestamp.from((Instant) val); + else if (val instanceof OffsetDateTime) + ts = Timestamp.from(((OffsetDateTime) val).toInstant()); + else if (val instanceof ZonedDateTime) + ts = Timestamp.from(((ZonedDateTime) val).toInstant()); + else if (val instanceof String) ts = BigQueryTemporalUtility.boxTimestamp((String) val); + else throw new BigQueryJdbcException("Cannot convert to TIMESTAMP: " + val); + + if (targetClass == Instant.class) return ts.toInstant(); + if (targetClass == OffsetDateTime.class) + return ts.toInstant().atOffset(java.time.ZoneOffset.UTC); + if (targetClass == ZonedDateTime.class) + return ts.toInstant().atZone(java.time.ZoneOffset.UTC); + return ts; + }); + } + + static TypeDescriptor createTimeDescriptor() { + return new TypeDescriptor<>( + Types.TIME, + Time.class, + StandardSQLTypeName.TIME, + Arrays.asList(Time.class, LocalTime.class), + (val, targetClass, zone) -> { + if (targetClass == LocalTime.class && val instanceof String) { + // Phase 3 Fast Path: Parse directly to LocalTime to preserve microsecond precision + return LocalTime.parse((String) val); + } + + Time sqlTime; + if (val instanceof Time) sqlTime = (Time) val; + else if (val instanceof java.util.Date) + sqlTime = new Time(((java.util.Date) val).getTime()); + else if (val instanceof LocalTime) sqlTime = Time.valueOf((LocalTime) val); + else if (val instanceof String) + sqlTime = BigQueryTemporalUtility.boxTime((String) val, zone); + else throw new BigQueryJdbcException("Cannot convert to TIME: " + val); + + if (targetClass == LocalTime.class) { + // java.sql.Time.toLocalTime() drops milliseconds (JDK bug). + // We manually convert it using the JVM offset to preserve millisecond precision. + long epochMillis = sqlTime.getTime(); + return Instant.ofEpochMilli(epochMillis).atZone(ZoneId.systemDefault()).toLocalTime(); + } + return sqlTime; + }); + } + + static TypeDescriptor createBytesDescriptor() { + return new TypeDescriptor<>( + Types.VARBINARY, + byte[].class, + StandardSQLTypeName.BYTES, + Arrays.asList(byte[].class), + (val, targetClass, zone) -> { + if (val instanceof byte[]) return val; + throw new BigQueryJdbcException("Cannot convert to BYTES: " + val); + }); + } + + static TypeDescriptor createArrayDescriptor() { + return new TypeDescriptor<>( + Types.ARRAY, + Array.class, + StandardSQLTypeName.ARRAY, + Arrays.asList(Array.class), + (val, targetClass, zone) -> { + if (val instanceof Array) return val; + throw new BigQueryJdbcException("Cannot convert to ARRAY: " + val); + }); + } + + static TypeDescriptor createStructDescriptor() { + return new TypeDescriptor<>( + Types.STRUCT, + Struct.class, + StandardSQLTypeName.STRUCT, + Arrays.asList(Struct.class), + (val, targetClass, zone) -> { + if (val instanceof Struct) return val; + throw new BigQueryJdbcException("Cannot convert to STRUCT: " + val); + }); + } + + static TypeDescriptor createJsonDescriptor() { + return new TypeDescriptor<>( + Types.OTHER, + String.class, + StandardSQLTypeName.JSON, + Arrays.asList(com.google.gson.JsonObject.class), + (val, targetClass, zone) -> String.valueOf(val)); + } + + static TypeDescriptor createBignumericDescriptor() { + return new TypeDescriptor<>( + Types.NUMERIC, + BigDecimal.class, + StandardSQLTypeName.BIGNUMERIC, + Arrays.asList(BigDecimal.class), + (val, targetClass, zone) -> { + if (val instanceof BigDecimal) return val; + if (val instanceof Number) return new BigDecimal(val.toString()); + if (val instanceof String) return new BigDecimal((String) val); + throw new BigQueryJdbcException("Cannot convert to BIGNUMERIC: " + val); + }); + } + + static TypeDescriptor createGeographyDescriptor() { + return new TypeDescriptor<>( + Types.OTHER, + String.class, + StandardSQLTypeName.GEOGRAPHY, + Arrays.asList(String.class), + (val, targetClass, zone) -> String.valueOf(val)); + } + + static TypeDescriptor createIntervalDescriptor() { + return new TypeDescriptor<>( + Types.OTHER, + String.class, + StandardSQLTypeName.INTERVAL, + Arrays.asList(String.class), + (val, targetClass, zone) -> String.valueOf(val)); + } + + static TypeDescriptor createRangeDescriptor() { + return new TypeDescriptor<>( + Types.OTHER, + String.class, + StandardSQLTypeName.RANGE, + Arrays.asList(String.class), + (val, targetClass, zone) -> String.valueOf(val)); + } + + private static void register(TypeDescriptor descriptor) { + if (DESCRIPTORS_BY_ORDINAL[descriptor.getBqType().ordinal()] != null) { + throw new IllegalStateException( + "Error: Duplicate TypeDescriptor registration attempted for BigQuery type '" + + descriptor.getBqType() + + "'. A StandardSQLTypeName can only be registered once."); + } + DESCRIPTORS_BY_ORDINAL[descriptor.getBqType().ordinal()] = descriptor; + DESCRIPTORS_BY_JDBC_TYPE.putIfAbsent(descriptor.getJdbcType(), descriptor); + for (Class clazz : descriptor.getSupportedJavaTypes()) { + DESCRIPTORS_BY_CLASS.putIfAbsent(clazz, descriptor); + } + } + + private BigQueryTypeRegistry() {} + + /** + * Returns the exact BigQuery StandardSQLTypeName for a given Java class. If no mapping is found, + * returns StandardSQLTypeName.STRING as a fallback to preserve backward compatibility. + */ + public static StandardSQLTypeName toBigQueryType(Class clazz) { + TypeDescriptor descriptor = getDescriptorForClass(clazz); + if (descriptor != null) { + return descriptor.getBqType(); + } + return StandardSQLTypeName.STRING; // Legacy fallback + } + + /** Returns the default Java target class for a given JDBC type constant. */ + public static Class toJavaClass(int jdbcType) { + TypeDescriptor descriptor = DESCRIPTORS_BY_JDBC_TYPE.get(jdbcType); + if (descriptor != null) { + return descriptor.getDefaultJavaClass(); + } + return String.class; // Legacy fallback + } + + /** + * Converts the input value to the target class type by looking up the target class descriptor. + */ + @SuppressWarnings("unchecked") + public static T convert(Object input, Class targetClass) throws BigQueryJdbcException { + if (input == null) { + return null; + } + TypeDescriptor descriptor = getDescriptorForClass(targetClass); + if (descriptor == null) { + throw new BigQueryJdbcException("Unsupported target class: " + targetClass.getName()); + } + return (T) descriptor.convert(input, targetClass, null); + } + + /** + * High-performance hotpath convert for ResultSets. Converts the input value using the default + * mapping for the given BigQuery type via O(1) array indexing. + */ + public static Object convert(Object input, StandardSQLTypeName bqType, ZoneId zoneId) + throws BigQueryJdbcException { + if (input == null) return null; + int ordinal = bqType.ordinal(); + if (ordinal >= DESCRIPTORS_BY_ORDINAL.length || DESCRIPTORS_BY_ORDINAL[ordinal] == null) { + throw new BigQueryJdbcException("No type descriptor registered for BigQuery type: " + bqType); + } + TypeDescriptor descriptor = DESCRIPTORS_BY_ORDINAL[ordinal]; + return descriptor.convert(input, descriptor.getDefaultJavaClass(), zoneId); + } + + /** + * High-performance hotpath convert for ResultSets. Converts the input value to the target class + * using the descriptor for the given BigQuery type via O(1) array indexing. + */ + @SuppressWarnings("unchecked") + public static T convert( + Object input, StandardSQLTypeName bqType, Class targetClass, ZoneId zoneId) + throws BigQueryJdbcException { + if (input == null) return null; + int ordinal = bqType.ordinal(); + if (ordinal >= DESCRIPTORS_BY_ORDINAL.length || DESCRIPTORS_BY_ORDINAL[ordinal] == null) { + throw new BigQueryJdbcException("No type descriptor registered for BigQuery type: " + bqType); + } + return (T) DESCRIPTORS_BY_ORDINAL[ordinal].convert(input, targetClass, zoneId); + } + + private static TypeDescriptor getDescriptorForClass(Class clazz) { + TypeDescriptor descriptor = DESCRIPTORS_BY_CLASS.get(clazz); + if (descriptor != null) { + return descriptor; + } + // Fallback logic for subclasses/interfaces (O(N) initial lookup) + for (Map.Entry, TypeDescriptor> entry : DESCRIPTORS_BY_CLASS.entrySet()) { + if (entry.getKey().isAssignableFrom(clazz)) { + TypeDescriptor matchedDescriptor = entry.getValue(); + // Cache the result in the ConcurrentHashMap to turn subsequent subclass lookups into O(1) + DESCRIPTORS_BY_CLASS.putIfAbsent(clazz, matchedDescriptor); + return matchedDescriptor; + } + } + return null; + } +} diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/TypeDescriptor.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/TypeDescriptor.java new file mode 100644 index 000000000000..56d161b11b6d --- /dev/null +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/TypeDescriptor.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc; + +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import java.time.ZoneId; +import java.util.List; + +/** + * Defines the bidirectional mapping between a JDBC SQL type, a default Java class, and a BigQuery + * StandardSQLTypeName. It also contains the coercion logic to convert raw values into the expected + * Java type. + */ +final class TypeDescriptor { + + @FunctionalInterface + interface TypeCoercer { + Object coerce(Object value, Class targetClass, ZoneId zoneId) throws BigQueryJdbcException; + } + + private final int jdbcType; + private final Class defaultJavaClass; + private final StandardSQLTypeName bqType; + private final List> supportedJavaTypes; + private final TypeCoercer coercer; + + TypeDescriptor( + int jdbcType, + Class defaultJavaClass, + StandardSQLTypeName bqType, + List> supportedJavaTypes, + TypeCoercer coercer) { + this.jdbcType = jdbcType; + this.defaultJavaClass = defaultJavaClass; + this.bqType = bqType; + this.supportedJavaTypes = supportedJavaTypes; + this.coercer = coercer; + } + + public int getJdbcType() { + return jdbcType; + } + + public Class getDefaultJavaClass() { + return defaultJavaClass; + } + + public StandardSQLTypeName getBqType() { + return bqType; + } + + public List> getSupportedJavaTypes() { + return supportedJavaTypes; + } + + public Object convert(Object value, Class targetClass, ZoneId zoneId) + throws BigQueryJdbcException { + if (value == null) { + return null; + } + return coercer.coerce(value, targetClass, zoneId); + } +} From 2dde172ec9365b528aec315a2a95e8f9058245eb Mon Sep 17 00:00:00 2001 From: Neenu Shaji Date: Fri, 7 Aug 2026 14:10:11 -0400 Subject: [PATCH 2/5] docs(bigquery-jdbc): add user guide with connection property and custom endpoint reference (#13878) b/538176465 --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- java-bigquery-jdbc/DEVELOPMENT.md | 179 ++++++ java-bigquery-jdbc/README.MD | 2 + java-bigquery-jdbc/docs/STORAGE_APIS.md | 69 +++ java-bigquery-jdbc/docs/USER_GUIDE.md | 765 ++++++++++++++++++++++++ 4 files changed, 1015 insertions(+) create mode 100644 java-bigquery-jdbc/DEVELOPMENT.md create mode 100644 java-bigquery-jdbc/docs/STORAGE_APIS.md create mode 100644 java-bigquery-jdbc/docs/USER_GUIDE.md diff --git a/java-bigquery-jdbc/DEVELOPMENT.md b/java-bigquery-jdbc/DEVELOPMENT.md new file mode 100644 index 000000000000..aa4dae950473 --- /dev/null +++ b/java-bigquery-jdbc/DEVELOPMENT.md @@ -0,0 +1,179 @@ +# BigQuery JDBC Developer & Contributor Guide + +This guide details the architectural design, core abstractions, coding principles, and testing workflows for developers contributing to the `google-cloud-bigquery-jdbc` module. + +--- + +## Table of Contents +1. [Core Architecture & Component Map](#1-core-architecture--component-map) +2. [Developer Guardrails & Rules of Engagement](#2-developer-guardrails--rules-of-engagement) +3. [Build & Test Playbook](#3-build--test-playbook) + - [Local Build Commands](#local-build-commands) + - [Running Unit Tests](#running-unit-tests) + - [Running Integration Tests](#running-integration-tests) + - [Dockerized Execution](#dockerized-execution) +4. [Logging Architecture & Developer Conventions](#4-logging-architecture--developer-conventions) + - [Instantiating Loggers](#instantiating-loggers) + - [Developer Logging Rules & Conventions](#developer-logging-rules--conventions) +5. [Pre-PR Checklist](#5-pre-pr-checklist) + +--- + +## 1. Core Architecture & Component Map + +The driver is structured to provide high performance, zero-allocation MDC log tracing, strict JDBC compliance, and seamless execution over the Google Cloud BigQuery REST and Storage APIs. + +```mermaid +graph TD + Client[Client Application / BI Tool] -->|DriverManager.getConnection| Driver[BigQueryDriver] + Driver -->|Parses URI & Options| UrlUtil[BigQueryJdbcUrlUtility] + Driver -->|Configures Logging| RootLogger[BigQueryJdbcRootLogger] + Driver -->|Creates| Conn[BigQueryConnection] + Conn -->|Dynamic Context Proxy| Proxy[BigQueryJdbcContextProxy] + Proxy -->|MDC Tracing| Mdc[BigQueryJdbcMdc] + Proxy -->|Delegates Exec| DirectConn[Client Session] + Conn -->|Type Mapping & Coercion| Coercion[BigQueryJdbcTypeMappings & BigQueryCoercion] + Conn -->|REST / Storage API| BQSDK[google-cloud-bigquery] +``` + +### Key Abstractions + +- **`BigQueryDriver`**: JDBC entry point registered with `java.sql.DriverManager`. Intercepts `jdbc:bigquery://` URLs, initializes early logger state, and instantiates `BigQueryConnection`. +- **`BigQueryConnection`**: Represents an active BigQuery session, holding dataset defaults, connection configuration maps, and transaction/session state (`EnableSession=true`, `session_id`). +- **`BigQueryJdbcUrlUtility`**: Parses and validates connection string parameters using a bounded LRU parse cache (`PARSE_CACHE`) to avoid heavy allocations during frequent connection creation. +- **`BigQueryJdbcContextProxy`**: A dynamic proxy layer (`java.lang.reflect.Proxy`) wrapping JDBC statements, connections, and metadata. Intercepts calls to propagate ThreadLocal MDC parameters (`connectionId`) across execution threads and enforce state validation (`checkClosed()`). +- **`BigQueryJdbcTypeMappings` & `BigQueryCoercion`**: Centralized mapping logic handling standard JDBC-to-BigQuery SQL type mappings (`StandardSQLTypeName`) and object coercions (`Date`, `Timestamp`, `BigDecimal`, etc.). +- **`BigQueryArrowResultSet`**: Custom result set implementation accelerating large query result retrieval via the BigQuery Storage Read API gRPC stream. + +--- + +## 2. Developer Guardrails & Rules of Engagement + +> [!IMPORTANT] +> **Adhere strictly to the following guardrails when making code changes:** + +1. **Visibility Principle**: Always default to the most restrictive access level (`private`, package-private, or `@InternalApi`). Do **NOT** expose classes or methods as `public` unless strictly required by standard JDBC interfaces. +2. **Explicit Class Imports**: Always write explicit `import` statements. Do **NOT** use wildcard star imports or inline fully qualified class names (e.g., use `import java.math.BigDecimal;` instead of `java.math.BigDecimal` inline). +3. **Logger Preference**: Always prefer `BigQueryJdbcCustomLogger` over `java.util.logging.Logger`. Format strings using `String.format(...)` before logging, as `BigQueryJdbcRootLogger` evaluates `record.getMessage()` directly. +4. **Exception Handling**: Always throw exceptions from the `com.google.cloud.bigquery.exception` package (`BigQueryJdbcException`, `BigQueryJdbcSqlSyntaxErrorException`, `BigQueryConversionException`). +5. **No Mocking of Final JDK Classes**: Do **NOT** mock final JDK types (`BigDecimal`, `LocalDate`, `Instant`, `UUID`) with Mockito. Mocking final JDK classes is unstable and can cause JVM crashes under JDK 21+. Always construct real instances in unit tests. + +--- + +## 3. Build & Test Playbook + +Builds and test tasks are managed via the module [Makefile](Makefile). + +### Local Build Commands + +```bash +# Build & install module locally +make install + +# Clean project target directory +make clean + +# Format code and check linter compliance +make lint +``` + +### Running Unit Tests + +```bash +# Run all unit tests +make unittest + +# Run a specific unit test class +make unittest test=BigQueryPreparedStatementTest + +# Run a specific unit test method +make unittest test=BigQueryPreparedStatementTest#testSetObjectWithTemporalTypes +``` + +### Running Integration Tests + +> [!WARNING] +> Integration tests connect to real GCP BigQuery resources and require valid GCP credentials. + +```bash +# Set GCP service account credentials +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account-key.json + +# Run a specific integration test +make integration-test test=ITBigQueryJDBCTest#testValidServiceAccountAuthenticationOAuthPvtKey +``` + +### Dockerized Execution + +If local Java/Maven environments are not available, use the dockerized environment: + +```bash +# Start an interactive shell session inside Docker container +make docker-session + +# Run unit tests inside Docker +make docker-unittest +``` + +--- + +## 4. Logging Architecture & Developer Conventions + +The driver uses a custom logging subsystem built on top of `java.util.logging`: `BigQueryJdbcCustomLogger` and `BigQueryJdbcRootLogger`. + +### Instantiating Loggers + +- **For Instance Components** (`BigQueryConnection`, `BigQueryStatement`, `BigQueryDatabaseMetaData`): + Use `this.toString()` to include instance identity in logger output: + ```java + private final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger(this.toString()); + ``` +- **For Static / Utility Components** (`BigQueryJdbcUrlUtility`, `BigQueryJdbcTypeMappings`): + Use the class name: + ```java + private static final BigQueryJdbcCustomLogger LOG = + new BigQueryJdbcCustomLogger(BigQueryJdbcTypeMappings.class.getName()); + ``` + +### Developer Logging Rules & Conventions + +1. **Method Entry / Exit Tracing**: + Methods at `FINER` level must log entrance and exit points: + ```java + public ResultSet executeQuery(String sql) throws SQLException { + LOG.finer("++enter++"); + try { + // ... execution logic ... + return rs; + } finally { + LOG.finer("++exit++"); + } + } + ``` +2. **Format Placeholders (Zero Allocation)**: + Avoid string concatenation in log calls. Use formatting placeholders or `Supplier` lambdas to prevent unneeded string allocation when the log level is disabled: + ```java + // Recommended: Use printf-style formatting + LOG.fine("Executing query on dataset: %s, table: %s", datasetId, tableId); + + // Recommended: Use supplier lambda for expensive calculations + LOG.fine(() -> "Parsed properties: " + complexObject.toDebugString()); + ``` +3. **Caller Inference & MDC Propagation**: + - `BigQueryJdbcCustomLogger` automatically wraps log records in `BigQueryJdbcLogRecord`, which inspects the stack trace to accurately infer caller class and method names. + - `BigQueryJdbcMdc` maintains `connectionId` in a `ThreadLocal` context. When logging from proxy or worker threads, always ensure MDC context is preserved or propagated via `BigQueryJdbcContextProxy`. + +--- + +## 5. Pre-PR Checklist + +Before submitting a Pull Request: + +- [ ] All new classes and methods use the narrowest possible visibility scope (`private` or package-private). +- [ ] No inline fully qualified names or wildcard star imports are present. +- [ ] All logger instances use `BigQueryJdbcCustomLogger`. +- [ ] Method entrance/exit logging (`++enter++` / `++exit++`) is included for complex internal routines. +- [ ] All method changes and feature additions are covered by corresponding JUnit 5 tests. +- [ ] Unit tests pass cleanly without Mockito `UnnecessaryStubbingException` warnings. +- [ ] All `Statement`, `ResultSet`, or `DatabaseMetaData` objects returned by public entry points are properly wrapped via `BigQueryJdbcContextProxy.wrap()`. +- [ ] Code formatting and linting pass via `make lint`. diff --git a/java-bigquery-jdbc/README.MD b/java-bigquery-jdbc/README.MD index 834e47a24b99..3b2d085ed91c 100644 --- a/java-bigquery-jdbc/README.MD +++ b/java-bigquery-jdbc/README.MD @@ -7,6 +7,8 @@ Java idiomatic client for [BigQuery JDBC][product-docs]. - [Product Documentation][product-docs] - [Client Library Documentation][javadocs] +- [Driver User Guide](docs/USER_GUIDE.md) +- [Storage APIs Deep-Dive Guide](docs/STORAGE_APIS.md) ## Quickstart diff --git a/java-bigquery-jdbc/docs/STORAGE_APIS.md b/java-bigquery-jdbc/docs/STORAGE_APIS.md new file mode 100644 index 000000000000..1c5fbf8dd12b --- /dev/null +++ b/java-bigquery-jdbc/docs/STORAGE_APIS.md @@ -0,0 +1,69 @@ +# BigQuery Storage APIs Deep Dive & Tuning Guide + +This document provides architectural details, property matrices, activation criteria, and workload tuning scenarios for the **BigQuery Storage Read API** and **BigQuery Storage Write API** integrated into the BigQuery JDBC Driver. + +--- + +## 1. High-Throughput Storage Read API (HTAPI) + +The Storage Read API streams query result sets over high-speed gRPC channels using Apache Arrow format, bypassing standard REST JSON serialization for large datasets. + +### Property Reference + +| Property Name | Connection Parameter | Default Value | Functional Role | +| :--- | :--- | :---: | :--- | +| **`EnableHighThroughputAPI`** | `EnableHighThroughputAPI=true` | `false` | **Master Toggle**: Must be `true` to enable Read API evaluation. | +| **`HighThroughputMinTableSize`** | `HighThroughputMinTableSize=10000` | `10000` | **Minimum Row Threshold**: Minimum total rows (`totalRows`) required. | +| **`HighThroughputActivationRatio`** | `HighThroughputActivationRatio=2` | `2` | **Page Ratio Threshold**: `totalRows / MaxResults` ratio required. | +| **`MaxResults`** | `MaxResults=10000` | `10000` | **Page Size**: Controls rows per page in standard REST calls. | + +### Activation Criteria & Fallback Mechanics + +When `EnableHighThroughputAPI=true` is set, the driver transparently switches to the Storage Read API if all of the following conditions are met: + +1. **Master Toggle**: `EnableHighThroughputAPI=true` is set. +2. **Minimum Row Threshold**: The query returns at least `HighThroughputMinTableSize` rows (default: $\ge 10,000$ rows). +3. **Multiple Response Pages**: The result set spans more than one page (total rows exceed `MaxResults`). If all rows fit on page 1, standard REST is used to avoid unnecessary gRPC stream setup. +4. **Activation Ratio Test**: The ratio of total rows to page size ($\frac{\text{totalRows}}{\text{MaxResults}}$) exceeds `HighThroughputActivationRatio` (default: $> 2$). + +> [!NOTE] +> **Automatic Permission Fallback**: If `EnableHighThroughputAPI=true` is set but the connecting principal lacks the `BigQuery Read Session User` IAM role, the driver catches the `PERMISSION_DENIED` status and automatically falls back to standard REST JSON pagination. + +### Workload Scenarios Matrix + +| Workload Scenario | `EnableHighThroughputAPI` | `HighThroughputMinTableSize` | `HighThroughputActivationRatio` | `MaxResults` | Execution Mechanism | Use Case | +| :--- | :---: | :---: | :---: | :---: | :--- | :--- | +| **Standard REST (Default)** | `false` | `10000` (ignored) | `2` (ignored) | `10000` | REST JSON Pagination | Small/medium queries; standard REST security policies. | +| **Default Production Extractions** | `true` | `10000` | `2` | `10000` | gRPC Storage Read API (for results $> 20,000$ rows) | Standard analytical reports and ETL extracts. | +| **Aggressive Streaming** | `true` | `100` | `0` | `50` | gRPC Storage Read API (for results $\ge 100$ rows) | High-speed streaming for smaller analytical datasets. | +| **Bulk ETL Analytics** | `true` | `50000` | `5` | `10000` | gRPC Storage Read API (for results $> 50,000$ rows) | Large multi-gigabyte dataset extractions. | + +--- + +## 2. Storage Write API (SWA) + +The Storage Write API streams high-throughput bulk insertions over gRPC channels for `PreparedStatement.executeBatch()` calls. + +### Property Reference + +| Property Name | Connection Parameter | Default Value | Functional Role | +| :--- | :--- | :---: | :--- | +| **`EnableWriteAPI`** | `EnableWriteAPI=true` | `false` | **Master Toggle**: Must be `true` to enable Storage Write API streaming. | +| **`SWA_ActivationRowCount`** | `SWA_ActivationRowCount=3` | `3` | **Activation Threshold**: Minimum batch size added via `addBatch()` required to trigger SWA. | +| **`SWA_AppendRowCount`** | `SWA_AppendRowCount=1000` | `1000` | **Chunk Size**: Maximum rows per gRPC append payload before flushing. | + +### Activation Criteria & Fallback Mechanics + +When `EnableWriteAPI=true` is set, the driver evaluates the batch size during `PreparedStatement.executeBatch()`: + +- **At or Above Threshold ($\ge \text{SWA\_ActivationRowCount}$)**: The driver opens a gRPC Storage Write stream and appends batch records in payload chunks governed by `SWA_AppendRowCount`. +- **Below Threshold ($< \text{SWA\_ActivationRowCount}$)**: The driver uses standard SQL DML (`INSERT INTO ...`) to avoid gRPC stream overhead for tiny batches. + +### Workload Scenarios Matrix + +| Workload Scenario | `EnableWriteAPI` | `SWA_ActivationRowCount` | `SWA_AppendRowCount` | Execution Mechanism | Use Case | +| :--- | :---: | :---: | :---: | :--- | :--- | +| **Standard SQL DML (Default)** | `false` | `3` (ignored) | `1000` (ignored) | Concatenated REST SQL DML | Small transactional DML; standard SQL compatibility. | +| **Default High-Throughput ETL** | `true` | `3` | `1000` | gRPC SWA stream (batches $\ge 3$, flushes per 1,000 rows) | Standard batch loader applications (Spring Batch, Spark). | +| **Real-Time Micro-Batching** | `true` | `1` | `100` | gRPC SWA stream (batches $\ge 1$, flushes per 100 rows) | High-frequency streaming events (Kafka/Flink consumers). | +| **High-Volume Bulk Ingestion** | `true` | `100` | `5000` | gRPC SWA stream (batches $\ge 100$, flushes per 5,000 rows) | Large nightly bulk ETL loading millions of records. | diff --git a/java-bigquery-jdbc/docs/USER_GUIDE.md b/java-bigquery-jdbc/docs/USER_GUIDE.md new file mode 100644 index 000000000000..0f38e0fe252e --- /dev/null +++ b/java-bigquery-jdbc/docs/USER_GUIDE.md @@ -0,0 +1,765 @@ +# Google BigQuery JDBC Driver User Guide + +This guide provides comprehensive instructions for configuring, developing with, and optimizing the **Google BigQuery JDBC Driver** (`google-cloud-bigquery-jdbc`). + +> [!NOTE] +> This guide is aligned with and references the official [Google Cloud BigQuery JDBC Documentation](https://cloud.google.com/bigquery/docs/jdbc-for-bigquery) and [Running Queries with the JDBC Driver](https://cloud.google.com/bigquery/docs/jdbc-for-bigquery#run_queries_with_the_driver). + +--- + +## Table of Contents +1. [Overview & Prerequisites](#1-overview--prerequisites) +2. [Connection URL Syntax & Quickstart](#2-connection-url-syntax--quickstart) +3. [Authentication Modes & Configuration](#3-authentication-modes--configuration) +4. [Connection Properties Reference](#4-connection-properties-reference) +5. [Data Type Mapping Reference](#5-data-type-mapping-reference) +6. [JDBC Driver Architecture & Core Features](#6-jdbc-driver-architecture--core-features) + - [Transaction Management & Multi-Statement Sessions](#transaction-management--multi-statement-sessions) + - [High-Throughput Storage Read & Write APIs](#high-throughput-storage-read--write-apis) +7. [Feature Examples & Code Snippets](#7-feature-examples--code-snippets) + - [Transactions (Manual Commit & Rollback)](#transactions-manual-commit--rollback) + - [Prepared Statements & Parameter Binding](#prepared-statements--parameter-binding) + - [Callable Statements & Stored Procedures](#callable-statements--stored-procedures) + - [Batch Ingestion with Storage Write API](#batch-ingestion-with-storage-write-api) + - [High-Throughput Storage Read API](#high-throughput-storage-read-api) + - [Struct & Array Column Operations](#struct--array-column-operations) + - [Service Account Impersonation](#service-account-impersonation) +8. [High-Throughput & Performance Tuning](#8-high-throughput--performance-tuning) +9. [Framework Integration & Deployment](#9-framework-integration--deployment) + - [Spring Boot (application.yml)](#spring-boot-applicationyml) + - [Key Deployment Guidelines](#key-deployment-guidelines) +10. [Logging, Diagnostics & Troubleshooting](#10-logging-diagnostics--troubleshooting) +11. [Official Documentation References](#11-official-documentation-references) + +--- + +## 1. Overview & Prerequisites + +The BigQuery JDBC Driver enables Java applications, BI tools, and ETL pipelines to interact with Google Cloud BigQuery using the standard Java Database Connectivity (JDBC) API (JDBC 4.2 compliant). + +### Prerequisites +- **Java Runtime**: JDK 8 or higher (JDK 11, 17, or 21 recommended). +- **Google Cloud Platform**: + - An active GCP Project with BigQuery API enabled. + - Required IAM roles (e.g., `BigQuery Data Viewer`, `BigQuery Job User`). + +### Installation Coordinates + +**Maven**: +```xml + + com.google.cloud + google-cloud-bigquery-jdbc + 1.1.0 + +``` + +**Gradle**: +```groovy +implementation 'com.google.cloud:google-cloud-bigquery-jdbc:1.1.0' +``` + +--- + +## 2. Connection URL Syntax & Quickstart + +The JDBC connection string format for BigQuery is: + +``` +jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=;[DefaultDataset=;][Property1=Value1;...] +``` + +### Basic Quickstart Example + +```java +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +public class BigQueryQuickstart { + public static void main(String[] args) throws Exception { + String url = "jdbc:bigquery://https://bigquery.googleapis.com:443" + + ";ProjectId=my-gcp-project-id" + + ";DefaultDataset=my_dataset" + + ";OAuthType=3"; // 3 = Application Default Credentials (ADC) + + try (Connection conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT name, age FROM `my-gcp-project-id.my_dataset.users` LIMIT 10")) { + + while (rs.next()) { + System.out.printf("User: %s, Age: %d%n", rs.getString("name"), rs.getInt("age")); + } + } + } +} +``` + +--- + +## 3. Authentication Modes & Configuration + +The driver supports multiple OAuth 2.0 and identity workflows specified via the `OAuthType` connection property. + +| `OAuthType` | Authentication Strategy / Variant | Required Properties | Supported / Optional Properties | Notes | +| :---: | :--- | :--- | :--- | :--- | +| **`0`** | **Service Account Key File (JSON)** | `OAuthPvtKeyPath` | `OAuthServiceAcctEmail` | Path to service account `.json` key file. Service account email and private key are automatically extracted from the file. | +| **`0`** | **Service Account Key File (P12)** | `OAuthPvtKeyPath`, `OAuthServiceAcctEmail` | `OAuthP12Password` | Path to `.p12` key file. Requires explicit `OAuthServiceAcctEmail` as `.p12` files do not contain email metadata. `OAuthP12Password` defaults to `notasecret`. | +| **`0`** | **Service Account Key String (JSON)** | `OAuthPvtKey` | `OAuthServiceAcctEmail` | Raw JSON key file string content in URL. Service account email and private key are automatically extracted from the JSON string. | +| **`0`** | **Service Account Key String (PKCS#8 / P12 Bytes)** | `OAuthPvtKey`, `OAuthServiceAcctEmail` | `OAuthP12Password` | Raw PKCS#8 or P12 bytes key string content in URL. Requires explicit `OAuthServiceAcctEmail`. | +| **`1`** | **User Account (Interactive 3-Legged OAuth)** | None (for built-in client app) | `OAuthClientId`, `OAuthClientSecret` | Launches local browser tab for interactive user login on desktop. Built-in client ID/secret used if omitted. | +| **`2`** | **Pre-generated Access Token** | `OAuthAccessToken` | `OAuthAccessTokenReadonly` | Uses short-lived bearer token directly. `OAuthAccessTokenReadonly` defaults to `false`. | +| **`2`** | **Pre-generated Refresh Token** | `OAuthRefreshToken`, `OAuthClientId`, `OAuthClientSecret` | None | Uses refresh token with client credentials to automatically mint access tokens. | +| **`3`** | **Application Default Credentials (ADC)** | None | None | Uses GCP standard credential resolution (`GOOGLE_APPLICATION_CREDENTIALS` env var or `gcloud` CLI context). | +| **`4`** | **Workload / Workforce Identity Federation (BYOID File)** | `OAuthPvtKeyPath` | None | Path to external account credential JSON configuration file. | +| **`4`** | **Workload / Workforce Identity Federation (BYOID Content)** | `OAuthPvtKey` | None | Raw JSON string content of external account credentials. | +| **`4`** | **Workload / Workforce Identity Federation (BYOID Properties)** | `BYOID_AudienceUri`, `BYOID_CredentialSource`, `BYOID_SubjectTokenType` | `BYOID_PoolUserProject`, `BYOID_SA_Impersonation_Uri`, `BYOID_TokenUri` | Inline BYOID connection properties configured directly in connection URL string. | + +> [!NOTE] +> **Global Authentication Properties**: +> The following properties are supported across **all** `OAuthType` authentication modes: +> - `universeDomain`: Custom universe domain name for Google Cloud Dedicated (GCD) or non-default Google Cloud environments (defaults to `googleapis.com`). +> - `ServiceAccountImpersonationEmail`, `ServiceAccountImpersonationChain`, `ServiceAccountImpersonationScopes`, `ServiceAccountImpersonationTokenLifetime`: Service account impersonation properties. + +### Service Account Authentication Example +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443" + + ";ProjectId=my-gcp-project-id" + + ";OAuthType=0" + + ";OAuthPvtKeyPath=/path/to/service-account-key.json"; +``` + +### Service Account Impersonation +To execute queries as an impersonated service account: +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443" + + ";ProjectId=my-gcp-project-id" + + ";OAuthType=3" + + ";ServiceAccountImpersonationEmail=target-sa@my-gcp-project.iam.gserviceaccount.com"; +``` + +--- + +## 4. Connection Properties Reference + +> [!TIP] +> **Boolean Connection Properties**: +> For boolean connection properties (such as `OAuthAccessTokenReadonly`, `RequestGoogleDriveScope`, `EnableHighThroughputAPI`, etc.), prefer using boolean values (`true` / `false`) when possible (e.g., `;OAuthAccessTokenReadonly=true`). Numeric representation (`1` / `0`) is also accepted for backwards compatibility. + +### Authentication & Impersonation Properties + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `OAuthType` | **Required** (Default: `-1`) | **Required.** Specifies the authentication mechanism: `0` (Service Account), `1` (User Account), `2` (Pre-generated Token / Refresh Token), `3` (ADC), `4` (Workload & Workforce Identity Federation / BYOID). | +| `OAuthServiceAcctEmail` | `null` | Service Account email address for Service Account authentication or token scope. | +| `OAuthPvtKeyPath` | `null` | Path to JSON or P12 private key file for Service Account authentication. | +| `OAuthPvtKey` | `null` | Raw PKCS#8 private key string content for Service Account authentication. | +| `OAuthP12Password` | `notasecret` | Password for accessing encrypted `.p12` key files. | +| `OAuthAccessToken` | `null` | Pre-generated OAuth 2.0 access token string. | +| `OAuthAccessTokenReadonly` | `false` | Indicates whether the pre-generated access token has read-only scope. | +| `OAuthRefreshToken` | `null` | Pre-generated refresh token to automatically mint access tokens. | +| `OAuthClientId` | *Google Client ID* | OAuth 2.0 Client ID for user authentication or refresh token flows. | +| `OAuthClientSecret` | *Google Secret* | OAuth 2.0 Client Secret for user authentication or refresh token flows. | +| `ServiceAccountImpersonationEmail` | `null` | Target Service Account email to impersonate for query job execution. | +| `ServiceAccountImpersonationChain` | `null` | Comma-separated list of service account emails representing the impersonation chain. | +| `ServiceAccountImpersonationScopes` | `https://www.googleapis.com/auth/bigquery` | Comma-separated OAuth 2.0 scopes for impersonated credentials. | +| `ServiceAccountImpersonationTokenLifetime` | `3600` | Lifetime (in seconds) for impersonated service account tokens. | +| `RequestGoogleDriveScope` | `0` | If `1` (or `true`), appends the Google Drive read-only scope (`https://www.googleapis.com/auth/drive.readonly`) to query external Drive tables. | + +### Workload & Workforce Identity Federation (BYOID) Properties + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `BYOID_AudienceUri` | `null` | Audience URI corresponding to the Workforce/Workload identity pool provider. | +| `BYOID_CredentialSource` | `null` | File path or JSON content defining the external subject token source. | +| `BYOID_PoolUserProject` | `null` | Google Cloud project number associated with the workforce pool. | +| `BYOID_SA_Impersonation_Uri` | `null` | Service Account impersonation URL for Workload Identity Federation. | +| `BYOID_SubjectTokenType` | `urn:ietf:params:oauth:tokentype:id_token` | Type of subject token provided (e.g., `urn:ietf:params:oauth:tokentype:jwt`). | +| `BYOID_TokenUri` | `https://sts.googleapis.com/v1/token` | Security Token Service (STS) token exchange endpoint URI. | + +### Core Query & Catalog Properties + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `ProjectId` | *Default Project* | Google Cloud Project ID for billing and query execution. | +| `DefaultDataset` | `null` | Default dataset scope for unqualified table references in SQL queries and metadata catalog filtering (when `FilterTablesOnDefaultDataset=true`). | +| `AdditionalProjects` | `null` | Comma-separated list of additional project IDs accessible for catalog discovery and querying. | +| `EnableProjectDiscovery` | `false` | Automatically discovers all accessible Google Cloud projects as catalog entries when `true`. | +| `FilterTablesOnDefaultDataset` | `false` | When `true`, restricts `DatabaseMetaData` catalog calls (such as `getTables()` and `getColumns()`) to `DefaultDataset` whenever catalog/schema parameters are omitted or set to wildcard (`%`), preventing full project scans. | +| `Location` | *Auto-detected* | GCP location/region for dataset and job execution (e.g., `US`, `EU`, `asia-east1`). | +| `QueryDialect` | `SQL` | Query dialect: `SQL` (Standard SQL) or `LEGACY_SQL`. | +| `UseQueryCache` | `true` | Enables or disables BigQuery query result caching. | +| `MaximumBytesBilled` | `0` (unlimited) | Limits bytes billed per query; query fails without cost if estimated bytes exceed this limit. | +| `Labels` | `null` | Comma-separated `key=value` pairs attached to query jobs (e.g., `env=prod,dept=analytics`). | +| `QueryProperties` | `null` | Connection-level query configuration properties passed to BigQuery job execution. | +| `JobCreationMode` | `2` | Job creation strategy: `1` (`JOB_CREATION_REQUIRED` - forces explicit query job creation for every query) or `2` (`JOB_CREATION_OPTIONAL` - default, allows queries to execute directly without creating explicit jobs when possible). | +| `MaxResults` | `10000` | Maximum number of rows returned per page during standard REST result set iteration. | +| `AllowLargeResults` | `true` | Allows large query results (required when using legacy SQL destination tables). | +| `LargeResultTable` | `null` | Destination table name for query result sets. If omitted when `LargeResultDataset` is set, the driver automatically generates a temporary table name (`_jdbc_tmp_`). | +| `LargeResultDataset` | `null` | Destination dataset name for query result sets. If the specified dataset does not exist, the driver automatically creates it. Under Legacy SQL mode, defaults to creating/using `_jdbc_tmp`. | +| `LargeResultsDatasetExpirationTime` | `3600000` (1 hour) | Expiration time (in milliseconds) for temporary destination tables in user-specified datasets. | +| `KMSKeyName` | `null` | Cloud KMS key resource name used for encrypting query results and destination tables. | + +### Session & Transaction Properties + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `EnableSession` | `false` | Enables multi-statement session creation and transaction support (`BEGIN`, `COMMIT`, `ROLLBACK`). | + +### High-Throughput Storage & Write API Properties + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `EnableHighThroughputAPI` | `false` | Enables BigQuery Storage Read API (gRPC/Arrow) for faster result retrieval. Requires `BigQuery Read Session User` (`roles/bigquery.readSessionUser`) IAM permission. | +| `HighThroughputMinTableSize` | `10000` | Minimum query result row count threshold required to trigger the Storage Read API. | +| `HighThroughputActivationRatio` | `2` | Minimum number of result pages required before switching to the Storage Read API. | +| `UnsupportedHTAPIFallback` | `true` | Automatically falls back to standard REST API when Storage Read API is unsupported or lacks permissions. | +| `EnableWriteAPI` | `false` | Enables BigQuery Storage Write API for high-performance batch insert streams. | +| `SWA_ActivationRowCount` | `3` | Minimum row threshold in `executeBatch()` to activate Storage Write API streaming. | +| `SWA_AppendRowCount` | `1000` | Batch row size per append stream request when using Storage Write API. | + +### Network, Proxy & Endpoint Overrides + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `ProxyHost` | `null` | Hostname or IP address of the HTTP/HTTPS proxy server. | +| `ProxyPort` | `null` | Port number of the proxy server. | +| `ProxyUid` | `null` | Username for proxy server authentication. | +| `ProxyPwd` | `null` | Password for proxy server authentication. | +| `EndpointOverrides` | `null` | Semicolon or comma-separated list of custom service endpoint overrides. Supported keys: `BIGQUERY`, `READ_API`, `OAUTH2`, `STS`, `ACCOUNTS`. | +| `PrivateServiceConnectUris` | `null` | Alias for `EndpointOverrides`. Accepts the same custom service endpoint override format and supported keys. | +| `universeDomain` | `googleapis.com` | Domain name for Google Cloud Dedicated (GCD) or custom Google universe instances. | +| `SSLTrustStore` | `null` | Path to custom Java TrustStore file containing trusted server certificates for SSL. | +| `SSLTrustStorePwd` | `null` | Password for accessing the custom Java TrustStore file. | +| `SSLTrustStoreType` | *System Default* (`JKS`/`PKCS12`) | Type of the custom Java TrustStore file. | +| `SSLTrustStoreProvider` | `null` | Security provider name for the custom Java TrustStore. | +| `HttpConnectTimeout` | `0` (system default) | HTTP socket connection timeout in milliseconds. | +| `HttpReadTimeout` | `0` (system default) | HTTP socket read timeout in milliseconds. | + +### Driver Logging, Retries & Concurrency Tuning + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `LogLevel` | `0` (OFF) | Logging verbosity level (`0` = OFF to `7` = FINEST). Controls internal `java.util.logging` detail. See [Logging, Diagnostics & Troubleshooting](#10-logging-diagnostics--troubleshooting) for level definitions and details. | +| `LogPath` | `""` | Directory path where log files are written when logging is enabled. | +| `Timeout` | `0` (unlimited) | Length of time (in seconds) the connector retries failed API calls before timing out. | +| `JobTimeout` | `0` (unlimited) | Job execution timeout (in seconds) after which BigQuery cancels the query job server-side. | +| `RetryInitialDelay` | `0` | Initial delay (in seconds) before executing the first retry attempt. | +| `RetryMaxDelay` | `0` | Maximum delay limit (in seconds) between retry attempts. | +| `MetaDataFetchThreadCount` | `32` | Thread pool size used to parallelize `DatabaseMetaData` catalog RPC calls. | +| `ConnectionPoolSize` | `10` | Maximum size of the internal connection pool when connection pooling is enabled. | +| `ListenerPoolSize` | `10` | Maximum size of the listener thread pool when connection pooling is enabled. | +| `RequestReason` | `null` | Reason string passed in the `x-goog-request-reason` HTTP header for auditing. | + +### OpenTelemetry & Cloud Observability Exporters + +| Property Name | Default Value | Description | +| :--- | :---: | :--- | +| `enableGcpTraceExporter` | `false` | Enables direct exporting of driver OpenTelemetry trace spans to Google Cloud Trace. | +| `enableGcpLogExporter` | `false` | Enables direct exporting of driver OpenTelemetry logs to Google Cloud Logging. | +| `gcpTelemetryProjectId` | `null` | GCP Project ID target for OpenTelemetry log and trace telemetry export. Defaults to query `ProjectId` if omitted. | +| `gcpTelemetryCredentials` | `null` | File path or raw JSON credentials string for OpenTelemetry GCP exporters. | +| `useGlobalOpenTelemetry` | `false` | Instructs the driver to register with the global OpenTelemetry instance (`GlobalOpenTelemetry`) in the JVM environment. | + +--- + +## 5. Data Type Mapping Reference + +When running queries through the JDBC driver for BigQuery, data types map as specified in the official [BigQuery JDBC Data Type Mapping](https://cloud.google.com/bigquery/docs/jdbc-for-bigquery#run_queries_with_the_driver): + +| BigQuery SQL Type | Java / JDBC Type | Recommended Getter / Setter | +| :--- | :--- | :--- | +| `ARRAY` | `java.sql.Array` | `rs.getArray(col)` | +| `BIGNUMERIC` | `java.math.BigDecimal` | `rs.getBigDecimal(col)` | +| `BOOL` | `java.lang.Boolean` | `rs.getBoolean(col)` | +| `BYTES` | `byte[]` | `rs.getBytes(col)` | +| `DATE` | `java.sql.Date` / `java.time.LocalDate` | `rs.getDate(col)` or `rs.getObject(col, LocalDate.class)` | +| `DATETIME` | `java.sql.Timestamp` / `java.time.LocalDateTime` | `rs.getTimestamp(col)` or `rs.getObject(col, LocalDateTime.class)` | +| `FLOAT64` | `java.lang.Double` | `rs.getDouble(col)` | +| `GEOGRAPHY` | `java.lang.String` | `rs.getString(col)` | +| `INT64` | `java.lang.Long` | `rs.getLong(col)` | +| `INTERVAL` | `java.lang.String` | `rs.getString(col)` | +| `JSON` | `java.lang.String` | `rs.getString(col)` | +| `NUMERIC` | `java.math.BigDecimal` | `rs.getBigDecimal(col)` | +| `STRING` | `java.lang.String` | `rs.getString(col)` | +| `STRUCT` | `java.sql.Struct` | `rs.getObject(col)` (casts to `java.sql.Struct`) | +| `TIME` | `java.sql.Time` / `java.time.LocalTime` | `rs.getTime(col)` or `rs.getObject(col, LocalTime.class)` | +| `TIMESTAMP` | `java.sql.Timestamp` / `java.time.Instant` | `rs.getTimestamp(col)` or `rs.getObject(col, Instant.class)` | + +--- + +## 6. JDBC Driver Architecture & Core Features + +### Transaction Management & Multi-Statement Sessions + +BigQuery supports **Multi-Statement Transactions** across tables using standard SQL primitives (`BEGIN TRANSACTION`, `COMMIT TRANSACTION`, `ROLLBACK TRANSACTION`). The driver bridges standard JDBC methods (`setAutoCommit`, `commit`, `rollback`) directly to BigQuery's underlying session engine. + +#### Session Lifecycle Flow: + +``` +[DriverManager.getConnection()] + │ + (EnableSession=true) + │ + ┌──────────▼──────────┐ + │ setAutoCommit(false)│ ──────► Begins transaction block in session + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ Execute DML & SQL │ ──────► Runs queries within active session + │ Statements │ + └──────────┬──────────┘ + │ + ┌───────┴───────┐ + │ │ + ▼ ▼ +┌─────────┐ ┌──────────┐ +│commit() │ │rollback()│ +└────┬────┘ └────┬─────┘ + │ │ + ▼ ▼ +Executes: Executes: +COMMIT ROLLBACK +TRANSACTION; TRANSACTION; + │ │ + └───────┬───────┘ + │ + ▼ +(Auto-re-executes BEGIN TRANSACTION; if setAutoCommit remains false) +``` + +1. **Pre-requisite Check**: Calling `setAutoCommit(false)`, `commit()`, or `rollback()` requires `;EnableSession=true` in the connection URL. If disabled or invoked without an active transaction, an exception is thrown by the driver. +2. **Session & Transaction Start**: + - `setAutoCommit(false)` initiates a multi-statement transaction session in BigQuery. +3. **Statement Propagation**: + - All `Statement` or `PreparedStatement` instances created on the connection execute within the scope of the active session. +4. **Commit & Rollback**: + - `commit()` executes `COMMIT TRANSACTION;` to commit changes. + - `rollback()` executes `ROLLBACK TRANSACTION;` to discard changes. + - If `autoCommit` remains `false`, the driver automatically starts the next transaction block. +5. **Connection Close Safety**: + - If an uncommitted transaction is pending when `conn.close()` is invoked, the driver automatically rolls back the transaction to prevent uncommitted changes from persisting. +6. **Isolation Level & Holdability**: + - Isolation level: `Connection.TRANSACTION_SERIALIZABLE` (BigQuery multi-statement snapshot isolation). + - Holdability: `ResultSet.CLOSE_CURSORS_AT_COMMIT`. + +--- + +### High-Throughput Storage Read & Write APIs + +For enterprise data ingestion and analytics extraction, the driver integrates with BigQuery Storage APIs: + +- **Storage Read API (`EnableHighThroughputAPI=true`)**: + Performs high-speed result extraction over gRPC streams using Apache Arrow instead of REST JSON pagination for large ResultSets. +- **Storage Write API (`EnableWriteAPI=true`)**: + Enables high-throughput streaming appends for batch operations executed via `PreparedStatement.executeBatch()`. + +--- + +## 7. Feature Examples & Code Snippets + +### Transactions (Manual Commit & Rollback) +Transactions require `;EnableSession=true` in the connection URL to enable multi-statement sessions in BigQuery. + +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=my-project;EnableSession=true;OAuthType=3"; + +try (Connection conn = DriverManager.getConnection(url)) { + conn.setAutoCommit(false); // Enable manual transaction control + + try (PreparedStatement debit = conn.prepareStatement("UPDATE finance.accounts SET balance = balance - ? WHERE account_id = ?"); + PreparedStatement credit = conn.prepareStatement("UPDATE finance.accounts SET balance = balance + ? WHERE account_id = ?")) { + + debit.setDouble(1, 500.00); + debit.setLong(2, 1001L); + debit.executeUpdate(); + + credit.setDouble(1, 500.00); + credit.setLong(2, 2002L); + credit.executeUpdate(); + + conn.commit(); // Commit transaction atomically + } catch (SQLException e) { + conn.rollback(); // Rollback on error + throw e; + } +} +``` + +--- + +### Prepared Statements & Parameter Binding +Use `PreparedStatement` to safely bind parameters including primitive types, decimals (`BigDecimal`), temporal values (`Date`, `Timestamp`), and byte arrays (`byte[]`). + +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=my-project;OAuthType=3"; +String sql = "SELECT order_id FROM sales.orders WHERE order_date = ? AND total_amount >= ? AND status_code = ?"; + +try (Connection conn = DriverManager.getConnection(url); + PreparedStatement pstmt = conn.prepareStatement(sql)) { + + pstmt.setDate(1, java.sql.Date.valueOf(LocalDate.of(2026, 7, 22))); + pstmt.setBigDecimal(2, new BigDecimal("12499.99")); + pstmt.setString(3, "COMPLETED"); + + try (ResultSet rs = pstmt.executeQuery()) { + while (rs.next()) { + long orderId = rs.getLong("order_id"); + } + } +} +``` + +--- + +### Callable Statements & Stored Procedures +BigQuery stored procedure outputs are returned as standard result sets (`TableResult`) rather than JDBC `OUT` parameters. + +#### Pattern A: Executing a Stored Procedure Returning a ResultSet +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=my-project;OAuthType=3"; + +try (Connection conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("CALL my_dataset.get_top_customers()")) { + + while (rs.next()) { + String customer = rs.getString("customer_name"); + double spent = rs.getDouble("total_spent"); + } +} +``` + +#### Pattern B: Retrieving Procedure Output Variables via Multi-Statement Script +To retrieve `OUT` parameters from a procedure, execute a procedural script that declares an output variable, calls the procedure, and returns the variable in a final `SELECT` statement: + +```java +String scriptSql = "DECLARE tax_out NUMERIC; " + + "CALL my_dataset.calculate_tax(?, tax_out); " + + "SELECT tax_out AS calculated_tax;"; + +try (Connection conn = DriverManager.getConnection(url); + PreparedStatement pstmt = conn.prepareStatement(scriptSql)) { + + pstmt.setBigDecimal(1, new BigDecimal("1000.00")); + + try (ResultSet rs = pstmt.executeQuery()) { + if (rs.next()) { + BigDecimal taxAmount = rs.getBigDecimal("calculated_tax"); + } + } +} +``` + +--- + +### Batch Ingestion with Storage Write API +Enable `;EnableWriteAPI=true` in the connection URL to stream bulk batches via `executeBatch()`. + +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=my-project;EnableWriteAPI=true;OAuthType=3"; +String sql = "INSERT INTO telemetry.sensor_readings (sensor_id, temperature, is_valid) VALUES (?, ?, ?)"; + +try (Connection conn = DriverManager.getConnection(url); + PreparedStatement pstmt = conn.prepareStatement(sql)) { + + for (int i = 0; i < 1000; i++) { + pstmt.setInt(1, 100 + (i % 10)); + pstmt.setDouble(2, 25.5); + pstmt.setBoolean(3, true); + pstmt.addBatch(); + + if (i % 500 == 0) { + pstmt.executeBatch(); // Flushes batch using Storage Write API + } + } + pstmt.executeBatch(); +} +``` + +--- + +### High-Throughput Storage Read API +Enable `;EnableHighThroughputAPI=true` to stream large query result sets over high-speed gRPC streams using Apache Arrow format. + +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=my-project;EnableHighThroughputAPI=true;OAuthType=3"; + +try (Connection conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT user_id, event_name FROM analytics.user_events WHERE event_date >= '2026-01-01'")) { + + while (rs.next()) { + String userId = rs.getString("user_id"); + String event = rs.getString("event_name"); + } +} +``` + +--- + +### Struct & Array Column Operations +Extract nested BigQuery `STRUCT` columns (using `java.sql.Struct`) and `ARRAY` columns (using `java.sql.Array`). + +```java +String query = "SELECT STRUCT('123 Main St' AS street, 'Seattle' AS city) AS address, ['tag1', 'tag2'] AS tags"; + +try (Connection conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(query)) { + + if (rs.next()) { + // Read STRUCT attributes + Struct address = (Struct) rs.getObject("address"); + Object[] attrs = address.getAttributes(); // [123 Main St, Seattle] + + // Read ARRAY elements + Array tagsArray = rs.getArray("tags"); + String[] tags = (String[]) tagsArray.getArray(); // [tag1, tag2] + } +} +``` + +--- + +### Service Account Impersonation +Impersonate a service account using Application Default Credentials (ADC) without managing static service account key files. + +> [!NOTE] +> **IAM Roles Required for Impersonation**: +> 1. **Caller Principal (ADC)**: Must have the `roles/iam.serviceAccountTokenCreator` role on the target Service Account. +> 2. **Impersonated Service Account**: Must have `roles/bigquery.jobUser` (to submit query jobs) and `roles/bigquery.dataViewer` (to read target datasets). + +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443" + + ";ProjectId=my-project" + + ";OAuthType=3" // Application Default Credentials (ADC) + + ";ServiceAccountImpersonationEmail=analytics-executor@my-project.iam.gserviceaccount.com"; + +try (Connection conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM analytics.user_events")) { + + if (rs.next()) { + long count = rs.getLong(1); + } +} +``` + +--- + +## 8. High-Throughput & Performance Tuning + +For large dataset extractions and high-concurrency ingestion, tune the following connection options: + +### Storage Read & Write APIs Summary + +The BigQuery JDBC driver supports gRPC-accelerated streaming APIs for both reading large result sets and appending batch records: + +- **Storage Read API (`EnableHighThroughputAPI=true`)**: + Streams query result sets over high-speed gRPC channels using Apache Arrow format instead of standard REST JSON pagination. Automatically evaluates minimum table size and page count thresholds before activating. +- **Storage Write API (`EnableWriteAPI=true`)**: + Streams bulk batch insertions (`PreparedStatement.executeBatch()`) over gRPC append streams instead of concatenated SQL DML statements. + +> [!TIP] +> For complete property matrices, activation criteria, and workload tuning scenario matrices for both Storage Read and Write APIs, see the [Storage APIs Guide](STORAGE_APIS.md). + +--- + +### Additional Performance Tuning Options + +- **Metadata Thread Pooling**: For applications fetching schema catalog metadata across multiple datasets, set `;MetaDataFetchThreadCount=64` to parallelize catalog RPC calls. + +--- + +### Custom Endpoints, Private Service Connect (PSC) & Universe Domains + +For enterprise environments using **VPC Service Controls**, **Private Service Connect (PSC)**, **Regional Endpoints**, or **Google Cloud Dedicated (GCD)** instances, the driver supports full endpoint overriding and universe domain configuration. + +#### 1. Endpoint Overrides (`EndpointOverrides`) + +The `EndpointOverrides` property allows redirecting traffic for individual Google Cloud services to private IP endpoints, regional gateways, or corporate proxy targets. + +**Format**: +``` +EndpointOverrides=BIGQUERY=https://bigquery.googleapis.com,OAUTH2=https://oauth2.googleapis.com/token; +``` + +**Supported Service Keys**: + +| Service Key | Target Service / Auth Flow | Default Endpoint URI | When to Use & Applicable Auth Modes | +| :--- | :--- | :--- | :--- | +| **`BIGQUERY`** | BigQuery Core REST API | `https://bigquery.googleapis.com` | Overrides BigQuery REST query execution endpoints for regional gateways or PSC VIPs (**Applies to all `OAuthType` modes**). | +| **`READ_API`** | Storage API (Read & Write) | `https://bigquerystorage.googleapis.com` | Overrides gRPC stream endpoints for Storage Read (HTAPI) and Storage Write (SWA) (**Applies to all `OAuthType` modes**). | +| **`OAUTH2`** | Google OAuth 2.0 Token Server | `https://oauth2.googleapis.com/token` | Overrides OAuth token exchange endpoints for **Service Account (`OAuthType=0`)**, **User Account (`OAuthType=1`)**, and **Refresh Token (`OAuthType=2`)**. | +| **`STS`** | Security Token Service | `https://sts.googleapis.com` | Overrides Security Token Service (`token_url`) endpoints for **Workload Identity / BYOID (`OAuthType=4`)**. | + +> [!NOTE] +> **Credentials File Endpoint Resolution**: +> Workload credentials files (such as Workload Identity / External Account JSON files or Application Default Credentials) may define their own endpoint URLs (`token_url`, `credential_source`, or `universe_domain`). Connection parameters (such as `EndpointOverrides`, `BYOIDTokenUri`, or `universeDomain`) take precedence when explicitly specified in the JDBC URL, allowing applications to override values configured inside credential files for private routing or custom proxies. + +--- + +#### 2. Private Service Connect URIs (`PrivateServiceConnectUris`) + +For VPC setups using explicit Private Service Connect URI aliases: +``` +jdbc:bigquery://https://bigquery.googleapis.com:443 + ;ProjectId=my-gcp-project + ;OAuthType=0 + ;PrivateServiceConnectUris=BIGQUERY=https://bigquery-privateendpoint.p.googleapis.com +``` + +#### 3. Custom Universe Domains (`universeDomain`) + +To target Google Cloud Dedicated (GCD) or non-default Universe Domain instances, specify `universeDomain`: + +``` +jdbc:bigquery://https://www.my-universe.cloud:443 + ;ProjectId=my-gcp-project + ;OAuthType=0 + ;universeDomain=my-universe.cloud +``` + +--- + +## 9. Framework Integration & Deployment + +### Spring Boot (`application.yml`) +```yaml +spring: + datasource: + url: "jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=my-gcp-project;DefaultDataset=my_dataset;OAuthType=0;OAuthPvtKeyPath=/path/to/key.json" + driver-class-name: com.google.cloud.bigquery.jdbc.BigQueryDriver + hikari: + maximum-pool-size: 10 + minimum-idle: 2 + idle-timeout: 300000 + connection-test-query: SELECT 1 +``` + +### Key Deployment Guidelines +1. **Driver Class**: `com.google.cloud.bigquery.jdbc.BigQueryDriver` +2. **Connection Pooling**: BigQuery connection objects are lightweight wrappers around thread-safe Google API clients. Standard connection pools like HikariCP can be safely used. +3. **IAM Permissions**: Ensure the principal has `BigQuery Data Viewer` and `BigQuery Job User` roles on the target project and dataset. + +--- + +## 10. Logging, Diagnostics & Troubleshooting + +### Configuring Connection-Level Logging + +The driver includes a built-in logging subsystem built on top of `java.util.logging`. Logging can be configured directly via JDBC URL connection properties or system environment variables: + +#### Connection URL Parameters: +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443" + + ";ProjectId=my-gcp-project" + + ";OAuthType=0" + + ";LogLevel=5" // Set log level (1=SEVERE to 7=FINEST) + + ";LogPath=/var/log/bigquery-jdbc"; // Output directory for log files +``` + +#### Environment Variables for Logging: + +Logging can also be configured globally across all driver connections using environment variables: + +| Environment Variable | Equivalent Connection Property | Description | Example | +| :--- | :--- | :--- | :--- | +| **`BIGQUERY_JDBC_LOG_LEVEL`** | `LogLevel` | Global default log verbosity level (`0`–`7` or `OFF`–`FINEST`). Overridden by `LogLevel` URL parameter. | `export BIGQUERY_JDBC_LOG_LEVEL=5` | +| **`BIGQUERY_JDBC_LOG_PATH`** | `LogPath` | Global output directory for driver log files. Overridden by `LogPath` URL parameter. | `export BIGQUERY_JDBC_LOG_PATH=/var/log/bigquery-jdbc` | + +#### Log Level Reference: + +| Integer Value | String Constant | `java.util.logging` Level | Description | +| :---: | :---: | :---: | :--- | +| `0` | `OFF` | `Level.OFF` | Logging completely disabled (default). | +| `1` | `SEVERE` | `Level.SEVERE` | Critical errors, connection failures, unrecoverable exceptions. | +| `2` | `WARNING` | `Level.WARNING` | Non-fatal warnings, API fallbacks (e.g., Read API permission denied fallback to REST). | +| `3` | `INFO` | `Level.INFO` | High-level driver events, connection establishment, job submission. | +| `4` | `CONFIG` | `Level.CONFIG` | Driver configuration initialization and property resolution details. | +| `5` | `FINE` | `Level.FINE` | SQL query execution statements, parameter binding, row counts. | +| `6` | `FINER` | `Level.FINER` | Internal method entry (`++enter++`) and exit (`++exit++`) tracing. | +| `7` | `FINEST` | `Level.FINEST` | Maximum diagnostic verbosity, raw batch payloads, Arrow stream packet details. | + +#### Log File Naming & Format + +Log files are generated inside the specified `LogPath` directory with per-connection log isolation. + +**Standard Log Record Format**: +```text +2026-07-22 14:25:10.123 [conn-bq-8f3a] INFO 12345 --- [ main ] com.google.cloud.bigquery.jdbc.BigQueryStatement executeQuery : Executing query: SELECT COUNT(*) FROM `my-project.analytics.orders` +``` + +- **Timestamp**: `yyyy-MM-dd HH:mm:ss.SSS` in local time. +- **Connection ID (`MDC`)**: Unique identifier `[conn-bq-XXXX]` isolating statements per connection context. +- **Level**: Padded log level (`INFO`, `FINE`, `WARNING`, etc.). +- **Process ID & Thread**: OS Process ID and centered Thread Name. +- **Class & Method**: Fully qualified source class name and method name originating the log message. + +--- + +### OpenTelemetry & Cloud Observability + +The BigQuery JDBC driver features native OpenTelemetry (OTel) instrumentation for distributed tracing and Cloud Logging correlation across SQL query executions, prepared statement batching, and gRPC storage streams. + +#### Telemetry Connection Properties: + +| Connection Property | Type | Default | Description | +| :--- | :---: | :---: | :--- | +| **`enableGcpTraceExporter`** | `Boolean` | `false` | Enables automatic exporting of OpenTelemetry trace spans directly to Google Cloud Trace over OTLP gRPC (`https://telemetry.googleapis.com:443`). | +| **`enableGcpLogExporter`** | `Boolean` | `false` | Enables exporting driver log events directly to Google Cloud Logging with correlated `traceId` and `spanId` context. | +| **`useGlobalOpenTelemetry`** | `Boolean` | `false` | Directs the driver to use the application's global `GlobalOpenTelemetry` instance registered in the JVM context. | +| **`gcpTelemetryProjectId`** | `String` | `null` | Specifies a custom GCP Project ID for telemetry export when different from the main query connection project catalog. | +| **`gcpTelemetryCredentials`** | `String` | `null` | Path or raw JSON string for dedicated service account credentials used for telemetry export. | + +#### OpenTelemetry Span Attributes: + +Spans generated by the driver automatically capture standard OpenTelemetry database semantic conventions: +- `db.system`: Always set to `bigquery`. +- `db.connection_id`: Unique connection identifier (e.g., `conn-bq-8f3a`). +- `db.statement`: Executed SQL statement text (safely truncated at 32KB to avoid trace payload overflow). +- `db.application`: Application client name (defaults to `Google-BigQuery-JDBC-Driver` or custom partner token). + +#### Enabling GCP Cloud Trace & Cloud Logging via JDBC URL: +```java +String url = "jdbc:bigquery://https://bigquery.googleapis.com:443" + + ";ProjectId=my-gcp-project" + + ";OAuthType=0" + + ";OAuthPvtKeyPath=/var/secrets/bq-key.json" + + ";enableGcpTraceExporter=true" + + ";enableGcpLogExporter=true"; + +Connection conn = DriverManager.getConnection(url); +``` + +#### Programmatic Custom OpenTelemetry Injection: +Applications with an existing `OpenTelemetry` SDK instance can pass it programmatically via connection `Properties`: +```java +OpenTelemetry customOtel = OpenTelemetrySdk.builder()...build(); + +Properties props = new Properties(); +props.put("customOpenTelemetry", customOtel); + +Connection conn = DriverManager.getConnection("jdbc:bigquery://https://bigquery.googleapis.com:443;ProjectId=my-project;OAuthType=3", props); +``` + +--- + +### Exception Hierarchy + +All driver exceptions inherit from `java.sql.SQLException`: + +- **`BigQueryJdbcException`**: Base exception class wrapping underlying Google Cloud API and network RPC errors. +- **`BigQueryJdbcSqlSyntaxErrorException`**: Thrown when standard SQL query validation or parsing fails on the server. +- **`BigQueryConversionException`**: Thrown on data type coercion failures or unparseable target representations. +- **`BigQueryJdbcSqlFeatureNotSupportedException`**: Thrown when an unsupported JDBC API method is invoked. + +--- + +## 11. Official Documentation References + +- 🌐 [Official Google Cloud BigQuery JDBC Driver Overview](https://cloud.google.com/bigquery/docs/jdbc-for-bigquery) +- ⚙️ [Running Queries with the JDBC Driver](https://cloud.google.com/bigquery/docs/jdbc-for-bigquery#run_queries_with_the_driver) +- 📊 [BigQuery Quotas & Limits](https://cloud.google.com/bigquery/quotas) From abf3d8b31b7d1491f79f8db56bcaff0748956e98 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Wed, 29 Jul 2026 10:30:04 -0400 Subject: [PATCH 3/5] feat(bigquery): add QueryResultsFormat and ArrowSerializationOptions configurations --- google-cloud-jar-parent/pom.xml | 2 +- .../bigquery/ArrowSerializationOptions.java | 118 ++++++++++++++++++ .../cloud/bigquery/QueryJobConfiguration.java | 47 ++++++- .../cloud/bigquery/QueryResultsFormat.java | 29 +++++ .../bigquery/QueryJobConfigurationTest.java | 29 +++++ 5 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowSerializationOptions.java create mode 100644 java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryResultsFormat.java diff --git a/google-cloud-jar-parent/pom.xml b/google-cloud-jar-parent/pom.xml index b1e26861cab0..1f30f4f224e6 100644 --- a/google-cloud-jar-parent/pom.xml +++ b/google-cloud-jar-parent/pom.xml @@ -142,7 +142,7 @@ com.google.apis google-api-services-bigquery - v2-rev20260612-2.0.0 + v2-rev20260707-2.0.0 diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowSerializationOptions.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowSerializationOptions.java new file mode 100644 index 000000000000..457ea4917daf --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowSerializationOptions.java @@ -0,0 +1,118 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.core.BetaApi; +import java.io.Serializable; +import java.util.Objects; + +/** Options specific to the Apache Arrow output format. */ +@BetaApi +public final class ArrowSerializationOptions implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String bufferCompression; + private final String picosTimestampPrecision; + + private ArrowSerializationOptions(Builder builder) { + this.bufferCompression = builder.bufferCompression; + this.picosTimestampPrecision = builder.picosTimestampPrecision; + } + + public String getBufferCompression() { + return bufferCompression; + } + + public String getPicosTimestampPrecision() { + return picosTimestampPrecision; + } + + public static Builder newBuilder() { + return new Builder(); + } + + @Override + public String toString() { + return com.google.common.base.MoreObjects.toStringHelper(this) + .add("bufferCompression", bufferCompression) + .add("picosTimestampPrecision", picosTimestampPrecision) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ArrowSerializationOptions that = (ArrowSerializationOptions) o; + return Objects.equals(bufferCompression, that.bufferCompression) + && Objects.equals(picosTimestampPrecision, that.picosTimestampPrecision); + } + + @Override + public int hashCode() { + return Objects.hash(bufferCompression, picosTimestampPrecision); + } + + com.google.api.services.bigquery.model.ArrowSerializationOptions toPb() { + com.google.api.services.bigquery.model.ArrowSerializationOptions optionsPb = + new com.google.api.services.bigquery.model.ArrowSerializationOptions(); + if (bufferCompression != null) { + optionsPb.setBufferCompression(bufferCompression); + } + if (picosTimestampPrecision != null) { + optionsPb.setPicosTimestampPrecision(picosTimestampPrecision); + } + return optionsPb; + } + + static ArrowSerializationOptions fromPb( + com.google.api.services.bigquery.model.ArrowSerializationOptions optionsPb) { + if (optionsPb == null) { + return null; + } + return newBuilder() + .setBufferCompression(optionsPb.getBufferCompression()) + .setPicosTimestampPrecision(optionsPb.getPicosTimestampPrecision()) + .build(); + } + + public static final class Builder { + private String bufferCompression; + private String picosTimestampPrecision; + + private Builder() {} + + public Builder setBufferCompression(String bufferCompression) { + this.bufferCompression = bufferCompression; + return this; + } + + public Builder setPicosTimestampPrecision(String picosTimestampPrecision) { + this.picosTimestampPrecision = picosTimestampPrecision; + return this; + } + + public ArrowSerializationOptions build() { + return new ArrowSerializationOptions(this); + } + } +} diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java index a62fbb5008d4..0d75d509d1a3 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java @@ -75,6 +75,8 @@ public final class QueryJobConfiguration extends JobConfiguration { private final Long maxResults; private final JobCreationMode jobCreationMode; private final String reservation; + private final QueryResultsFormat queryResultsFormat; + private final ArrowSerializationOptions arrowSerializationOptions; /** * Priority levels for a query. If not specified the priority is assumed to be {@link @@ -144,6 +146,8 @@ public static final class Builder private Long maxResults; private JobCreationMode jobCreationMode; private String reservation; + private QueryResultsFormat queryResultsFormat; + private ArrowSerializationOptions arrowSerializationOptions; private Builder() { super(Type.QUERY); @@ -181,6 +185,8 @@ private Builder(QueryJobConfiguration jobConfiguration) { this.maxResults = jobConfiguration.maxResults; this.jobCreationMode = jobConfiguration.jobCreationMode; this.reservation = jobConfiguration.reservation; + this.queryResultsFormat = jobConfiguration.queryResultsFormat; + this.arrowSerializationOptions = jobConfiguration.arrowSerializationOptions; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -701,6 +707,17 @@ public Builder setReservation(String reservation) { return this; } + public Builder setQueryResultsFormat(QueryResultsFormat queryResultsFormat) { + this.queryResultsFormat = queryResultsFormat; + return this; + } + + public Builder setArrowSerializationOptions( + ArrowSerializationOptions arrowSerializationOptions) { + this.arrowSerializationOptions = arrowSerializationOptions; + return this; + } + public QueryJobConfiguration build() { return new QueryJobConfiguration(this); } @@ -747,6 +764,8 @@ private QueryJobConfiguration(Builder builder) { this.maxResults = builder.maxResults; this.jobCreationMode = builder.jobCreationMode; this.reservation = builder.reservation; + this.queryResultsFormat = builder.queryResultsFormat; + this.arrowSerializationOptions = builder.arrowSerializationOptions; } /** @@ -973,6 +992,14 @@ public Builder toBuilder() { return new Builder(this); } + public QueryResultsFormat getQueryResultsFormat() { + return queryResultsFormat; + } + + public ArrowSerializationOptions getArrowSerializationOptions() { + return arrowSerializationOptions; + } + @Override ToStringHelper toStringHelper() { return super.toStringHelper() @@ -1004,13 +1031,23 @@ ToStringHelper toStringHelper() { .add("rangePartitioning", rangePartitioning) .add("connectionProperties", connectionProperties) .add("jobCreationMode", jobCreationMode) - .add("reservation", reservation); + .add("reservation", reservation) + .add("queryResultsFormat", queryResultsFormat) + .add("arrowSerializationOptions", arrowSerializationOptions); } @Override public boolean equals(Object obj) { - return obj == this - || obj instanceof QueryJobConfiguration && baseEquals((QueryJobConfiguration) obj); + if (obj == this) { + return true; + } + if (obj == null || !(obj instanceof QueryJobConfiguration)) { + return false; + } + QueryJobConfiguration other = (QueryJobConfiguration) obj; + return baseEquals(other) + && Objects.equals(queryResultsFormat, other.queryResultsFormat) + && Objects.equals(arrowSerializationOptions, other.arrowSerializationOptions); } @Override @@ -1043,7 +1080,9 @@ public int hashCode() { labels, rangePartitioning, connectionProperties, - reservation); + reservation, + queryResultsFormat, + arrowSerializationOptions); } @Override diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryResultsFormat.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryResultsFormat.java new file mode 100644 index 000000000000..61c12d34e326 --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryResultsFormat.java @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.core.BetaApi; + +/** The format of the query results. */ +@BetaApi +public enum QueryResultsFormat { + /** Serialized row data in Apache Arrow format. */ + ARROW, + + /** Default encoding of results as JSON struct array. */ + STRUCT_ENCODING +} diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java index 7fe41daa0608..1d60d904bfab 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java @@ -241,6 +241,35 @@ public void testJobCreationMode() { QUERY_JOB_CONFIGURATION_SET_JOB_CREATION_MODE.toBuilder().build()); } + @Test + public void testArrowConfigurations() { + QueryResultsFormat format = QueryResultsFormat.ARROW; + ArrowSerializationOptions options = + ArrowSerializationOptions.newBuilder() + .setBufferCompression("LZ4") + .setPicosTimestampPrecision("PRECISION_MILLIS") + .build(); + QueryJobConfiguration job = + QueryJobConfiguration.newBuilder(QUERY) + .setQueryResultsFormat(format) + .setArrowSerializationOptions(options) + .build(); + + assertEquals(format, job.getQueryResultsFormat()); + assertEquals(options, job.getArrowSerializationOptions()); + + // Test toBuilder + QueryJobConfiguration copiedJob = job.toBuilder().build(); + assertEquals(job, copiedJob); + assertEquals(format, copiedJob.getQueryResultsFormat()); + assertEquals(options, copiedJob.getArrowSerializationOptions()); + + // Test toPb/fromPb (not preserved) + QueryJobConfiguration jobFromPb = QueryJobConfiguration.fromPb(job.toPb()); + assertNull(jobFromPb.getQueryResultsFormat()); + assertNull(jobFromPb.getArrowSerializationOptions()); + } + private void compareQueryJobConfiguration( QueryJobConfiguration expected, QueryJobConfiguration value) { assertEquals(expected, value); From 20cc629d343c87174f518a686a50f7c2cf6991f9 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Wed, 29 Jul 2026 10:33:19 -0400 Subject: [PATCH 4/5] feat(bigquery): add ArrowDeserializer helper utility --- .../cloud/bigquery/ArrowDeserializer.java | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java new file mode 100644 index 000000000000..ab586fe9b2b2 --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java @@ -0,0 +1,205 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.BaseEncoding; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorLoader; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.ipc.ReadChannel; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; + +final class ArrowDeserializer { + + private ArrowDeserializer() {} + + static Schema arrowSchemaToBigQuerySchema(org.apache.arrow.vector.types.pojo.Schema arrowSchema) { + List fields = new ArrayList<>(); + for (Field arrowField : arrowSchema.getFields()) { + fields.add(arrowFieldToBigQueryField(arrowField)); + } + return Schema.of(fields); + } + + private static com.google.cloud.bigquery.Field arrowFieldToBigQueryField(Field arrowField) { + String name = arrowField.getName(); + ArrowType type = arrowField.getType(); + com.google.cloud.bigquery.Field.Builder builder; + + if (type instanceof ArrowType.List) { + Field innerField = arrowField.getChildren().get(0); + LegacySQLTypeName innerType = arrowTypeToLegacySQLTypeName(innerField.getType()); + builder = com.google.cloud.bigquery.Field.newBuilder(name, innerType); + builder.setMode(com.google.cloud.bigquery.Field.Mode.REPEATED); + if (!innerField.getChildren().isEmpty()) { + List subFields = new ArrayList<>(); + for (Field childField : innerField.getChildren()) { + subFields.add(arrowFieldToBigQueryField(childField)); + } + builder.setType(LegacySQLTypeName.RECORD, FieldList.of(subFields)); + } + } else { + LegacySQLTypeName bqType = arrowTypeToLegacySQLTypeName(type); + builder = com.google.cloud.bigquery.Field.newBuilder(name, bqType); + if (arrowField.isNullable()) { + builder.setMode(com.google.cloud.bigquery.Field.Mode.NULLABLE); + } else { + builder.setMode(com.google.cloud.bigquery.Field.Mode.REQUIRED); + } + if (!arrowField.getChildren().isEmpty()) { + List subFields = new ArrayList<>(); + for (Field childField : arrowField.getChildren()) { + subFields.add(arrowFieldToBigQueryField(childField)); + } + builder.setType(LegacySQLTypeName.RECORD, FieldList.of(subFields)); + } + } + return builder.build(); + } + + private static LegacySQLTypeName arrowTypeToLegacySQLTypeName(ArrowType type) { + switch (type.getTypeID()) { + case Int: + return LegacySQLTypeName.INTEGER; + case FloatingPoint: + return LegacySQLTypeName.FLOAT; + case Utf8: + return LegacySQLTypeName.STRING; + case Bool: + return LegacySQLTypeName.BOOLEAN; + case Binary: + return LegacySQLTypeName.BYTES; + case Decimal: + return LegacySQLTypeName.NUMERIC; + case Timestamp: + return LegacySQLTypeName.TIMESTAMP; + case Date: + return LegacySQLTypeName.DATE; + case Time: + return LegacySQLTypeName.TIME; + case Struct: + return LegacySQLTypeName.RECORD; + default: + throw new IllegalArgumentException("Unsupported Arrow type: " + type.getTypeID()); + } + } + + static List deserializeRecordBatch( + byte[] recordBatchBytes, Schema schema, org.apache.arrow.vector.types.pojo.Schema arrowSchema) + throws IOException { + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + List vectors = new ArrayList<>(); + for (Field field : arrowSchema.getFields()) { + vectors.add(field.createVector(allocator)); + } + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + VectorLoader loader = new VectorLoader(root); + try (org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = + MessageSerializer.deserializeRecordBatch( + new ReadChannel(new ByteArrayReadableSeekableByteChannel(recordBatchBytes)), + allocator)) { + loader.load(deserializedBatch); + int rowCount = root.getRowCount(); + List rows = new ArrayList<>(rowCount); + for (int i = 0; i < rowCount; i++) { + rows.add(arrowRootToFieldValueList(root, i, schema)); + } + return ImmutableList.copyOf(rows); + } + } + } + } + + static FieldValueList arrowRootToFieldValueList( + VectorSchemaRoot root, int rowIndex, Schema schema) { + List fieldValues = new ArrayList<>(); + for (int colIndex = 0; colIndex < root.getFieldVectors().size(); colIndex++) { + FieldVector vector = root.getVector(colIndex); + com.google.cloud.bigquery.Field bqField = schema.getFields().get(colIndex); + fieldValues.add(arrowVectorToFieldValue(vector, rowIndex, bqField)); + } + return FieldValueList.of(fieldValues, schema.getFields()); + } + + private static FieldValue arrowVectorToFieldValue( + FieldVector vector, int rowIndex, com.google.cloud.bigquery.Field bqField) { + if (vector.isNull(rowIndex)) { + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, null); + } + + // Handle repeated fields + if (bqField.getMode() == com.google.cloud.bigquery.Field.Mode.REPEATED) { + ListVector listVector = (ListVector) vector; + FieldVector dataVector = (FieldVector) listVector.getDataVector(); + int start = listVector.getElementStartIndex(rowIndex); + int end = listVector.getElementEndIndex(rowIndex); + List elements = new ArrayList<>(end - start); + com.google.cloud.bigquery.Field elementBqField = + com.google.cloud.bigquery.Field.newBuilder(bqField.getName(), bqField.getType()) + .setMode(com.google.cloud.bigquery.Field.Mode.NULLABLE) + .build(); + for (int k = start; k < end; k++) { + elements.add(arrowVectorToFieldValue(dataVector, k, elementBqField)); + } + return FieldValue.of( + FieldValue.Attribute.REPEATED, FieldValueList.of(elements, bqField.getSubFields())); + } + + // Handle RECORD/STRUCT fields + if (bqField.getType() == LegacySQLTypeName.RECORD) { + StructVector structVector = (StructVector) vector; + List elements = new ArrayList<>(structVector.size()); + for (int colIndex = 0; colIndex < structVector.size(); colIndex++) { + FieldVector childVector = (FieldVector) structVector.getChildByOrdinal(colIndex); + com.google.cloud.bigquery.Field childBqField = bqField.getSubFields().get(colIndex); + elements.add(arrowVectorToFieldValue(childVector, rowIndex, childBqField)); + } + return FieldValue.of( + FieldValue.Attribute.RECORD, FieldValueList.of(elements, bqField.getSubFields())); + } + + // Handle primitive types - convert everything to String representations to match BQ standard + Object value = vector.getObject(rowIndex); + String stringVal; + if (value instanceof byte[]) { + stringVal = BaseEncoding.base64().encode((byte[]) value); + } else if (bqField.getType() == LegacySQLTypeName.TIMESTAMP) { + // Arrow timestamps are long values representing epoch micro/milli/nano seconds. + // Standard BigQuery JSON returns timestamps as string of epoch seconds with micro precision + // (e.g. "1408452095.220000"). + long micros = (long) value; + // Convert to seconds with 6 decimal places of precision + stringVal = String.format(Locale.US, "%.6f", micros / 1000000.0); + } else { + stringVal = String.valueOf(value); + } + + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal); + } +} From 796b77f24ebd4db1f1ccb92cb67f93ed854a1225 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 7 Aug 2026 14:51:32 -0400 Subject: [PATCH 5/5] fix(bigquery): resolve review comments in ArrowDeserializer --- .../cloud/bigquery/ArrowDeserializer.java | 81 +++++++++++++++---- 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java index ab586fe9b2b2..66a88e9c55c8 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java @@ -25,6 +25,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; @@ -53,6 +54,10 @@ private static com.google.cloud.bigquery.Field arrowFieldToBigQueryField(Field a com.google.cloud.bigquery.Field.Builder builder; if (type instanceof ArrowType.List) { + if (arrowField.getChildren().isEmpty()) { + throw new IllegalArgumentException( + "Arrow List field must have at least one child field: " + name); + } Field innerField = arrowField.getChildren().get(0); LegacySQLTypeName innerType = arrowTypeToLegacySQLTypeName(innerField.getType()); builder = com.google.cloud.bigquery.Field.newBuilder(name, innerType); @@ -115,8 +120,19 @@ static List deserializeRecordBatch( throws IOException { try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { List vectors = new ArrayList<>(); - for (Field field : arrowSchema.getFields()) { - vectors.add(field.createVector(allocator)); + try { + for (Field field : arrowSchema.getFields()) { + vectors.add(field.createVector(allocator)); + } + } catch (Throwable t) { + for (int i = vectors.size() - 1; i >= 0; i--) { + try { + vectors.get(i).close(); + } catch (Exception e) { + t.addSuppressed(e); + } + } + throw t; } try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { VectorLoader loader = new VectorLoader(root); @@ -138,6 +154,12 @@ static List deserializeRecordBatch( static FieldValueList arrowRootToFieldValueList( VectorSchemaRoot root, int rowIndex, Schema schema) { + if (root.getFieldVectors().size() != schema.getFields().size()) { + throw new IllegalArgumentException( + String.format( + "Schema mismatch: Arrow vector count (%d) does not match BigQuery schema field count (%d)", + root.getFieldVectors().size(), schema.getFields().size())); + } List fieldValues = new ArrayList<>(); for (int colIndex = 0; colIndex < root.getFieldVectors().size(); colIndex++) { FieldVector vector = root.getVector(colIndex); @@ -160,10 +182,13 @@ private static FieldValue arrowVectorToFieldValue( int start = listVector.getElementStartIndex(rowIndex); int end = listVector.getElementEndIndex(rowIndex); List elements = new ArrayList<>(end - start); + com.google.cloud.bigquery.Field.Builder elementBuilder = + com.google.cloud.bigquery.Field.newBuilder(bqField.getName(), bqField.getType()); + if (bqField.getType() == LegacySQLTypeName.RECORD && bqField.getSubFields() != null) { + elementBuilder.setType(LegacySQLTypeName.RECORD, bqField.getSubFields()); + } com.google.cloud.bigquery.Field elementBqField = - com.google.cloud.bigquery.Field.newBuilder(bqField.getName(), bqField.getType()) - .setMode(com.google.cloud.bigquery.Field.Mode.NULLABLE) - .build(); + elementBuilder.setMode(com.google.cloud.bigquery.Field.Mode.NULLABLE).build(); for (int k = start; k < end; k++) { elements.add(arrowVectorToFieldValue(dataVector, k, elementBqField)); } @@ -174,6 +199,12 @@ private static FieldValue arrowVectorToFieldValue( // Handle RECORD/STRUCT fields if (bqField.getType() == LegacySQLTypeName.RECORD) { StructVector structVector = (StructVector) vector; + if (structVector.size() != bqField.getSubFields().size()) { + throw new IllegalArgumentException( + String.format( + "Schema mismatch for field '%s': Arrow struct size (%d) does not match BigQuery subfields size (%d)", + bqField.getName(), structVector.size(), bqField.getSubFields().size())); + } List elements = new ArrayList<>(structVector.size()); for (int colIndex = 0; colIndex < structVector.size(); colIndex++) { FieldVector childVector = (FieldVector) structVector.getChildByOrdinal(colIndex); @@ -184,20 +215,40 @@ private static FieldValue arrowVectorToFieldValue( FieldValue.Attribute.RECORD, FieldValueList.of(elements, bqField.getSubFields())); } - // Handle primitive types - convert everything to String representations to match BQ standard - Object value = vector.getObject(rowIndex); + // Handle primitive types String stringVal; - if (value instanceof byte[]) { - stringVal = BaseEncoding.base64().encode((byte[]) value); - } else if (bqField.getType() == LegacySQLTypeName.TIMESTAMP) { - // Arrow timestamps are long values representing epoch micro/milli/nano seconds. + if (bqField.getType() == LegacySQLTypeName.TIMESTAMP) { + // Arrow timestamps are long values representing epoch seconds/millis/micros/nanos. // Standard BigQuery JSON returns timestamps as string of epoch seconds with micro precision // (e.g. "1408452095.220000"). - long micros = (long) value; - // Convert to seconds with 6 decimal places of precision - stringVal = String.format(Locale.US, "%.6f", micros / 1000000.0); + TimeStampVector tsVector = (TimeStampVector) vector; + long rawVal = tsVector.get(rowIndex); + ArrowType.Timestamp tsType = (ArrowType.Timestamp) vector.getField().getType(); + long micros; + switch (tsType.getUnit()) { + case SECOND: + micros = rawVal * 1_000_000L; + break; + case MILLISECOND: + micros = rawVal * 1_000L; + break; + case MICROSECOND: + micros = rawVal; + break; + case NANOSECOND: + micros = rawVal / 1_000L; + break; + default: + micros = rawVal; + } + stringVal = String.format(Locale.US, "%.6f", micros / 1_000_000.0); } else { - stringVal = String.valueOf(value); + Object value = vector.getObject(rowIndex); + if (value instanceof byte[]) { + stringVal = BaseEncoding.base64().encode((byte[]) value); + } else { + stringVal = String.valueOf(value); + } } return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal);