From 8a8ebe16372943281beb27bbf3d9e9814cee600e Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Fri, 6 Nov 2026 10:39:42 -0500 Subject: [PATCH 1/5] fix(bigquery-jdbc): correct usage-telemetry counts and disable telemetry in integration tests --- java-bigquery-jdbc/pom.xml | 2 + .../cloud/bigquery/jdbc/BigQueryDriver.java | 15 +-- .../jdbc/BigQueryParameterHandler.java | 2 + .../jdbc/BigQueryPooledConnection.java | 3 - .../bigquery/jdbc/BigQueryStatement.java | 14 ++- .../bigquery/jdbc/BigQueryDriverTest.java | 95 +++++++++++++++---- .../jdbc/BigQueryPooledConnectionTest.java | 21 ++++ .../BigQueryPreparedStatementSettersTest.java | 19 ++++ .../bigquery/jdbc/BigQueryStatementTest.java | 31 ++++++ 9 files changed, 169 insertions(+), 33 deletions(-) diff --git a/java-bigquery-jdbc/pom.xml b/java-bigquery-jdbc/pom.xml index 91c12b2c06de..0280f2c2ba04 100644 --- a/java-bigquery-jdbc/pom.xml +++ b/java-bigquery-jdbc/pom.xml @@ -67,6 +67,8 @@ true + + false diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java index 140636632316..0068b4341fc0 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java @@ -27,6 +27,7 @@ import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.Driver; import java.sql.DriverManager; import java.sql.DriverPropertyInfo; @@ -196,10 +197,12 @@ public Connection connect(String url, Properties info) throws SQLException { LOG.log(Level.SEVERE, "Failed to parse connection URL", e); throw new BigQueryJdbcException("Failed to parse connection URL", e); } + authType = TelemetryManager.toAuthenticationType(ds.getOAuthType()); if (customOpenTelemetryObj instanceof OpenTelemetry) { ds.setCustomOpenTelemetry((OpenTelemetry) customOpenTelemetryObj); } BigQueryConnection connection = new BigQueryConnection(connectionUri, ds); + DatabaseMetaData metaData = connection.getMetaData(); LOG.info( "Driver info : { {Database Product Name : %s}, " + "{Database Product Version : %s}, " @@ -208,17 +211,16 @@ public Connection connect(String url, Properties info) throws SQLException { + "{LogLevel : %s}, " + "{LogPath : %s}, " + "{Driver Instance : %s} }", - connection.getMetaData().getDatabaseProductName(), - connection.getMetaData().getDatabaseProductVersion(), - connection.getMetaData().getDriverName(), - connection.getMetaData().getDriverVersion(), + metaData.getDatabaseProductName(), + metaData.getDatabaseProductVersion(), + metaData.getDriverName(), + metaData.getDriverVersion(), logLevel, logPath, this.toString()); Connection wrapped = BigQueryJdbcContextProxy.wrap(connection, Connection.class); - authType = TelemetryManager.toAuthenticationType(ds.getOAuthType()); TelemetryManager.recordConnectionAttempt(Status.STATUS_SUCCESS, 0, authType); return wrapped; } else { @@ -227,8 +229,7 @@ public Connection connect(String url, Properties info) throws SQLException { } catch (IOException e) { TelemetryManager.recordConnectionAttempt( Status.STATUS_ERROR, TelemetryManager.extractErrorCode(e), authType); - LOG.warning("Getting a warning: %s", e.getMessage()); - return null; + throw new BigQueryJdbcException("Failed to establish connection", e); } catch (Throwable t) { TelemetryManager.recordConnectionAttempt( Status.STATUS_ERROR, TelemetryManager.extractErrorCode(t), authType); diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java index 6e65e978880c..2fc89afbf173 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java @@ -289,6 +289,8 @@ void setParameter(int parameterIndex, Object value, Class type) { parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED); parameter.setScale(-1); + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + LOG.finest("Parameter set { %s }", parameter.toString()); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java index 6b90421d5d3d..24571e0480b6 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java @@ -16,8 +16,6 @@ package com.google.cloud.bigquery.jdbc; -import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; -import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.common.annotations.VisibleForTesting; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -237,7 +235,6 @@ public void rollback() throws SQLException { @Override public DatabaseMetaData getMetaData() throws SQLException { - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL); return bqConnectionDelegate.getMetaData(); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index e036030e6d6f..5b010b000374 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -406,7 +406,7 @@ public void close() throws SQLException { boolean cancelSucceeded = false; try { - cancel(); // This attempts to cancel jobs and calls closeStatementResources() + cancelInternal(); // Cancels jobs and calls closeStatementResources(); no telemetry. cancelSucceeded = true; } catch (SQLException e) { LOG.warning("Failed to cancel statement during close().", e); @@ -467,6 +467,16 @@ public void setQueryTimeout(int seconds) { @Override public void cancel() throws SQLException { LOG.finer("Statement %s cancelled", this); + cancelInternal(); + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL); + } + + /** + * Cancels running jobs and releases statement resources without recording telemetry. Used by + * {@link #close()}, where cancellation is housekeeping, not a user-initiated cancel. + */ + private void cancelInternal() throws SQLException { + LOG.finer("Statement %s cancelled internally", this); synchronized (cancelLock) { this.isCanceled = true; for (JobId jobId : this.jobIds) { @@ -488,8 +498,6 @@ public void cancel() throws SQLException { // If a ResultSet exists, then it will be closed as well, closing the // ownedThreads closeStatementResources(); - - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL); } @Override diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java index 4c4030d3e1f8..c98e8dfabb53 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java @@ -16,12 +16,23 @@ package com.google.cloud.bigquery.jdbc; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.AuthenticationType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.Status; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryTestUtils; import com.google.cloud.bigquery.jdbc.utils.BigQueryJdbcVersionUtility; import io.opentelemetry.api.OpenTelemetry; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverPropertyInfo; import java.sql.SQLException; @@ -32,11 +43,19 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; public class BigQueryDriverTest extends BigQueryJdbcLoggingBaseTest { static BigQueryDriver bigQueryDriver; + /** Token-auth URL that connects without network access; shared by the telemetry tests. */ + private static final String VALID_TOKEN_URL = + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "OAuthType=2;ProjectId=MyBigQueryProject;" + + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" + + "OAuthClientSecret=redactedToken;"; + @BeforeEach @AfterEach public void resetTelemetry() { @@ -55,13 +74,7 @@ public void testInvalidURLReturnsNull() throws SQLException { @Test public void testValidURLDoesConnect() throws SQLException { - Connection connection = - bigQueryDriver.connect( - "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" - + "OAuthType=2;ProjectId=MyBigQueryProject;" - + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" - + "OAuthClientSecret=redactedToken;", - new Properties()); + Connection connection = bigQueryDriver.connect(VALID_TOKEN_URL, new Properties()); assertThat(connection.isClosed()).isFalse(); } @@ -199,13 +212,7 @@ public void testInvalidLogLevelExceptionIsLogged() { @Test public void testConnect_recordsSuccessfulConnectionTelemetry() throws SQLException { - Connection connection = - bigQueryDriver.connect( - "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" - + "OAuthType=2;ProjectId=MyBigQueryProject;" - + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" - + "OAuthClientSecret=redactedToken;", - new Properties()); + Connection connection = bigQueryDriver.connect(VALID_TOKEN_URL, new Properties()); assertThat(connection).isNotNull(); assertThat(connection.isClosed()).isFalse(); // Verify TelemetryManager is initialized and recorded the connection @@ -227,14 +234,62 @@ public void testConnect_recordsFailedConnectionTelemetry() { @Test public void testConnect_optOut_noTelemetryRecorded() throws SQLException { Connection connection = - bigQueryDriver.connect( - "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" - + "OAuthType=2;ProjectId=MyBigQueryProject;" - + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" - + "OAuthClientSecret=redactedToken;EnableDiagnosticTelemetry=0;", - new Properties()); + bigQueryDriver.connect(VALID_TOKEN_URL + "EnableDiagnosticTelemetry=0;", new Properties()); assertThat(connection).isNotNull(); // Since opt-out was requested, TelemetryManager should NOT be initialized assertThat(TelemetryManager.isInitialized()).isFalse(); } + + @Test + public void testConnect_recordsMetadataRetrievalOnce() throws SQLException { + // connect() logs four DatabaseMetaData values; it must fetch the object once, not four times. + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + bigQueryDriver.connect(VALID_TOKEN_URL, new Properties()); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + times(1)); + } + } + + @Test + public void testConnect_failedAttemptCarriesAuthType() { + // Service-account auth with no key material parses fine but fails while building the + // connection. The failed attempt must report the auth type, not + // AUTHENTICATION_TYPE_UNSPECIFIED. + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + Assertions.assertThrows( + Exception.class, + () -> + bigQueryDriver.connect( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "OAuthType=0;ProjectId=MyBigQueryProject;", + new Properties())); + telemetry.verify( + () -> + TelemetryManager.recordConnectionAttempt( + eq(Status.STATUS_ERROR), + anyInt(), + eq(AuthenticationType.AUTHENTICATION_TYPE_SERVICE_ACCOUNT))); + } + } + + @Test + public void testConnect_ioExceptionThrowsInsteadOfReturningNull() { + // BigQueryJdbcRootLogger.setLevel is the only IOException source on the connect path. JDBC + // requires connect() to throw for an accepted URL that fails, not return null. + try (MockedStatic rootLogger = + mockStatic(BigQueryJdbcRootLogger.class)) { + rootLogger + .when(() -> BigQueryJdbcRootLogger.setLevel(any(), any())) + .thenThrow(new IOException("simulated log path failure")); + BigQueryJdbcException thrown = + Assertions.assertThrows( + BigQueryJdbcException.class, + () -> bigQueryDriver.connect(VALID_TOKEN_URL, new Properties())); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java index f21dbcb5eff1..03ddee8535a6 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java @@ -23,11 +23,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import java.io.IOException; import java.sql.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; public class BigQueryPooledConnectionTest { private BigQueryConnection bigQueryConnection; @@ -173,4 +179,19 @@ public void testFireConnectionError() throws SQLException { assertFalse(pooledConnection.isListenerPooled(listner)); } + + @Test + public void testPooledGetMetaData_delegatesWithoutRecordingTelemetry() throws SQLException { + // The physical BigQueryConnection records DRIVER_FEATURE_METADATA_RETRIEVAL; the pooled + // wrapper must only delegate, otherwise every pooled call is counted twice. + BigQueryPooledConnection pooledConnection = new BigQueryPooledConnection(bigQueryConnection); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + pooledConnection.getConnection().getMetaData(); + verify(bigQueryConnection).getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + never()); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java index 59fc26767278..3ea3b61b633e 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java @@ -25,6 +25,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; import com.google.cloud.bigquery.Field; @@ -34,6 +36,8 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.gson.Gson; import com.google.gson.JsonNull; import com.google.gson.JsonObject; @@ -59,6 +63,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; public class BigQueryPreparedStatementSettersTest { @@ -532,4 +537,18 @@ public void testSetObjectLocalDateTime_zeroSecondsStillHasFullDigits() throws SQ preparedStatement.setObject(1, LocalDateTime.of(2025, 1, 1, 12, 0)); assertEquals("2025-01-01 12:00:00.000000", preparedStatement.parameterHandler.getParameter(1)); } + + @Test + public void testPrimitiveSetters_recordParameterBindingFeature() throws SQLException { + // These setters use the (int, Object, Class) overload of BigQueryParameterHandler, which + // previously did not record DRIVER_FEATURE_PARAMETER_BINDING. One record per binding. + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + preparedStatement.setInt(1, 42); + preparedStatement.setLong(2, 42L); + preparedStatement.setString(3, "x"); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING), + times(3)); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index 4ad8358f3894..aff1a1d70d1a 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -64,6 +64,8 @@ import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.jdbc.BigQueryStatement.JobIdWrapper; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.spi.BigQueryRpcFactory; import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; @@ -122,6 +124,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; import org.mockito.Mockito; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -1715,4 +1718,32 @@ public ReadRowsResponse next() { assertTrue(thrown.getCause().getCause() instanceof OutOfMemoryError); } } + + @Test + public void testClose_doesNotRecordStatementCancelFeature() throws SQLException { + // close() cancels jobs as housekeeping; that must not count as a user-initiated cancel. + assertFeatureRecorded( + () -> bigQueryStatement.close(), DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL, 0); + } + + @Test + public void testCancel_recordsStatementCancelFeatureOnce() throws SQLException { + assertFeatureRecorded( + () -> bigQueryStatement.cancel(), DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL, 1); + } + + /** Runs {@code action} and asserts how many times {@code feature} was recorded by it. */ + private static void assertFeatureRecorded( + SqlRunnable action, DriverFeature feature, int expectedCount) throws SQLException { + try (MockedStatic telemetry = Mockito.mockStatic(TelemetryManager.class)) { + action.run(); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(feature), Mockito.times(expectedCount)); + } + } + + @FunctionalInterface + private interface SqlRunnable { + void run() throws SQLException; + } } From a8f64a357f30a5c5fa3fe9abfce4259bb21c56d2 Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 16 Nov 2026 15:59:39 -0500 Subject: [PATCH 2/5] fix(bigquery-jdbc): record adoption features once per object --- .../bigquery/jdbc/BigQueryBaseResultSet.java | 15 +- .../bigquery/jdbc/BigQueryConnection.java | 51 ++++++- .../jdbc/BigQueryDatabaseMetaData.java | 2 +- .../cloud/bigquery/jdbc/BigQueryDriver.java | 2 +- .../jdbc/BigQueryParameterHandler.java | 30 +++- .../jdbc/BigQueryPreparedStatement.java | 34 ++++- .../bigquery/jdbc/BigQueryStatement.java | 36 ++++- .../bigquery/jdbc/BigQueryConnectionTest.java | 91 ++++++++++++ .../jdbc/BigQueryDatabaseMetaDataTest.java | 8 +- .../bigquery/jdbc/BigQueryDriverTest.java | 27 +++- .../jdbc/BigQueryJsonResultSetTest.java | 38 +++++ .../BigQueryPreparedStatementSettersTest.java | 140 +++++++++++++++++- .../bigquery/jdbc/BigQueryStatementTest.java | 43 ++++++ 13 files changed, 479 insertions(+), 38 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java index d0dd7a3f3c53..b253a8ad33f3 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java @@ -77,6 +77,9 @@ public abstract class BigQueryBaseResultSet extends BigQueryNoOpsResultSet protected final boolean isNested; protected boolean isClosed = false; protected boolean wasNull = false; + // getMetaData() is commonly called inside row loops; record the adoption signal once per + // result set. A ResultSet is single-threaded by JDBC contract, so a plain field suffices. + private boolean metadataRetrievalRecorded; private int fetchSize = -1; private Job job; private SQLWarning warnings; @@ -274,9 +277,15 @@ public ResultSetMetaData getMetaData() throws SQLException { } } - TelemetryManager.recordFeatureUsage( - DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, - "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + // Always a user call: no driver code invokes ResultSet.getMetaData(), so this is recorded + // regardless of whether the owning statement is internal (e.g. the result set that + // DatabaseMetaData.getExportedKeys hands back to the user). + if (!this.metadataRetrievalRecorded) { + this.metadataRetrievalRecorded = true; + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + } return BigQueryJdbcContextProxy.wrap(metaData, ResultSetMetaData.class, connectionId); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index ed63da4a8671..5190b7464016 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -81,6 +81,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; @@ -245,6 +246,9 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { Tracer tracer = OpenTelemetry.noop().getTracer(BigQueryJdbcOpenTelemetry.INSTRUMENTATION_SCOPE_NAME); DatabaseMetaData databaseMetaData; + // A Connection may be used from several threads; compareAndSet keeps the adoption signal at + // exactly one record per connection without a lock. + private final AtomicBoolean metadataRetrievalRecorded = new AtomicBoolean(); Boolean reqGoogleDriveScope; private final Properties clientInfo = new Properties(); private boolean isReadOnlyTokenUsed = false; @@ -505,6 +509,34 @@ public Statement createStatement() throws SQLException { return currentStatement; } + /** + * Creates a {@code Statement} for the driver's own use (for example the {@code isValid} + * health-check query). Identical to {@link #createStatement()} except that neither the statement + * nor anything it executes is reported as user activity in usage telemetry. + */ + BigQueryStatement createStatementInternal() throws SQLException { + checkClosed(); + BigQueryStatement statement = new BigQueryStatement(this); + statement.markInternal(); + LOG.fine("Internal Statement %s created.", statement); + addOpenStatements(statement); + return statement; + } + + /** + * Creates a {@code PreparedStatement} for the driver's own metadata queries. Identical to {@link + * #prepareStatement(String)} except that neither the statement, its parameter bindings, nor + * anything it executes is reported as user activity in usage telemetry. + */ + BigQueryPreparedStatement prepareStatementInternal(String sql) throws SQLException { + checkClosed(); + BigQueryPreparedStatement statement = new BigQueryPreparedStatement(this, sql); + statement.markInternal(); + LOG.fine("Internal Prepared Statement %s created.", statement); + addOpenStatements(statement); + return statement; + } + /** * Creates and returns a new {@code Statement} object for executing BigQuery SQL queries. This * method is similar to {@link BigQueryConnection#createStatement()}, but it overrides the type @@ -873,7 +905,8 @@ public boolean isValid(int timeout) throws SQLException { throw new BigQueryJdbcException("timeout must be >= 0"); } if (!isClosed()) { - try (Statement statement = createStatement(); + // Internal statement: a pool health-check must not count as user statements or SELECTs. + try (Statement statement = createStatementInternal(); ResultSet rs = statement.executeQuery("SELECT 1")) { LOG.finer("Running validation query"); if (rs.next()) { @@ -1021,12 +1054,22 @@ private void rollbackImpl() throws SQLException { @Override public DatabaseMetaData getMetaData() throws SQLException { + DatabaseMetaData metaData = getMetaDataInternal(); + // Adoption signal, recorded once per connection on the first user call. Tied to this flag + // rather than to instance creation so the driver's own startup lookup does not consume it. + if (this.metadataRetrievalRecorded.compareAndSet(false, true)) { + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL); + } + return metaData; + } + + /** + * Same instance as {@link #getMetaData()} without the usage-telemetry record. Driver-internal. + */ + DatabaseMetaData getMetaDataInternal() { if (databaseMetaData == null) { databaseMetaData = new BigQueryDatabaseMetaData(this); } - - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL); - return databaseMetaData; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 3e12ca300294..e530e66a9221 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -2197,7 +2197,7 @@ public ResultSet getExportedKeys(String catalog, String schema, String table) String sql = getExportedKeysSqlContent(); String formattedSql = replaceSqlParameters(sql, catalog, schema, table); - PreparedStatement stmt = this.connection.prepareStatement(formattedSql); + PreparedStatement stmt = this.connection.prepareStatementInternal(formattedSql); if (stmt == null) { throw new BigQueryJdbcException("Failed to prepare statement for getExportedKeys"); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java index 0068b4341fc0..f30a6bcf57e9 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java @@ -202,7 +202,7 @@ public Connection connect(String url, Properties info) throws SQLException { ds.setCustomOpenTelemetry((OpenTelemetry) customOpenTelemetryObj); } BigQueryConnection connection = new BigQueryConnection(connectionUri, ds); - DatabaseMetaData metaData = connection.getMetaData(); + DatabaseMetaData metaData = connection.getMetaDataInternal(); LOG.info( "Driver info : { {Database Product Name : %s}, " + "{Database Product Version : %s}, " diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java index 2fc89afbf173..d2edbfea4909 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java @@ -45,6 +45,28 @@ class BigQueryParameterHandler { new BigQueryJdbcCustomLogger(BigQueryParameterHandler.class.getName()); private final int parametersArraySize; private final boolean enableTimestampPicos; + + /** False when the owning statement is driver-internal; see BigQueryStatement#markInternal. */ + private boolean recordTelemetry = true; + + /** + * DRIVER_FEATURE_PARAMETER_BINDING is an adoption signal, recorded once per handler (i.e. per + * prepared statement) rather than once per setX call. Handlers are confined to one statement, + * which JDBC defines as single-threaded, so a plain field is sufficient. + */ + private boolean bindingRecorded; + + void markInternal() { + this.recordTelemetry = false; + } + + private void recordBinding() { + if (this.recordTelemetry && !this.bindingRecorded) { + this.bindingRecorded = true; + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + } + } + final ArrayList parametersList; // TIMESTAMP is rendered in UTC with a trailing 'Z': the instant is preserved, the output does not // depend on the JVM zone, and QueryParameterValue's validator accepts 'Z' but rejects "-05:00". @@ -271,7 +293,7 @@ void setNullParameter(int parameterIndex, Class declaredJavaType) { parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED); parameter.setScale(-1); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + recordBinding(); LOG.finest("Parameter set { %s }", parameter.toString()); } @@ -289,7 +311,7 @@ void setParameter(int parameterIndex, Object value, Class type) { parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED); parameter.setScale(-1); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + recordBinding(); LOG.finest("Parameter set { %s }", parameter.toString()); } @@ -385,7 +407,7 @@ void setParameter( parametersList.add(parameter); } - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + recordBinding(); LOG.finest("Parameter set { %s }", parameter.toString()); } @@ -411,7 +433,7 @@ void setParameter( parameter.setParamType(paramType); parameter.setScale(scale); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + recordBinding(); LOG.finest("Parameter set { %s }", parameter.toString()); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java index b035dd3f9333..81a13d080610 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java @@ -82,6 +82,8 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta Schema resultSchema = null; // Feeds getMetaData(). Schema insertSchema = null; // Feeds the Storage Write API. private TableName insertTableName = null; + // Per-object adoption signal for getMetaData(); see BigQueryParameterHandler#bindingRecorded. + private boolean resultSetMetadataRecorded; BigQueryPreparedStatement(BigQueryConnection connection, String query) throws SQLException { super(connection); @@ -97,6 +99,12 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta } } + @Override + void markInternal() { + super.markInternal(); + this.parameterHandler.markInternal(); + } + private int resolveParameterCount(String query, QueryStatistics queryStatistics) { if (queryStatistics != null) { return queryStatistics.getQueryParameters() == null @@ -426,6 +434,9 @@ public int[] executeBatch() throws SQLException { writeApiExecutionBuilder.setStatus( com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_SUCCESS); + // Success-only, matching BigQueryStatement.executeBatch; failures are already captured + // by the StatementExecution status recorded in the finally block. + recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); return insertArray; @@ -440,10 +451,17 @@ public int[] executeBatch() throws SQLException { .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) .setErrorCode(TelemetryManager.extractErrorCode(e)); throw new BigQueryJdbcException("Failed to execute batch with Write API", e); + } catch (SQLException | RuntimeException e) { + // bulkInsertWithWriteAPI throws BigQueryJdbcException itself, and the Storage Write API + // client throws unchecked ApiException; neither matched the catches above, so these + // failures used to be recorded as STATUS_UNSPECIFIED. + writeApiExecutionBuilder + .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) + .setErrorCode(TelemetryManager.extractErrorCode(e)); + throw e; } finally { long durationMs = System.currentTimeMillis() - startTime; - TelemetryManager.recordStatementExecution(writeApiExecutionBuilder, durationMs); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); + recordExecution(writeApiExecutionBuilder, durationMs); } } else { @@ -469,11 +487,10 @@ public int[] executeBatch() throws SQLException { getMoreResults(); i++; } + recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); return result; } catch (SQLException e) { throw new BigQueryJdbcException("SQL error during individual INSERT batch", e); - } finally { - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); } } } @@ -628,9 +645,12 @@ public ResultSetMetaData getMetaData() throws SQLException { return null; } - TelemetryManager.recordFeatureUsage( - DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, - "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + if (!this.resultSetMetadataRecorded && isRecordingTelemetry()) { + this.resultSetMetadataRecorded = true; + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + } return BigQueryResultSetMetadata.of(this.resultSchema.getFields(), this); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 5b010b000374..b6408da1bc06 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -161,6 +161,36 @@ public class BigQueryStatement extends BigQueryNoOpsStatement { protected StatementExecution.Builder currentExecutionBuilder = StatementExecution.newBuilder(); + /** + * False for statements the driver creates for its own purposes (connection validation, metadata + * queries). Those must not be reported as user activity, neither as feature usage nor as + * statement executions. + */ + private boolean recordTelemetry = true; + + /** Excludes this statement from usage telemetry. Called once, right after construction. */ + void markInternal() { + this.recordTelemetry = false; + } + + boolean isRecordingTelemetry() { + return this.recordTelemetry; + } + + /** Single choke point for execution records so internal statements never reach the batcher. */ + protected void recordExecution(StatementExecution.Builder builder, long durationMs) { + if (this.recordTelemetry) { + TelemetryManager.recordStatementExecution(builder, durationMs); + } + } + + /** Records {@code feature} unless this statement is internal. */ + protected void recordFeature(DriverFeature feature) { + if (this.recordTelemetry) { + TelemetryManager.recordFeatureUsage(feature); + } + } + static { BigQueryDaemonPollingTask.startGcDaemonTask( referenceQueueArrowRs, @@ -468,7 +498,7 @@ public void setQueryTimeout(int seconds) { public void cancel() throws SQLException { LOG.finer("Statement %s cancelled", this); cancelInternal(); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL); + recordFeature(DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL); } /** @@ -716,7 +746,7 @@ void runQuery(String query, QueryJobConfiguration jobConfiguration) throws SQLEx .setErrorCode(errorCode); } - TelemetryManager.recordStatementExecution(this.currentExecutionBuilder, durationMs); + recordExecution(this.currentExecutionBuilder, durationMs); } } @@ -1814,7 +1844,7 @@ private int[] executeBatchImpl(String combinedQueries) throws SQLException { clearBatch(); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); + recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); return result; } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java index 5c76648ca55a..44e0b825c644 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java @@ -25,9 +25,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; @@ -47,12 +49,16 @@ import com.google.cloud.bigquery.DataFormatOptions; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; import com.google.cloud.bigquery.JobStatistics.SessionInfo; import com.google.cloud.bigquery.Project; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryJobConfiguration.JobCreationMode; +import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.cloud.logging.Logging; @@ -960,4 +966,89 @@ public void testDuplicateSessionIdKeysAreRejected() { assertThrows(BigQueryJdbcException.class, () -> new BigQueryConnection(url)); assertTrue(ex.getMessage().contains("multiple 'session_id' entries")); } + + // --- Usage-telemetry attribution: driver-internal calls must not count as user activity --- + + private static final String TELEMETRY_TEST_URL = + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "OAuthType=2;ProjectId=MyBigQueryProject;" + + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" + + "OAuthClientSecret=redactedToken;"; + + @Test + public void testGetMetaData_recordsOncePerConnection() throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + connection.getMetaData(); + connection.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + times(1)); + } + } + + @Test + public void testGetMetaDataInternal_recordsNothingAndDoesNotConsumeUserRecord() + throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + connection.getMetaDataInternal(); + telemetry.verifyNoInteractions(); + // The first *user* call still records, even though the instance already exists. + connection.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + times(1)); + } + } + + @Test + public void testCreateStatementInternal_recordsNoFeatureUsage() throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + BigQueryStatement statement = connection.createStatementInternal(); + assertFalse(statement.isRecordingTelemetry()); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testPrepareStatementInternal_recordsNoFeatureUsageIncludingBindings() + throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + BigQueryPreparedStatement statement = connection.prepareStatementInternal("SELECT ?"); + statement.setString(1, "x"); + assertFalse(statement.isRecordingTelemetry()); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testIsValid_healthCheckRecordsNoTelemetry() throws Exception { + // Regression guard: isValid() runs "SELECT 1" on an internal statement. Pools call it on + // every checkout, so counting it as a user statement or SELECT would swamp real usage. + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL)) { + BigQuery mockBigQuery = mock(BigQuery.class); + connection.bigQuery = mockBigQuery; + TableResult tableResult = mock(TableResult.class); + when(tableResult.getSchema()).thenReturn(Schema.of()); + when(tableResult.getStatementType()).thenReturn(StatementType.SELECT); + when(mockBigQuery.queryWithTimeout(any(QueryJobConfiguration.class), any(), any())) + .thenReturn(tableResult); + + // CALLS_REAL_METHODS keeps TelemetryManager's pure helpers working; the record* calls + // are no-ops without an initialized instance but are still counted by verify. + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + connection.isValid(1); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any()), never()); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any(), any()), never()); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(any(), anyLong()), never()); + } + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java index 70dff53103df..f71bffa5cc48 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java @@ -42,7 +42,6 @@ import java.io.IOException; import java.io.InputStream; import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -3615,15 +3614,18 @@ public void testGetImportedKeys_noKeys() throws SQLException { @Test public void testGetExportedKeys_infoSchema() throws SQLException { - PreparedStatement mockStmt = mock(PreparedStatement.class); + BigQueryPreparedStatement mockStmt = mock(BigQueryPreparedStatement.class); ResultSet mockRs = mock(ResultSet.class); - when(bigQueryConnection.prepareStatement(anyString())).thenReturn(mockStmt); + when(bigQueryConnection.prepareStatementInternal(anyString())).thenReturn(mockStmt); when(mockStmt.executeQuery()).thenReturn(mockRs); ResultSet rs = dbMetadata.getExportedKeys("test-project", "dataset_p", "ref_table"); assertEquals(mockRs, rs); verify(mockStmt).closeOnCompletion(); verify(mockStmt).executeQuery(); + // Driver-internal metadata queries must not go through the user-facing, telemetry-recording + // prepareStatement. + verify(bigQueryConnection, never()).prepareStatement(anyString()); } @Test diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java index c98e8dfabb53..c7202cc103aa 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java @@ -22,9 +22,10 @@ import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.times; +import static org.mockito.Mockito.never; import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException; import com.google.cloud.bigquery.jdbc.telemetry.v1.AuthenticationType; import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; import com.google.cloud.bigquery.jdbc.telemetry.v1.Status; @@ -241,15 +242,16 @@ public void testConnect_optOut_noTelemetryRecorded() throws SQLException { } @Test - public void testConnect_recordsMetadataRetrievalOnce() throws SQLException { - // connect() logs four DatabaseMetaData values; it must fetch the object once, not four times. + public void testConnect_doesNotRecordMetadataRetrieval() throws SQLException { + // connect() reads DatabaseMetaData for its startup log line. That is driver-internal and + // must not be attributed to the user as DRIVER_FEATURE_METADATA_RETRIEVAL. try (MockedStatic telemetry = - mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { - bigQueryDriver.connect(VALID_TOKEN_URL, new Properties()); + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS); + Connection ignored = bigQueryDriver.connect(VALID_TOKEN_URL, new Properties())) { telemetry.verify( () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), - times(1)); + never()); } } @@ -261,7 +263,7 @@ public void testConnect_failedAttemptCarriesAuthType() { try (MockedStatic telemetry = mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { Assertions.assertThrows( - Exception.class, + BigQueryJdbcRuntimeException.class, () -> bigQueryDriver.connect( "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" @@ -281,7 +283,9 @@ public void testConnect_ioExceptionThrowsInsteadOfReturningNull() { // BigQueryJdbcRootLogger.setLevel is the only IOException source on the connect path. JDBC // requires connect() to throw for an accepted URL that fails, not return null. try (MockedStatic rootLogger = - mockStatic(BigQueryJdbcRootLogger.class)) { + mockStatic(BigQueryJdbcRootLogger.class); + MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { rootLogger .when(() -> BigQueryJdbcRootLogger.setLevel(any(), any())) .thenThrow(new IOException("simulated log path failure")); @@ -290,6 +294,13 @@ public void testConnect_ioExceptionThrowsInsteadOfReturningNull() { BigQueryJdbcException.class, () -> bigQueryDriver.connect(VALID_TOKEN_URL, new Properties())); assertThat(thrown.getCause()).isInstanceOf(IOException.class); + // The failure happens before the URL's auth type is resolved, hence UNSPECIFIED. + telemetry.verify( + () -> + TelemetryManager.recordConnectionAttempt( + eq(Status.STATUS_ERROR), + anyInt(), + eq(AuthenticationType.AUTHENTICATION_TYPE_UNSPECIFIED))); } } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java index 50b795be9470..d83dd5a457ef 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java @@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Mode; @@ -34,6 +36,8 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.jdbc.rules.TimeZoneRule; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.common.collect.ImmutableList; import com.google.common.io.BaseEncoding; import com.google.common.io.CharStreams; @@ -73,6 +77,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; public class BigQueryJsonResultSetTest { @@ -909,4 +914,37 @@ public void testTimestampPicosAccessorsAcrossJvmZones(String jvmZone) throws SQL TimeZone.setDefault(originalDefault); } } + + @Test + public void testGetMetaData_recordsFeatureOncePerResultSet() throws SQLException { + // Called inside row loops by many tools; the adoption signal must be recorded once. + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + bigQueryJsonResultSet.getMetaData(); + bigQueryJsonResultSet.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"), + times(1)); + } + } + + @Test + public void testGetMetaData_recordsEvenWhenOwningStatementIsInternal() throws SQLException { + // ResultSet.getMetaData() is only ever a user call, even on result sets produced by + // driver-internal statements (e.g. DatabaseMetaData.getExportedKeys), so it must count. + BigQueryStatement internalStatement = mock(BigQueryStatement.class); + BigQueryJsonResultSet internalResultSet = + BigQueryJsonResultSet.of(QUERY_SCHEMA, 1L, buffer, internalStatement, new Future[0]); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + internalResultSet.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"), + times(1)); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java index 3ea3b61b633e..419c46f68777 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java @@ -23,9 +23,17 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; @@ -37,7 +45,11 @@ import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; +import com.google.cloud.bigquery.jdbc.telemetry.v1.Status; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.gson.Gson; import com.google.gson.JsonNull; import com.google.gson.JsonObject; @@ -63,6 +75,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; public class BigQueryPreparedStatementSettersTest { @@ -539,16 +552,135 @@ public void testSetObjectLocalDateTime_zeroSecondsStillHasFullDigits() throws SQ } @Test - public void testPrimitiveSetters_recordParameterBindingFeature() throws SQLException { - // These setters use the (int, Object, Class) overload of BigQueryParameterHandler, which - // previously did not record DRIVER_FEATURE_PARAMETER_BINDING. One record per binding. + public void testSetters_recordParameterBindingFeatureOncePerStatement() throws SQLException { + // DRIVER_FEATURE_PARAMETER_BINDING is an adoption signal: one record per prepared statement, + // regardless of how many setX calls are made or which handler overload they use. try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { preparedStatement.setInt(1, 42); preparedStatement.setLong(2, 42L); preparedStatement.setString(3, "x"); + preparedStatement.setNull(4, Types.VARCHAR); telemetry.verify( () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING), - times(3)); + times(1)); + } + } + + @Test + public void testExecuteBatch_success_recordsBatchOperationsOnce() throws SQLException { + // Standard (non-Write-API) batch path. Stub the query run so no BigQuery client is needed. + BigQueryPreparedStatement spy = spy(preparedStatement); + doReturn(null).when(spy).getStandardBatchJobConfiguration(anyString()); + doNothing().when(spy).runQuery(anyString(), any()); + doReturn(-1).when(spy).getUpdateCount(); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + spy.executeBatch(); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS), + times(1)); + } + } + + @Test + public void testExecuteBatch_failure_doesNotRecordBatchOperations() throws SQLException { + // Regression guard: BATCH_OPERATIONS is success-only; the failure is already captured by + // the StatementExecution status, so recording the feature here would double-report it. + BigQueryPreparedStatement spy = spy(preparedStatement); + doReturn(null).when(spy).getStandardBatchJobConfiguration(anyString()); + doThrow(new BigQueryJdbcException("simulated batch failure")) + .when(spy) + .runQuery(anyString(), any()); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + assertThrows(BigQueryJdbcException.class, spy::executeBatch); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS), + never()); + } + } + + @Test + public void testExecuteBatch_writeApiFailure_recordsErrorStatusAndNoBatchFeature() + throws SQLException { + // Write API branch. bulkInsertWithWriteAPI throws BigQueryJdbcException when the statement + // has no insert metadata; that failure must be recorded as STATUS_ERROR (previously it fell + // through to STATUS_UNSPECIFIED) and must not count as a BATCH_OPERATIONS feature use. + BigQueryPreparedStatement spy = spy(preparedStatement); + doReturn(true).when(spy).useWriteAPI(); + when(connection.getBigQueryWriteClient()).thenReturn(mock(BigQueryWriteClient.class)); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + assertThrows(Exception.class, spy::executeBatch); + ArgumentCaptor execution = + ArgumentCaptor.forClass(StatementExecution.Builder.class); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(execution.capture(), anyLong()), + times(1)); + assertEquals(Status.STATUS_ERROR, execution.getValue().getStatus()); + assertEquals(QueryApiType.QUERY_API_TYPE_WRITE_API, execution.getValue().getQueryApiType()); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS), + never()); + } + } + + @Test + public void testExecuteBatch_internalStatement_recordsNothing() throws SQLException { + BigQueryPreparedStatement spy = spy(preparedStatement); + spy.markInternal(); + doReturn(null).when(spy).getStandardBatchJobConfiguration(anyString()); + doNothing().when(spy).runQuery(anyString(), any()); + doReturn(-1).when(spy).getUpdateCount(); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + spy.executeBatch(); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testGetMetaData_recordsResultSetMetadataRetrievalOncePerStatement() + throws SQLException { + preparedStatement.resultSchema = Schema.of(Field.of("c", StandardSQLTypeName.INT64)); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + preparedStatement.getMetaData(); + preparedStatement.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"), + times(1)); + } + } + + @Test + public void testGetMetaData_noResultSchema_returnsNullAndRecordsNothing() throws SQLException { + // JDBC: null for a statement that produces no result set. No metadata, no record. + preparedStatement.resultSchema = null; + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + assertNull(preparedStatement.getMetaData()); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testGetMetaData_internalStatement_recordsNothing() throws SQLException { + preparedStatement.resultSchema = Schema.of(Field.of("c", StandardSQLTypeName.INT64)); + preparedStatement.markInternal(); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + assertNotNull(preparedStatement.getMetaData()); + telemetry.verifyNoInteractions(); } } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index aff1a1d70d1a..64580b78bc3d 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -26,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -1732,6 +1733,48 @@ public void testCancel_recordsStatementCancelFeatureOnce() throws SQLException { () -> bigQueryStatement.cancel(), DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL, 1); } + @Test + public void testInternalStatement_recordsNoTelemetryForExecutionOrCancel() throws Exception { + // Statements the driver creates for itself (isValid, metadata queries) must not be reported + // as user activity: no StatementExecution, no feature usage. + TableResult tableResultMock = setupMockQueryResults(null, StatementType.SELECT, null); + BigQueryStatement internalStatement = new BigQueryStatement(bigQueryConnection); + internalStatement.markInternal(); + BigQueryStatement internalSpy = Mockito.spy(internalStatement); + doReturn(mock(BigQueryJsonResultSet.class)) + .when(internalSpy) + .processJsonResultSet(eq(tableResultMock), any()); + + // CALLS_REAL_METHODS keeps TelemetryManager's pure helpers (toStatementType etc.) working; + // the record* methods are no-ops without an initialized instance but are still counted. + try (MockedStatic telemetry = + Mockito.mockStatic(TelemetryManager.class, Mockito.CALLS_REAL_METHODS)) { + internalSpy.executeQuery("SELECT 1"); + internalSpy.cancel(); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(any(), anyLong()), Mockito.never()); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any()), Mockito.never()); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any(), any()), Mockito.never()); + } + } + + @Test + public void testUserStatement_recordsExecution() throws Exception { + // Control for the test above: the same flow on a regular statement does reach the batcher. + TableResult tableResultMock = setupMockQueryResults(null, StatementType.SELECT, null); + BigQueryStatement userSpy = Mockito.spy(new BigQueryStatement(bigQueryConnection)); + doReturn(mock(BigQueryJsonResultSet.class)) + .when(userSpy) + .processJsonResultSet(eq(tableResultMock), any()); + + try (MockedStatic telemetry = + Mockito.mockStatic(TelemetryManager.class, Mockito.CALLS_REAL_METHODS)) { + userSpy.executeQuery("SELECT 1"); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(any(), anyLong()), Mockito.times(1)); + } + } + /** Runs {@code action} and asserts how many times {@code feature} was recorded by it. */ private static void assertFeatureRecorded( SqlRunnable action, DriverFeature feature, int expectedCount) throws SQLException { From 5e6b578bde03071bf2999703672ec7ad7f8d0f74 Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 16 Nov 2026 17:44:21 -0500 Subject: [PATCH 3/5] chore(bigquery-jdbc): remove personal bigquery-jdbc-development skill from tracked skills It was swept into the main merge by accident; main tracks only the shared bigquery-jdbc-testing and bigquery-jdbc-prepare-pr skills. --- .../skills/bigquery-jdbc-development/SKILL.md | 399 ------------------ 1 file changed, 399 deletions(-) delete mode 100644 java-bigquery-jdbc/.agents/skills/bigquery-jdbc-development/SKILL.md diff --git a/java-bigquery-jdbc/.agents/skills/bigquery-jdbc-development/SKILL.md b/java-bigquery-jdbc/.agents/skills/bigquery-jdbc-development/SKILL.md deleted file mode 100644 index f6baf90244d8..000000000000 --- a/java-bigquery-jdbc/.agents/skills/bigquery-jdbc-development/SKILL.md +++ /dev/null @@ -1,399 +0,0 @@ ---- -name: bigquery-jdbc-development -description: >- - Module-specific guide for developing the google-cloud-bigquery-jdbc driver - (java-bigquery-jdbc). Use this skill when adding, modifying, debugging, or - testing code under java-bigquery-jdbc — including JDBC surface classes - (Connection, Statement, PreparedStatement, ResultSet, DatabaseMetaData), type - coercion, logging, OpenTelemetry instrumentation, connection-URL properties, - or when running the module's unit and integration tests. ---- - -# BigQuery JDBC Driver Development - -Module root: [java-bigquery-jdbc](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc) -Maven artifact: `google-cloud-bigquery-jdbc`. Packages: `com.google.cloud.bigquery.jdbc` (driver) -and `com.google.cloud.bigquery.exception` (exceptions). - -This is a **handwritten, split-repo** module. Nothing here is generated — edit source directly. - -> [!IMPORTANT] -> [DEVELOPMENT.md](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/DEVELOPMENT.md) -> has drifted in places (it references `BigQueryJdbcTypeMappings` and `BigQueryCoercion`, which -> do not exist). The class names in this skill reflect the current source. When in doubt, grep -> the source rather than trusting either document. - ---- - -## 1. Component Map - -```mermaid -graph TD - Client["Client App / BI Tool"] -->|DriverManager.getConnection| Driver[BigQueryDriver] - Driver -->|parse + validate URL| Url[BigQueryJdbcUrlUtility] - Driver -->|configure JUL| RootLog[BigQueryJdbcRootLogger] - Driver --> Conn[BigQueryConnection] - Conn -->|wrap every child object| Proxy[BigQueryJdbcContextProxy] - Proxy -->|ThreadLocal connectionId| Mdc[BigQueryJdbcMdc] - Conn --> Stmt[BigQueryStatement / PreparedStatement / CallableStatement] - Stmt --> Arrow[BigQueryArrowResultSet] - Stmt --> Json[BigQueryJsonResultSet] - Arrow --> Base[BigQueryBaseResultSet] - Json --> Base - Base -->|all conversions| Registry[BigQueryTypeRegistry] - Conn -->|spans + GCP log export| Otel[BigQueryJdbcOpenTelemetry] -``` - -### Key classes - -| Class | Role | -| :--- | :--- | -| [BigQueryDriver](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java) | `java.sql.Driver` entry point for `jdbc:bigquery://` URLs. | -| [BigQueryConnection](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java) | Session state, dataset defaults, BigQuery client, tracer, connection ID. | -| [BigQueryJdbcUrlUtility](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java) | Property-name constants, `VALID_PROPERTIES`, URL parsing with an LRU parse cache. | -| [BigQueryJdbcContextProxy](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcContextProxy.java) | Dynamic proxy: MDC scoping, `++entry++`/`++exit++` tracing, single-point exception logging, cascading re-wrap of child objects. | -| [BigQueryTypeRegistry](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java) | **The** conversion entry point: `convert(value, targetClass)` and `convert(value, bqType, targetClass, zone)`. | -| [BigQueryTemporalUtility](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTemporalUtility.java) | Date/Time/Timestamp/interval parsing and zone handling. | -| [BigQueryBaseResultSet](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java) | Shared getter logic for both Arrow and JSON result sets. | -| [BigQueryJdbcOpenTelemetry](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java) | SDK caching/ref-counting, span helper `withTracing(...)`, GCP log export. | - ---- - -## 2. Guardrails - -1. **Most restrictive visibility.** Many core classes are package-private (`class BigQueryJdbcContextProxy`). Only widen when a `java.sql` interface forces it. Prefer `@InternalApi` over bare `public`. -2. **Explicit imports only.** No wildcard imports, no inline fully-qualified names. -3. **Never use `java.util.logging.Logger` directly** in driver code — use `BigQueryJdbcCustomLogger` (see §4). -4. **Throw from `com.google.cloud.bigquery.exception`**, not bare `SQLException`, at public boundaries. Pick the specific subclass: - - `BigQueryJdbcException` — general checked JDBC error; has a `(String, BigQueryException)` ctor that formats the BQ error. - - `BigQueryJdbcSqlSyntaxErrorException` — malformed SQL. - - `BigQueryJdbcSqlFeatureNotSupportedException` — unsupported JDBC feature. - - `BigQueryConversionException` / `BigQueryJdbcCoercionException` / `BigQueryJdbcCoercionNotFoundException` — type coercion failures. - - `BigQueryJdbcRuntimeException` — unchecked, for paths that cannot declare `throws`. -5. **Do not mock final JDK types** (`BigDecimal`, `LocalDate`, `Instant`, `UUID`, `String`) with Mockito. Construct real instances. `mockito-inline` is on the classpath, but mocking final JDK classes is unstable on JDK 21+. -6. **Do not add external dependencies.** Prefer the JDK, then existing deps. Do not bump versions except for CVEs. -7. **Never break the public JDBC surface** without following [api-lifecycle](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/.agents/skills/api-lifecycle/SKILL.md) (`@ObsoleteApi` before `@Deprecated`). -8. Always follow java coding best practices. tott rules and module coding conventions. -9. Use blocks even for single statement if and loops. -10. Prefer early return. -11. Avoid unnecessary null checks. Always confirm there is a real possibility of a null value before adding the check. (This is about defensive checks — it does not relax the JDBC SQL-NULL handling required in §6.) -12. **Ignore the OAuth p12 test failure** — do not change driver code to chase it. Note that `ITAuthTests#testValidServiceAccountAuthenticationP12` is already `@Disabled` and needs `SA_SECRET_P12`, so a p12 failure is most likely `BigQueryJdbcOAuthUtilityTest#testPrivateKeyFromP12Bytes` or `#testPrivateKeyFromP12Bytes_wrong_password`, which load the `/fake.p12` test resource. -13. When adding or suggesting code, always include logs for debuggability — following the §4 conventions. "More logs" never means hand-written `++enter++`/`++exit++` on proxied methods or logging an exception that the proxy already logs. -14. When ever the code is changed, be performance aware and JDBC compliant. If a code is likely to affect either, notify me. -15. When writing code, always make sure real life use cases are considered and edge cases are dealt with. -16. **Never use jargon without explanation.** In code comments, docs, commit messages, and answers to me, define a term the first time it appears — spell out acronyms and say what the concept means in plain words. Write "safe publication (the guarantee that a thread seeing the object also sees its initialized fields)", not "safe publication". This applies to domain jargon (Clearcut, MDC, substrate) as well as Java/concurrency jargon. -17. **Confirm a failure is real before guarding against it, and keep fields `final`.** Do not add a `try`/`catch` around a helper that already handles its own failures, and do not make a field mutable or lazily initialized without a measured reason. See §8. - ---- - -## 3. Build & Test - -All tasks go through the module [Makefile](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/Makefile). Run from the module directory. - -```bash -make install # build + install into the local repo (delegates to .kokoro/common.sh) -make clean -make lint # spotify fmt-maven-plugin:format — ALWAYS run before finishing -``` - -### Resolving the module after upstream snapshot bumps - -The module inherits versions from `google-cloud-jar-parent`, which imports -`google-cloud-shared-dependencies` and, through it, the `sdk-platform-java` snapshots (gax, -java-core, auth, common-protos, ...). When the monorepo bumps those `-SNAPSHOT` versions, nothing in -`~/.m2` matches any more and every `mvn` / `make` invocation in this module fails during POM -resolution with errors like: - -``` -Non-resolvable import POM: com.google.cloud:google-cloud-shared-dependencies:pom:-SNAPSHOT -'dependencies.dependency.version' for com.google.api:gax:jar is missing -``` - -Do **not** fix this by building the whole monorepo or by hand-picking `-pl` / `-am` module lists. -Run the Kokoro helper from the **monorepo root** — it installs only the upstream modules the JDBC -driver needs, with tests and javadoc skipped: - -```bash -cd /usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java -source .kokoro/common.sh && install_modules java-bigquery-jdbc -``` - -Then return to `java-bigquery-jdbc/` and re-run the original `make` target. This step needs network -access (it may pull new third-party versions), so run it outside the sandbox. - -### Unit tests (surefire, JUnit 5) - -```bash -make unittest # all -make unittest test=BigQueryStatementTest # one class -make unittest test=BigQueryStatementTest#testExecute # one method -make unittest excludedGroups="known_issues" # skip tagged tests -``` - -### Integration tests (failsafe, real GCP) - -> [!WARNING] -> ITs hit live BigQuery and cost money. Never run the full IT suite speculatively — always scope -> with `test=`. They require `GOOGLE_APPLICATION_CREDENTIALS`; some also need `KMS_RESOURCE_PATH`, -> `SA_EMAIL`, or (for TPC) `ENDPOINT_URL` / `UNIVERSE_DOMAIN` / `PROJECT_ID`. - -```bash -export GOOGLE_APPLICATION_CREDENTIALS=/usr/local/google/home/neenushaji/Documents/gcp-keys/bq-devtools.json -export GOOGLE_CLOUD_PROJECT=bigquery-devtools-drivers -export SA_SECRET=/usr/local/google/home/neenushaji/Documents/gcp-keys/bq-devtools.json -make integration-test test=ITStatementTest#testBatchExecute -``` - -Defaults: `skipSurefire=true`, `skipShade=true`, and `-Dit.failIfNoSpecifiedTests=true` — a typo in -`test=` fails the build rather than silently passing. - -### Coverage and Docker - -```bash -make unit-test-coverage # jacoco report -> jacoco-unittests.zip -make full-coverage # ITs + unit tests -> jacoco-full.zip (expensive) -make docker-session # interactive shell if no local JDK/Maven -make docker-unittest test=BigQueryConnectionTest -``` - ---- - -## 4. Logging - -Two loggers, chosen by call-site frequency: - -- **`BigQueryJdbcCustomLogger`** — general driver code. -- **`BigQueryJdbcResultSetLogger`** — result-set hot paths only. Use `LOG.finestTrace("getInt")`; it is allocation-free when the level is disabled. - -### Instantiation - -```java -// Instance components (Connection, Statement, DatabaseMetaData) — include instance identity -private final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger(this.toString()); - -// Static / utility components — use the class name -private static final BigQueryJdbcCustomLogger LOG = - new BigQueryJdbcCustomLogger(BigQueryTypeRegistry.class.getName()); -``` - -### Rules - -1. **Never concatenate.** The logger takes printf-style varargs or a `Supplier`, both of - which skip work when the level is disabled. - ```java - LOG.fine("Executing on dataset: %s, table: %s", datasetId, tableId); // good - LOG.fine(() -> "Parsed: " + expensive.toDebugString()); // good for expensive work - LOG.fine("Executing on dataset: " + datasetId); // BAD — always allocates - ``` -2. **Do not hand-write `++enter++`/`++exit++`** on proxied methods. `BigQueryJdbcContextProxy` already - emits `++entry++`/`++exit++` at `FINER` for every method reached through the proxy. Add manual - tracing only for internal routines the proxy never sees. -3. **Do not log-and-rethrow.** The proxy is the unified exception logger and records each exception - exactly once with connection context. Logging again produces duplicates. -4. **Preserve MDC across threads.** Anything running off the calling thread (daemon pollers, - executors) must re-establish context: - ```java - try (BigQueryJdbcMdc.MdcCloseable mdc = BigQueryJdbcMdc.registerInstance(connectionId)) { - // ... - } - ``` - ---- - -## 5. OpenTelemetry - -Spans are created through the helper, never by hand: - -```java -return BigQueryJdbcOpenTelemetry.withTracing("BigQueryStatement.executeQuery", connection, sql, - () -> { /* work */ }); -``` - -`withTracing` sets `db.system`, `db.connection_id`, `db.application`, and `db.statement`, puts the -connection ID into OTel Baggage, records exceptions, sets `StatusCode.ERROR`, and always ends the -span. Use the `DB_*` constants on `BigQueryJdbcOpenTelemetry` for any extra attributes — do not -invent attribute keys. - -Lifecycle rules: -- SDKs are cached and **ref-counted** by `SdkCacheKey`. A connection that obtains one via - `getOpenTelemetry(...)` must call `releaseSdk(...)` on close, and `registerConnection(...)` / - `unregisterConnection(...)` must be symmetric — otherwise the SDK and Logging client leak. -- **Telemetry failures must never break JDBC.** Catch, log at `warning`, and continue. - -> [!TIP] -> Unit tests that touch instrumented code paths need a stubbed `Tracer`, or -> `BigQueryJdbcOpenTelemetry` will NPE. Use -> [OpenTelemetryTestUtility](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/OpenTelemetryTestUtility.java). - ---- - -## 6. JDBC Compliance - -### Null handling - -Distinguish **SQL NULL** (legal, returns a default) from **invalid coercion** (throws). - -- Validate impossible type conversions **up front**, before fetching, using the column's - `StandardSQLTypeName` — e.g. `getTimestamp` on an `INT64` column throws immediately. -- Primitive getters return the JDBC default on NULL; object getters return `null`: - ```java - Integer result = BigQueryTypeRegistry.convert(value, Integer.class); - return result != null ? result : 0; // getInt -> 0, getLong -> 0L, getDouble -> 0.0d - ``` - `getBoolean` returns `false`; `getString`/`getDate`/`getBigDecimal`/`getBytes` return `null`. -- Call `setWasNull(value)` so `wasNull()` is accurate — BI tools rely on it. -- Wrap coercion failures with `createCoercionException(columnIndex, TargetType.class, cause)`. - -### Other invariants - -- **Every getter and state-mutating method starts with `checkClosed()`.** -- **All conversion goes through `BigQueryTypeRegistry`.** Do not add ad-hoc casts or parsing in - result sets, statements, or metadata — add the mapping to the registry instead. -- **Wrap objects returned from public entry points** with `BigQueryJdbcContextProxy.wrap(obj, Iface.class, connectionId)`. - `ResultSet` is deliberately **not** proxied (JIT inlining on the hot iteration path) — keep it that way. -- Result sets are `TYPE_FORWARD_ONLY` + `CONCUR_READ_ONLY`. Unsupported operations throw - `BigQueryJdbcSqlFeatureNotSupportedException`; harmless no-ops (e.g. `setFetchSize`) should be - accepted so BI tools don't crash. -- Adding a connection property means updating **both** the constant and `VALID_PROPERTIES` in - `BigQueryJdbcUrlUtility`, plus `DataSource` and `getPropertyInfo` if user-visible. -- `SAFE_TO_LOG_PROPERTIES` lives in **`BigQueryConnection`**, not `BigQueryJdbcUrlUtility`. It is an - **allowlist**: at `CONFIG` level the connection logs every property, masking as `*****` anything - absent from the set. Omitting a new property is therefore the safe default — only add it if the - value can never carry a secret. Adding a credential-bearing property leaks it into the logs. - ---- - -## 7. Testing Conventions - -JUnit 5 (`org.junit.jupiter.*`) + Mockito. **Never** import `org.junit.Test` (JUnit 4). - -- Unit tests: `src/test/java/com/google/cloud/bigquery/jdbc/*Test.java` -- Integration tests: `.../jdbc/it/IT*.java`, extending - [ITBase](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBase.java) -- Shared helpers: `.../jdbc/utils/` (`TestUtilities`, `URIBuilder`, `ArrowUtilities`) - -Extend [BigQueryJdbcBaseTest](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBaseTest.java) -for connection-URL tests and build URLs with its helpers rather than string literals: - -```java -String url = getUriOAuthServiceAccount().append("ProjectId", "my-project").toString(); -``` - -It also exposes `fake_pkcs8_key`, a throwaway key for auth tests — never add a real credential. - -**Tags** (`@Tag`), excluded via `excludedGroups=`: - -| Tag | Meaning | -| :--- | :--- | -| `disable_tpc` | Cannot run against Trusted Partner Cloud universes. | -| `advanced` | Needs elevated permissions or extra setup. | -| `known_issues` | Known-failing; excluded in CI. | - -Other expectations: -- Use `@ParameterizedTest` + `@MethodSource`/`@CsvSource`/`@ValueSource` for coercion matrices. -- No `UnnecessaryStubbingException` warnings — remove stubs the test doesn't use. -- Prefer `Assertions.assertThrows` and assert on the **specific** exception subclass. -- Every new branch needs a test; every bug fix needs a regression test. - ---- - -## 8. Defensive Code & Concurrency - -### 8.1 Don't duplicate a guarantee the callee already makes - -Some helpers are deliberately **total** — meaning they always return a value and never throw, -falling back to something harmless (`"unknown"`, a random UUID, an empty list) instead of failing. -Wrapping a total helper in `try`/`catch` at the call site adds no safety. It copies the guarantee to -a second place where no test covers it, and obscures the fact that the callee is meant to be total. - -Before adding a guard, trace every leaf call and ask whether it can actually throw. If you find a -real hole, **fix it in the callee**, so every caller benefits and the contract stays in one tested -place. - -Worked example: -[DriverEnvironmentBuilder.build()](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/telemetry/v1/DriverEnvironmentBuilder.java#L42-L56) -catches `SecurityException` around each system-property read and degrades to `"restricted"` / -`"unknown"`, and wraps telemetry-tag file I/O in `catch (Exception)` / `catch (IOException)`. It is -total by construction, so a `catch (Throwable)` wrapper at the call site is dead code. The single -real gap was `Paths.get(userHome, ...)`, which throws `InvalidPathException` — an unchecked -`IllegalArgumentException` that the method's `catch (SecurityException)` does not cover. That fix -belongs in the builder, not at the call site. - -This is guardrail 11 (avoid unnecessary null checks) applied to exceptions: confirm the failure is -possible before writing code to handle it. - -### 8.2 Know whether your caller swallows or rethrows - -"Telemetry must never break JDBC" holds only if something actually stops the exception. -[BigQueryDriver.connect](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java#L232-L235) -ends in `catch (Throwable t) { recordConnectionAttempt(...); throw t; }` — it records and -**rethrows**. Anything that throws on the connect path therefore fails the user's -`getConnection()`. Worse, `TelemetryManager.getInstance` publishes its singleton only after -successful construction, so a throw there repeats on every subsequent connect for the life of the -JVM. - -`TelemetryManager.runSafely` is the seam that genuinely contains telemetry failures (it catches -`Throwable`). A bare call is not. Check the enclosing handler before assuming a failure is contained. - -### 8.3 Prefer `final` over lazy initialization - -Keep fields `final` and assign them in the constructor unless there is a measured reason not to. - -This is not style. The Java Memory Model — the rules that say when one thread is guaranteed to see -another thread's writes — gives final fields **safe publication**: once the constructor finishes, -any thread that can see the object is guaranteed to see its correctly initialized final fields, with -no synchronization required. Remove `final` and that guarantee downgrades to a convention a future -edit can silently break, producing a **data race** — two threads accessing the same field with no -ordering between them, where one can read a stale or half-written value. - -Costs of going lazy that are easy to overlook: - -| Cost | Why it bites | -| :--- | :--- | -| Lost compiler-checked invariant | "Assigned once, always visible" stops being enforced by `javac` and becomes a comment. | -| `null` gains two meanings | It now means "not resolved yet" *or* "legitimately absent", so you need a second `resolved` flag — otherwise the absent case retries the expensive work on every pass, forever. | -| Behavior change | Work happens on first use rather than at construction, so the first operation can observe a half-configured object. | - -Performance is usually **not** the deciding cost, so don't justify laziness with it. On x86-64 a -`volatile` read — a read the JVM may not reorder or cache in a CPU register — compiles to the same -instruction as an ordinary read; only the write needs a barrier. `TelemetryBatcher.driverEnvironment` -is read once per flush, that is once per `DEFAULT_TELEMETRY_UPLOAD_INTERVAL_VALUE` (5 minutes). -Optimizing that read is meaningless. - -### 8.4 If a field must be mutable, reuse the existing lock - -Don't reach for `volatile` reflexively. If every read and write already happens inside one lock, the -lock supplies the visibility and a plain field is correct — `ReentrantLock` establishes the same -ordering guarantee as `synchronized`. Mark it `@GuardedBy("lockName")` -(`javax.annotation.concurrent.GuardedBy`, already used in -[BigQueryJdbcBulkInsertWriter](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java#L46)) -so the requirement is documented rather than assumed. - -Add `volatile` only when a field is genuinely read outside the lock. - -### 8.5 Put one-time work where it runs once - -`TelemetryConfiguration.Builder.resolveProperties` runs on **every** `getConnection()` — it is -called merely to read the telemetry opt-out flag. Per-JVM setup such as filesystem reads or -environment detection belongs in the singleton-creation block of `TelemetryManager.getInstance`, -which runs once. Placing it in `resolveProperties` would add a file access to every connection. - -Before adding work to a method, check how often it is actually called. - ---- - -## 9. Pre-PR Checklist - -- [ ] `make lint` run and clean. -- [ ] `make unittest` passes, with no unnecessary-stubbing warnings. -- [ ] Narrowest possible visibility; no wildcard imports or inline FQNs. -- [ ] Exceptions come from `com.google.cloud.bigquery.exception` and are the specific subclass. -- [ ] No defensive `try`/`catch` or null check added without confirming the failure is possible; fields left `final` unless there is a stated reason (§8). -- [ ] Logging uses the custom loggers with format args or suppliers — no concatenation, no duplicate exception logging. -- [ ] `checkClosed()` at the top of new getters/mutators; `setWasNull(...)` where relevant. -- [ ] New conversions live in `BigQueryTypeRegistry`; new properties registered in `VALID_PROPERTIES` (`BigQueryJdbcUrlUtility`) and, **only if the value is never sensitive**, in `SAFE_TO_LOG_PROPERTIES` (`BigQueryConnection`). -- [ ] Objects returned from public entry points are proxy-wrapped (except `ResultSet`). -- [ ] OTel SDK acquire/release and connection register/unregister are symmetric. -- [ ] Public API changes follow [api-lifecycle](file:///usr/local/google/home/neenushaji/IdeaProjects/google-cloud-java/.agents/skills/api-lifecycle/SKILL.md). -- [ ] Conventional commit: `feat(bigquery-jdbc): ...` / `fix(bigquery-jdbc): ...`. -- [ ] Consider whether the change must be mirrored to the split repo. From a3d8e857cc268c8361e7b49d7f1e049d9b70bd0d Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 16 Nov 2026 17:54:45 -0500 Subject: [PATCH 4/5] make internal methods sync --- .../bigquery/jdbc/BigQueryConnection.java | 12 +++++- .../bigquery/jdbc/BigQueryConnectionTest.java | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 3ff1044a9ae8..8d8b71a6a873 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -86,6 +86,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; +import javax.annotation.concurrent.GuardedBy; /** * An implementation of {@link java.sql.Connection} for establishing a connection with BigQuery and @@ -250,7 +251,10 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { private Context otelContext; Tracer tracer = OpenTelemetry.noop().getTracer(BigQueryJdbcOpenTelemetry.INSTRUMENTATION_SCOPE_NAME); - DatabaseMetaData databaseMetaData; + + @GuardedBy("this") + private DatabaseMetaData databaseMetaData; + // A Connection may be used from several threads; compareAndSet keeps the adoption signal at // exactly one record per connection without a lock. private final AtomicBoolean metadataRetrievalRecorded = new AtomicBoolean(); @@ -1075,8 +1079,12 @@ public DatabaseMetaData getMetaData() throws SQLException { /** * Same instance as {@link #getMetaData()} without the usage-telemetry record. Driver-internal. + * + *

Synchronized: a Connection may be shared across threads, and without the lock two threads + * could each construct a BigQueryDatabaseMetaData, or one could observe a partially constructed + * instance. The lock is uncontended after first use and this is not a hot path. */ - DatabaseMetaData getMetaDataInternal() { + synchronized DatabaseMetaData getMetaDataInternal() { if (databaseMetaData == null) { databaseMetaData = new BigQueryDatabaseMetaData(this); } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java index 1dbbadfeb074..af40a05585eb 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java @@ -77,13 +77,23 @@ import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.sql.DatabaseMetaData; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; @@ -1133,6 +1143,36 @@ private static JsonObject extractBearerJwtPayload(Credentials credentials, URI u + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" + "OAuthClientSecret=redactedToken;"; + @Test + public void testGetMetaData_concurrentFirstCallsShareOneInstance() throws Exception { + // getMetaDataInternal() lazily creates the DatabaseMetaData; concurrent first calls must + // observe a single instance (the method is synchronized). The once-per-connection telemetry + // record is covered by testGetMetaData_recordsOncePerConnection: Mockito static mocks are + // thread-scoped, so they cannot be verified across the worker threads used here. + int threads = 8; + ExecutorService pool = Executors.newFixedThreadPool(threads); + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL)) { + CountDownLatch start = new CountDownLatch(1); + List> results = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + results.add( + pool.submit( + () -> { + start.await(); + return connection.getMetaData(); + })); + } + start.countDown(); + Set distinct = Collections.newSetFromMap(new IdentityHashMap<>()); + for (Future f : results) { + distinct.add(f.get(10, TimeUnit.SECONDS)); + } + assertEquals(1, distinct.size()); + } finally { + pool.shutdownNow(); + } + } + @Test public void testGetMetaData_recordsOncePerConnection() throws SQLException { try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); From 4ee599d8f48a05e78f646fe922fcd3027d5f75cb Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 16 Nov 2026 19:54:02 -0500 Subject: [PATCH 5/5] bound IT JVM memory and right-size the script IT --- java-bigquery-jdbc/pom.xml | 11 ++++++++++- .../bigquery/jdbc/it/ITStatementTest.java | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/java-bigquery-jdbc/pom.xml b/java-bigquery-jdbc/pom.xml index ff5f6a1e8739..199339beb33f 100644 --- a/java-bigquery-jdbc/pom.xml +++ b/java-bigquery-jdbc/pom.xml @@ -65,6 +65,14 @@ org.apache.maven.plugins maven-failsafe-plugin + + -Xmx1536m -XX:MaxDirectMemorySize=1g -XX:+ExitOnOutOfMemoryError true @@ -520,7 +528,8 @@ org.apache.maven.plugins maven-failsafe-plugin - --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + -Xmx1536m -XX:MaxDirectMemorySize=1g -XX:+ExitOnOutOfMemoryError --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java index 3fb7a816cf5e..663bd025e4db 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java @@ -229,9 +229,15 @@ public void testExecuteUpdate() throws SQLException { public void testScript() throws SQLException { Properties withReadApi = new Properties(); withReadApi.setProperty("EnableHighThroughputAPI", "1"); - String longQuery = String.format(GENERATE_ARRAY_QUERY, 500000); - String longerQuery = String.format(GENERATE_ARRAY_QUERY, 700000); - String longerQuery2 = String.format(GENERATE_ARRAY_QUERY, 900000); + // Each child result must exceed the Read API activation point (>= HighThroughputMinTableSize + // rows, default 10,000, and > HighThroughputActivationRatio pages, default 2, at the default + // MaxResults page size of 10,000), so >= 30,000 rows per child takes the Arrow path. Keep the + // counts modest: the presubmit suite runs all IT classes in one JVM and Arrow buffers are + // off-heap, so multi-million-row scripts have pushed the build container past its memory + // limit (exit 137). + String longQuery = String.format(GENERATE_ARRAY_QUERY, 30000); + String longerQuery = String.format(GENERATE_ARRAY_QUERY, 40000); + String longerQuery2 = String.format(GENERATE_ARRAY_QUERY, 50000); try (Connection connection = DriverManager.getConnection(ITBase.connectionUrl, withReadApi); Statement statement = connection.createStatement()) { @@ -239,17 +245,17 @@ public void testScript() throws SQLException { ResultSet arrowResultSet = statement.getResultSet(); assertArrowResultSet(arrowResultSet); - assertEquals(500000, resultSetRowCount(arrowResultSet)); + assertEquals(30000, resultSetRowCount(arrowResultSet)); assertTrue(statement.getMoreResults()); arrowResultSet = statement.getResultSet(); assertArrowResultSet(arrowResultSet); - assertEquals(700000, resultSetRowCount(arrowResultSet)); + assertEquals(40000, resultSetRowCount(arrowResultSet)); assertTrue(statement.getMoreResults()); arrowResultSet = statement.getResultSet(); assertArrowResultSet(arrowResultSet); - assertEquals(900000, resultSetRowCount(arrowResultSet)); + assertEquals(50000, resultSetRowCount(arrowResultSet)); assertFalse(statement.getMoreResults()); }