diff --git a/java-bigquery-jdbc/pom.xml b/java-bigquery-jdbc/pom.xml index e4d8e8a3b036..199339beb33f 100644 --- a/java-bigquery-jdbc/pom.xml +++ b/java-bigquery-jdbc/pom.xml @@ -65,8 +65,18 @@ org.apache.maven.plugins maven-failsafe-plugin + + -Xmx1536m -XX:MaxDirectMemorySize=1g -XX:+ExitOnOutOfMemoryError true + + false @@ -518,7 +528,8 @@ org.apache.maven.plugins maven-failsafe-plugin - --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + -Xmx1536m -XX:MaxDirectMemorySize=1g -XX:+ExitOnOutOfMemoryError --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java index 97e04b6d9564..4dbfbe2deea7 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java @@ -71,6 +71,9 @@ public abstract class BigQueryBaseResultSet extends BigQueryNoOpsResultSet protected final boolean isNested; protected boolean isClosed = false; protected boolean wasNull = false; + // getMetaData() is commonly called inside row loops; record the adoption signal once per + // result set. A ResultSet is single-threaded by JDBC contract, so a plain field suffices. + private boolean metadataRetrievalRecorded; private int fetchSize = -1; private Job job; private SQLWarning warnings; @@ -268,9 +271,15 @@ public ResultSetMetaData getMetaData() throws SQLException { } } - TelemetryManager.recordFeatureUsage( - DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, - "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + // Always a user call: no driver code invokes ResultSet.getMetaData(), so this is recorded + // regardless of whether the owning statement is internal (e.g. the result set that + // DatabaseMetaData.getExportedKeys hands back to the user). + if (!this.metadataRetrievalRecorded) { + this.metadataRetrievalRecorded = true; + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + } return BigQueryJdbcContextProxy.wrap(metaData, ResultSetMetaData.class, connectionId); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index f26751102db7..8d8b71a6a873 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -83,8 +83,10 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; +import javax.annotation.concurrent.GuardedBy; /** * An implementation of {@link java.sql.Connection} for establishing a connection with BigQuery and @@ -249,7 +251,13 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { private Context otelContext; Tracer tracer = OpenTelemetry.noop().getTracer(BigQueryJdbcOpenTelemetry.INSTRUMENTATION_SCOPE_NAME); - DatabaseMetaData databaseMetaData; + + @GuardedBy("this") + private DatabaseMetaData databaseMetaData; + + // A Connection may be used from several threads; compareAndSet keeps the adoption signal at + // exactly one record per connection without a lock. + private final AtomicBoolean metadataRetrievalRecorded = new AtomicBoolean(); Boolean reqGoogleDriveScope; private final Properties clientInfo = new Properties(); private boolean isReadOnlyTokenUsed = false; @@ -511,6 +519,34 @@ public Statement createStatement() throws SQLException { return currentStatement; } + /** + * Creates a {@code Statement} for the driver's own use (for example the {@code isValid} + * health-check query). Identical to {@link #createStatement()} except that neither the statement + * nor anything it executes is reported as user activity in usage telemetry. + */ + BigQueryStatement createStatementInternal() throws SQLException { + checkClosed(); + BigQueryStatement statement = new BigQueryStatement(this); + statement.markInternal(); + LOG.fine("Internal Statement %s created.", statement); + addOpenStatements(statement); + return statement; + } + + /** + * Creates a {@code PreparedStatement} for the driver's own metadata queries. Identical to {@link + * #prepareStatement(String)} except that neither the statement, its parameter bindings, nor + * anything it executes is reported as user activity in usage telemetry. + */ + BigQueryPreparedStatement prepareStatementInternal(String sql) throws SQLException { + checkClosed(); + BigQueryPreparedStatement statement = new BigQueryPreparedStatement(this, sql); + statement.markInternal(); + LOG.fine("Internal Prepared Statement %s created.", statement); + addOpenStatements(statement); + return statement; + } + /** * Creates and returns a new {@code Statement} object for executing BigQuery SQL queries. This * method is similar to {@link BigQueryConnection#createStatement()}, but it overrides the type @@ -883,7 +919,8 @@ public boolean isValid(int timeout) throws SQLException { throw new BigQueryJdbcException("timeout must be >= 0"); } if (!isClosed()) { - try (Statement statement = createStatement(); + // Internal statement: a pool health-check must not count as user statements or SELECTs. + try (Statement statement = createStatementInternal(); ResultSet rs = statement.executeQuery("SELECT 1")) { LOG.finer("Running validation query"); if (rs.next()) { @@ -1031,12 +1068,26 @@ private void rollbackImpl() throws SQLException { @Override public DatabaseMetaData getMetaData() throws SQLException { + DatabaseMetaData metaData = getMetaDataInternal(); + // Adoption signal, recorded once per connection on the first user call. Tied to this flag + // rather than to instance creation so the driver's own startup lookup does not consume it. + if (this.metadataRetrievalRecorded.compareAndSet(false, true)) { + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL); + } + return metaData; + } + + /** + * Same instance as {@link #getMetaData()} without the usage-telemetry record. Driver-internal. + * + *

Synchronized: a Connection may be shared across threads, and without the lock two threads + * could each construct a BigQueryDatabaseMetaData, or one could observe a partially constructed + * instance. The lock is uncontended after first use and this is not a hot path. + */ + synchronized DatabaseMetaData getMetaDataInternal() { if (databaseMetaData == null) { databaseMetaData = new BigQueryDatabaseMetaData(this); } - - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL); - return databaseMetaData; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 964ef6f69878..df8218ea74fa 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -2610,7 +2610,7 @@ public ResultSet getExportedKeys(String catalog, String schema, String table) validateSchemaIdentifier(schema); String sql = getExportedKeysSqlContent(); String formattedSql = replaceSqlParameters(sql, catalog, schema, table); - PreparedStatement stmt = this.connection.prepareStatement(formattedSql); + PreparedStatement stmt = this.connection.prepareStatementInternal(formattedSql); if (stmt == null) { throw new BigQueryJdbcException("Failed to prepare statement for getExportedKeys"); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java index 204d3fc4e2ee..b34900b1aa63 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java @@ -27,6 +27,7 @@ import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.Driver; import java.sql.DriverManager; import java.sql.DriverPropertyInfo; @@ -196,10 +197,12 @@ public Connection connect(String url, Properties info) throws SQLException { LOG.log(Level.SEVERE, "Failed to parse connection URL", e); throw new BigQueryJdbcException("Failed to parse connection URL", e); } + authType = TelemetryManager.toAuthenticationType(ds.getOAuthType()); if (customOpenTelemetryObj instanceof OpenTelemetry) { ds.setCustomOpenTelemetry((OpenTelemetry) customOpenTelemetryObj); } BigQueryConnection connection = new BigQueryConnection(connectionUri, ds); + DatabaseMetaData metaData = connection.getMetaDataInternal(); LOG.info( "Driver info : { {Database Product Name : %s}, " + "{Database Product Version : %s}, " @@ -208,17 +211,16 @@ public Connection connect(String url, Properties info) throws SQLException { + "{LogLevel : %s}, " + "{LogPath : %s}, " + "{Driver Instance : %s} }", - connection.getMetaData().getDatabaseProductName(), - connection.getMetaData().getDatabaseProductVersion(), - connection.getMetaData().getDriverName(), - connection.getMetaData().getDriverVersion(), + metaData.getDatabaseProductName(), + metaData.getDatabaseProductVersion(), + metaData.getDriverName(), + metaData.getDriverVersion(), logLevel, logPath, this.toString()); Connection wrapped = BigQueryJdbcContextProxy.wrap(connection, Connection.class); - authType = TelemetryManager.toAuthenticationType(ds.getOAuthType()); TelemetryManager.recordConnectionAttempt(Status.STATUS_SUCCESS, 0, authType); return wrapped; } else { @@ -227,8 +229,7 @@ public Connection connect(String url, Properties info) throws SQLException { } catch (IOException e) { TelemetryManager.recordConnectionAttempt( Status.STATUS_ERROR, TelemetryManager.extractErrorCode(e), authType); - LOG.warning("Getting a warning: %s", e.getMessage()); - return null; + throw new BigQueryJdbcException("Failed to establish connection", e); } catch (Throwable t) { TelemetryManager.recordConnectionAttempt( Status.STATUS_ERROR, TelemetryManager.extractErrorCode(t), authType); diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java index 6e65e978880c..d2edbfea4909 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java @@ -45,6 +45,28 @@ class BigQueryParameterHandler { new BigQueryJdbcCustomLogger(BigQueryParameterHandler.class.getName()); private final int parametersArraySize; private final boolean enableTimestampPicos; + + /** False when the owning statement is driver-internal; see BigQueryStatement#markInternal. */ + private boolean recordTelemetry = true; + + /** + * DRIVER_FEATURE_PARAMETER_BINDING is an adoption signal, recorded once per handler (i.e. per + * prepared statement) rather than once per setX call. Handlers are confined to one statement, + * which JDBC defines as single-threaded, so a plain field is sufficient. + */ + private boolean bindingRecorded; + + void markInternal() { + this.recordTelemetry = false; + } + + private void recordBinding() { + if (this.recordTelemetry && !this.bindingRecorded) { + this.bindingRecorded = true; + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + } + } + final ArrayList parametersList; // TIMESTAMP is rendered in UTC with a trailing 'Z': the instant is preserved, the output does not // depend on the JVM zone, and QueryParameterValue's validator accepts 'Z' but rejects "-05:00". @@ -271,7 +293,7 @@ void setNullParameter(int parameterIndex, Class declaredJavaType) { parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED); parameter.setScale(-1); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + recordBinding(); LOG.finest("Parameter set { %s }", parameter.toString()); } @@ -289,6 +311,8 @@ void setParameter(int parameterIndex, Object value, Class type) { parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED); parameter.setScale(-1); + recordBinding(); + LOG.finest("Parameter set { %s }", parameter.toString()); } @@ -383,7 +407,7 @@ void setParameter( parametersList.add(parameter); } - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + recordBinding(); LOG.finest("Parameter set { %s }", parameter.toString()); } @@ -409,7 +433,7 @@ void setParameter( parameter.setParamType(paramType); parameter.setScale(scale); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING); + recordBinding(); LOG.finest("Parameter set { %s }", parameter.toString()); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java index 6b90421d5d3d..24571e0480b6 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java @@ -16,8 +16,6 @@ package com.google.cloud.bigquery.jdbc; -import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; -import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.common.annotations.VisibleForTesting; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -237,7 +235,6 @@ public void rollback() throws SQLException { @Override public DatabaseMetaData getMetaData() throws SQLException { - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL); return bqConnectionDelegate.getMetaData(); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java index b035dd3f9333..81a13d080610 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java @@ -82,6 +82,8 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta Schema resultSchema = null; // Feeds getMetaData(). Schema insertSchema = null; // Feeds the Storage Write API. private TableName insertTableName = null; + // Per-object adoption signal for getMetaData(); see BigQueryParameterHandler#bindingRecorded. + private boolean resultSetMetadataRecorded; BigQueryPreparedStatement(BigQueryConnection connection, String query) throws SQLException { super(connection); @@ -97,6 +99,12 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta } } + @Override + void markInternal() { + super.markInternal(); + this.parameterHandler.markInternal(); + } + private int resolveParameterCount(String query, QueryStatistics queryStatistics) { if (queryStatistics != null) { return queryStatistics.getQueryParameters() == null @@ -426,6 +434,9 @@ public int[] executeBatch() throws SQLException { writeApiExecutionBuilder.setStatus( com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_SUCCESS); + // Success-only, matching BigQueryStatement.executeBatch; failures are already captured + // by the StatementExecution status recorded in the finally block. + recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); return insertArray; @@ -440,10 +451,17 @@ public int[] executeBatch() throws SQLException { .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) .setErrorCode(TelemetryManager.extractErrorCode(e)); throw new BigQueryJdbcException("Failed to execute batch with Write API", e); + } catch (SQLException | RuntimeException e) { + // bulkInsertWithWriteAPI throws BigQueryJdbcException itself, and the Storage Write API + // client throws unchecked ApiException; neither matched the catches above, so these + // failures used to be recorded as STATUS_UNSPECIFIED. + writeApiExecutionBuilder + .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) + .setErrorCode(TelemetryManager.extractErrorCode(e)); + throw e; } finally { long durationMs = System.currentTimeMillis() - startTime; - TelemetryManager.recordStatementExecution(writeApiExecutionBuilder, durationMs); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); + recordExecution(writeApiExecutionBuilder, durationMs); } } else { @@ -469,11 +487,10 @@ public int[] executeBatch() throws SQLException { getMoreResults(); i++; } + recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); return result; } catch (SQLException e) { throw new BigQueryJdbcException("SQL error during individual INSERT batch", e); - } finally { - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); } } } @@ -628,9 +645,12 @@ public ResultSetMetaData getMetaData() throws SQLException { return null; } - TelemetryManager.recordFeatureUsage( - DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, - "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + if (!this.resultSetMetadataRecorded && isRecordingTelemetry()) { + this.resultSetMetadataRecorded = true; + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"); + } return BigQueryResultSetMetadata.of(this.resultSchema.getFields(), this); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 521a4f4b0910..20e0c72cd161 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -161,6 +161,36 @@ public class BigQueryStatement extends BigQueryNoOpsStatement { protected StatementExecution.Builder currentExecutionBuilder = StatementExecution.newBuilder(); + /** + * False for statements the driver creates for its own purposes (connection validation, metadata + * queries). Those must not be reported as user activity, neither as feature usage nor as + * statement executions. + */ + private boolean recordTelemetry = true; + + /** Excludes this statement from usage telemetry. Called once, right after construction. */ + void markInternal() { + this.recordTelemetry = false; + } + + boolean isRecordingTelemetry() { + return this.recordTelemetry; + } + + /** Single choke point for execution records so internal statements never reach the batcher. */ + protected void recordExecution(StatementExecution.Builder builder, long durationMs) { + if (this.recordTelemetry) { + TelemetryManager.recordStatementExecution(builder, durationMs); + } + } + + /** Records {@code feature} unless this statement is internal. */ + protected void recordFeature(DriverFeature feature) { + if (this.recordTelemetry) { + TelemetryManager.recordFeatureUsage(feature); + } + } + static { BigQueryDaemonPollingTask.startGcDaemonTask( referenceQueueArrowRs, @@ -406,7 +436,7 @@ public void close() throws SQLException { boolean cancelSucceeded = false; try { - cancel(); // This attempts to cancel jobs and calls closeStatementResources() + cancelInternal(); // Cancels jobs and calls closeStatementResources(); no telemetry. cancelSucceeded = true; } catch (SQLException e) { LOG.warning("Failed to cancel statement during close().", e); @@ -467,6 +497,16 @@ public void setQueryTimeout(int seconds) { @Override public void cancel() throws SQLException { LOG.finer("Statement %s cancelled", this); + cancelInternal(); + recordFeature(DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL); + } + + /** + * Cancels running jobs and releases statement resources without recording telemetry. Used by + * {@link #close()}, where cancellation is housekeeping, not a user-initiated cancel. + */ + private void cancelInternal() throws SQLException { + LOG.finer("Statement %s cancelled internally", this); synchronized (cancelLock) { this.isCanceled = true; for (JobId jobId : this.jobIds) { @@ -488,8 +528,6 @@ public void cancel() throws SQLException { // If a ResultSet exists, then it will be closed as well, closing the // ownedThreads closeStatementResources(); - - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL); } @Override @@ -708,7 +746,7 @@ void runQuery(String query, QueryJobConfiguration jobConfiguration) throws SQLEx .setErrorCode(errorCode); } - TelemetryManager.recordStatementExecution(this.currentExecutionBuilder, durationMs); + recordExecution(this.currentExecutionBuilder, durationMs); } } @@ -1820,7 +1858,7 @@ private int[] executeBatchImpl(String combinedQueries) throws SQLException { clearBatch(); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); + recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); return result; } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java index fd82c6babb62..af40a05585eb 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java @@ -25,9 +25,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; @@ -49,12 +51,16 @@ import com.google.cloud.bigquery.DataFormatOptions; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; import com.google.cloud.bigquery.JobStatistics.SessionInfo; import com.google.cloud.bigquery.Project; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryJobConfiguration.JobCreationMode; +import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryReadSettings; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; @@ -71,13 +77,23 @@ import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.sql.DatabaseMetaData; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; @@ -1118,4 +1134,119 @@ private static JsonObject extractBearerJwtPayload(Credentials credentials, URI u new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); return JsonParser.parseString(payloadJson).getAsJsonObject(); } + + // --- Usage-telemetry attribution: driver-internal calls must not count as user activity --- + + private static final String TELEMETRY_TEST_URL = + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "OAuthType=2;ProjectId=MyBigQueryProject;" + + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" + + "OAuthClientSecret=redactedToken;"; + + @Test + public void testGetMetaData_concurrentFirstCallsShareOneInstance() throws Exception { + // getMetaDataInternal() lazily creates the DatabaseMetaData; concurrent first calls must + // observe a single instance (the method is synchronized). The once-per-connection telemetry + // record is covered by testGetMetaData_recordsOncePerConnection: Mockito static mocks are + // thread-scoped, so they cannot be verified across the worker threads used here. + int threads = 8; + ExecutorService pool = Executors.newFixedThreadPool(threads); + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL)) { + CountDownLatch start = new CountDownLatch(1); + List> results = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + results.add( + pool.submit( + () -> { + start.await(); + return connection.getMetaData(); + })); + } + start.countDown(); + Set distinct = Collections.newSetFromMap(new IdentityHashMap<>()); + for (Future f : results) { + distinct.add(f.get(10, TimeUnit.SECONDS)); + } + assertEquals(1, distinct.size()); + } finally { + pool.shutdownNow(); + } + } + + @Test + public void testGetMetaData_recordsOncePerConnection() throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + connection.getMetaData(); + connection.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + times(1)); + } + } + + @Test + public void testGetMetaDataInternal_recordsNothingAndDoesNotConsumeUserRecord() + throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + connection.getMetaDataInternal(); + telemetry.verifyNoInteractions(); + // The first *user* call still records, even though the instance already exists. + connection.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + times(1)); + } + } + + @Test + public void testCreateStatementInternal_recordsNoFeatureUsage() throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + BigQueryStatement statement = connection.createStatementInternal(); + assertFalse(statement.isRecordingTelemetry()); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testPrepareStatementInternal_recordsNoFeatureUsageIncludingBindings() + throws SQLException { + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL); + MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + BigQueryPreparedStatement statement = connection.prepareStatementInternal("SELECT ?"); + statement.setString(1, "x"); + assertFalse(statement.isRecordingTelemetry()); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testIsValid_healthCheckRecordsNoTelemetry() throws Exception { + // Regression guard: isValid() runs "SELECT 1" on an internal statement. Pools call it on + // every checkout, so counting it as a user statement or SELECT would swamp real usage. + try (BigQueryConnection connection = new BigQueryConnection(TELEMETRY_TEST_URL)) { + BigQuery mockBigQuery = mock(BigQuery.class); + connection.bigQuery = mockBigQuery; + TableResult tableResult = mock(TableResult.class); + when(tableResult.getSchema()).thenReturn(Schema.of()); + when(tableResult.getStatementType()).thenReturn(StatementType.SELECT); + when(mockBigQuery.queryWithTimeout(any(QueryJobConfiguration.class), any(), any())) + .thenReturn(tableResult); + + // CALLS_REAL_METHODS keeps TelemetryManager's pure helpers working; the record* calls + // are no-ops without an initialized instance but are still counted by verify. + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + connection.isValid(1); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any()), never()); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any(), any()), never()); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(any(), anyLong()), never()); + } + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java index 47489057b0a5..195a83ce0847 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java @@ -43,7 +43,6 @@ import java.io.IOException; import java.io.InputStream; import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -3628,15 +3627,18 @@ public void testGetImportedKeys_noKeys() throws SQLException { @Test public void testGetExportedKeys_infoSchema() throws SQLException { - PreparedStatement mockStmt = mock(PreparedStatement.class); + BigQueryPreparedStatement mockStmt = mock(BigQueryPreparedStatement.class); ResultSet mockRs = mock(ResultSet.class); - when(bigQueryConnection.prepareStatement(anyString())).thenReturn(mockStmt); + when(bigQueryConnection.prepareStatementInternal(anyString())).thenReturn(mockStmt); when(mockStmt.executeQuery()).thenReturn(mockRs); ResultSet rs = dbMetadata.getExportedKeys("test-project", "dataset_p", "ref_table"); assertEquals(mockRs, rs); verify(mockStmt).closeOnCompletion(); verify(mockStmt).executeQuery(); + // Driver-internal metadata queries must not go through the user-facing, telemetry-recording + // prepareStatement. + verify(bigQueryConnection, never()).prepareStatement(anyString()); } @Test diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java index 4c4030d3e1f8..c7202cc103aa 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDriverTest.java @@ -16,12 +16,24 @@ package com.google.cloud.bigquery.jdbc; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.AuthenticationType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.Status; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryTestUtils; import com.google.cloud.bigquery.jdbc.utils.BigQueryJdbcVersionUtility; import io.opentelemetry.api.OpenTelemetry; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverPropertyInfo; import java.sql.SQLException; @@ -32,11 +44,19 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; public class BigQueryDriverTest extends BigQueryJdbcLoggingBaseTest { static BigQueryDriver bigQueryDriver; + /** Token-auth URL that connects without network access; shared by the telemetry tests. */ + private static final String VALID_TOKEN_URL = + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "OAuthType=2;ProjectId=MyBigQueryProject;" + + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" + + "OAuthClientSecret=redactedToken;"; + @BeforeEach @AfterEach public void resetTelemetry() { @@ -55,13 +75,7 @@ public void testInvalidURLReturnsNull() throws SQLException { @Test public void testValidURLDoesConnect() throws SQLException { - Connection connection = - bigQueryDriver.connect( - "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" - + "OAuthType=2;ProjectId=MyBigQueryProject;" - + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" - + "OAuthClientSecret=redactedToken;", - new Properties()); + Connection connection = bigQueryDriver.connect(VALID_TOKEN_URL, new Properties()); assertThat(connection.isClosed()).isFalse(); } @@ -199,13 +213,7 @@ public void testInvalidLogLevelExceptionIsLogged() { @Test public void testConnect_recordsSuccessfulConnectionTelemetry() throws SQLException { - Connection connection = - bigQueryDriver.connect( - "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" - + "OAuthType=2;ProjectId=MyBigQueryProject;" - + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" - + "OAuthClientSecret=redactedToken;", - new Properties()); + Connection connection = bigQueryDriver.connect(VALID_TOKEN_URL, new Properties()); assertThat(connection).isNotNull(); assertThat(connection.isClosed()).isFalse(); // Verify TelemetryManager is initialized and recorded the connection @@ -227,14 +235,72 @@ public void testConnect_recordsFailedConnectionTelemetry() { @Test public void testConnect_optOut_noTelemetryRecorded() throws SQLException { Connection connection = - bigQueryDriver.connect( - "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" - + "OAuthType=2;ProjectId=MyBigQueryProject;" - + "OAuthAccessToken=redactedToken;OAuthClientId=redactedToken;" - + "OAuthClientSecret=redactedToken;EnableDiagnosticTelemetry=0;", - new Properties()); + bigQueryDriver.connect(VALID_TOKEN_URL + "EnableDiagnosticTelemetry=0;", new Properties()); assertThat(connection).isNotNull(); // Since opt-out was requested, TelemetryManager should NOT be initialized assertThat(TelemetryManager.isInitialized()).isFalse(); } + + @Test + public void testConnect_doesNotRecordMetadataRetrieval() throws SQLException { + // connect() reads DatabaseMetaData for its startup log line. That is driver-internal and + // must not be attributed to the user as DRIVER_FEATURE_METADATA_RETRIEVAL. + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS); + Connection ignored = bigQueryDriver.connect(VALID_TOKEN_URL, new Properties())) { + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + never()); + } + } + + @Test + public void testConnect_failedAttemptCarriesAuthType() { + // Service-account auth with no key material parses fine but fails while building the + // connection. The failed attempt must report the auth type, not + // AUTHENTICATION_TYPE_UNSPECIFIED. + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + Assertions.assertThrows( + BigQueryJdbcRuntimeException.class, + () -> + bigQueryDriver.connect( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "OAuthType=0;ProjectId=MyBigQueryProject;", + new Properties())); + telemetry.verify( + () -> + TelemetryManager.recordConnectionAttempt( + eq(Status.STATUS_ERROR), + anyInt(), + eq(AuthenticationType.AUTHENTICATION_TYPE_SERVICE_ACCOUNT))); + } + } + + @Test + public void testConnect_ioExceptionThrowsInsteadOfReturningNull() { + // BigQueryJdbcRootLogger.setLevel is the only IOException source on the connect path. JDBC + // requires connect() to throw for an accepted URL that fails, not return null. + try (MockedStatic rootLogger = + mockStatic(BigQueryJdbcRootLogger.class); + MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + rootLogger + .when(() -> BigQueryJdbcRootLogger.setLevel(any(), any())) + .thenThrow(new IOException("simulated log path failure")); + BigQueryJdbcException thrown = + Assertions.assertThrows( + BigQueryJdbcException.class, + () -> bigQueryDriver.connect(VALID_TOKEN_URL, new Properties())); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + // The failure happens before the URL's auth type is resolved, hence UNSPECIFIED. + telemetry.verify( + () -> + TelemetryManager.recordConnectionAttempt( + eq(Status.STATUS_ERROR), + anyInt(), + eq(AuthenticationType.AUTHENTICATION_TYPE_UNSPECIFIED))); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java index c47e58112e05..235690e78131 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonResultSetTest.java @@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Field.Mode; @@ -34,6 +36,8 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.jdbc.rules.TimeZoneRule; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.common.collect.ImmutableList; import com.google.common.io.BaseEncoding; import com.google.common.io.CharStreams; @@ -73,6 +77,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; public class BigQueryJsonResultSetTest { @@ -914,4 +919,37 @@ public void testTimestampPicosAccessorsAcrossJvmZones(String jvmZone) throws SQL TimeZone.setDefault(originalDefault); } } + + @Test + public void testGetMetaData_recordsFeatureOncePerResultSet() throws SQLException { + // Called inside row loops by many tools; the adoption signal must be recorded once. + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + bigQueryJsonResultSet.getMetaData(); + bigQueryJsonResultSet.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"), + times(1)); + } + } + + @Test + public void testGetMetaData_recordsEvenWhenOwningStatementIsInternal() throws SQLException { + // ResultSet.getMetaData() is only ever a user call, even on result sets produced by + // driver-internal statements (e.g. DatabaseMetaData.getExportedKeys), so it must count. + BigQueryStatement internalStatement = mock(BigQueryStatement.class); + BigQueryJsonResultSet internalResultSet = + BigQueryJsonResultSet.of(QUERY_SCHEMA, 1L, buffer, internalStatement, new Future[0]); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + internalResultSet.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"), + times(1)); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java index f21dbcb5eff1..03ddee8535a6 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnectionTest.java @@ -23,11 +23,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import java.io.IOException; import java.sql.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; public class BigQueryPooledConnectionTest { private BigQueryConnection bigQueryConnection; @@ -173,4 +179,19 @@ public void testFireConnectionError() throws SQLException { assertFalse(pooledConnection.isListenerPooled(listner)); } + + @Test + public void testPooledGetMetaData_delegatesWithoutRecordingTelemetry() throws SQLException { + // The physical BigQueryConnection records DRIVER_FEATURE_METADATA_RETRIEVAL; the pooled + // wrapper must only delegate, otherwise every pooled call is counted twice. + BigQueryPooledConnection pooledConnection = new BigQueryPooledConnection(bigQueryConnection); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + pooledConnection.getConnection().getMetaData(); + verify(bigQueryConnection).getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL), + never()); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java index 59fc26767278..419c46f68777 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java @@ -23,8 +23,18 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; import com.google.cloud.bigquery.Field; @@ -34,6 +44,12 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; +import com.google.cloud.bigquery.jdbc.telemetry.v1.Status; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.gson.Gson; import com.google.gson.JsonNull; import com.google.gson.JsonObject; @@ -59,6 +75,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; public class BigQueryPreparedStatementSettersTest { @@ -532,4 +550,137 @@ public void testSetObjectLocalDateTime_zeroSecondsStillHasFullDigits() throws SQ preparedStatement.setObject(1, LocalDateTime.of(2025, 1, 1, 12, 0)); assertEquals("2025-01-01 12:00:00.000000", preparedStatement.parameterHandler.getParameter(1)); } + + @Test + public void testSetters_recordParameterBindingFeatureOncePerStatement() throws SQLException { + // DRIVER_FEATURE_PARAMETER_BINDING is an adoption signal: one record per prepared statement, + // regardless of how many setX calls are made or which handler overload they use. + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + preparedStatement.setInt(1, 42); + preparedStatement.setLong(2, 42L); + preparedStatement.setString(3, "x"); + preparedStatement.setNull(4, Types.VARCHAR); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING), + times(1)); + } + } + + @Test + public void testExecuteBatch_success_recordsBatchOperationsOnce() throws SQLException { + // Standard (non-Write-API) batch path. Stub the query run so no BigQuery client is needed. + BigQueryPreparedStatement spy = spy(preparedStatement); + doReturn(null).when(spy).getStandardBatchJobConfiguration(anyString()); + doNothing().when(spy).runQuery(anyString(), any()); + doReturn(-1).when(spy).getUpdateCount(); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + spy.executeBatch(); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS), + times(1)); + } + } + + @Test + public void testExecuteBatch_failure_doesNotRecordBatchOperations() throws SQLException { + // Regression guard: BATCH_OPERATIONS is success-only; the failure is already captured by + // the StatementExecution status, so recording the feature here would double-report it. + BigQueryPreparedStatement spy = spy(preparedStatement); + doReturn(null).when(spy).getStandardBatchJobConfiguration(anyString()); + doThrow(new BigQueryJdbcException("simulated batch failure")) + .when(spy) + .runQuery(anyString(), any()); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + assertThrows(BigQueryJdbcException.class, spy::executeBatch); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS), + never()); + } + } + + @Test + public void testExecuteBatch_writeApiFailure_recordsErrorStatusAndNoBatchFeature() + throws SQLException { + // Write API branch. bulkInsertWithWriteAPI throws BigQueryJdbcException when the statement + // has no insert metadata; that failure must be recorded as STATUS_ERROR (previously it fell + // through to STATUS_UNSPECIFIED) and must not count as a BATCH_OPERATIONS feature use. + BigQueryPreparedStatement spy = spy(preparedStatement); + doReturn(true).when(spy).useWriteAPI(); + when(connection.getBigQueryWriteClient()).thenReturn(mock(BigQueryWriteClient.class)); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + assertThrows(Exception.class, spy::executeBatch); + ArgumentCaptor execution = + ArgumentCaptor.forClass(StatementExecution.Builder.class); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(execution.capture(), anyLong()), + times(1)); + assertEquals(Status.STATUS_ERROR, execution.getValue().getStatus()); + assertEquals(QueryApiType.QUERY_API_TYPE_WRITE_API, execution.getValue().getQueryApiType()); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS), + never()); + } + } + + @Test + public void testExecuteBatch_internalStatement_recordsNothing() throws SQLException { + BigQueryPreparedStatement spy = spy(preparedStatement); + spy.markInternal(); + doReturn(null).when(spy).getStandardBatchJobConfiguration(anyString()); + doNothing().when(spy).runQuery(anyString(), any()); + doReturn(-1).when(spy).getUpdateCount(); + spy.setInt(1, 1); + spy.addBatch(); + + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + spy.executeBatch(); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testGetMetaData_recordsResultSetMetadataRetrievalOncePerStatement() + throws SQLException { + preparedStatement.resultSchema = Schema.of(Field.of("c", StandardSQLTypeName.INT64)); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + preparedStatement.getMetaData(); + preparedStatement.getMetaData(); + telemetry.verify( + () -> + TelemetryManager.recordFeatureUsage( + DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL, + "DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL"), + times(1)); + } + } + + @Test + public void testGetMetaData_noResultSchema_returnsNullAndRecordsNothing() throws SQLException { + // JDBC: null for a statement that produces no result set. No metadata, no record. + preparedStatement.resultSchema = null; + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + assertNull(preparedStatement.getMetaData()); + telemetry.verifyNoInteractions(); + } + } + + @Test + public void testGetMetaData_internalStatement_recordsNothing() throws SQLException { + preparedStatement.resultSchema = Schema.of(Field.of("c", StandardSQLTypeName.INT64)); + preparedStatement.markInternal(); + try (MockedStatic telemetry = mockStatic(TelemetryManager.class)) { + assertNotNull(preparedStatement.getMetaData()); + telemetry.verifyNoInteractions(); + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index f4e7b406cd90..4e5776e4759c 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -26,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -67,6 +68,8 @@ import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.jdbc.BigQueryStatement.JobIdWrapper; +import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.spi.BigQueryRpcFactory; import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; @@ -125,6 +128,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; import org.mockito.Mockito; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -1887,4 +1891,74 @@ private Job createScriptChildJobMock( doReturn(mock(ScriptStatistics.class)).when(stats).getScriptStatistics(); return childJob; } + + @Test + public void testClose_doesNotRecordStatementCancelFeature() throws SQLException { + // close() cancels jobs as housekeeping; that must not count as a user-initiated cancel. + assertFeatureRecorded( + () -> bigQueryStatement.close(), DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL, 0); + } + + @Test + public void testCancel_recordsStatementCancelFeatureOnce() throws SQLException { + assertFeatureRecorded( + () -> bigQueryStatement.cancel(), DriverFeature.DRIVER_FEATURE_STATEMENT_CANCEL, 1); + } + + @Test + public void testInternalStatement_recordsNoTelemetryForExecutionOrCancel() throws Exception { + // Statements the driver creates for itself (isValid, metadata queries) must not be reported + // as user activity: no StatementExecution, no feature usage. + TableResult tableResultMock = setupMockQueryResults(null, StatementType.SELECT, null); + BigQueryStatement internalStatement = new BigQueryStatement(bigQueryConnection); + internalStatement.markInternal(); + BigQueryStatement internalSpy = Mockito.spy(internalStatement); + doReturn(mock(BigQueryJsonResultSet.class)) + .when(internalSpy) + .processJsonResultSet(eq(tableResultMock), any()); + + // CALLS_REAL_METHODS keeps TelemetryManager's pure helpers (toStatementType etc.) working; + // the record* methods are no-ops without an initialized instance but are still counted. + try (MockedStatic telemetry = + Mockito.mockStatic(TelemetryManager.class, Mockito.CALLS_REAL_METHODS)) { + internalSpy.executeQuery("SELECT 1"); + internalSpy.cancel(); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(any(), anyLong()), Mockito.never()); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any()), Mockito.never()); + telemetry.verify(() -> TelemetryManager.recordFeatureUsage(any(), any()), Mockito.never()); + } + } + + @Test + public void testUserStatement_recordsExecution() throws Exception { + // Control for the test above: the same flow on a regular statement does reach the batcher. + TableResult tableResultMock = setupMockQueryResults(null, StatementType.SELECT, null); + BigQueryStatement userSpy = Mockito.spy(new BigQueryStatement(bigQueryConnection)); + doReturn(mock(BigQueryJsonResultSet.class)) + .when(userSpy) + .processJsonResultSet(eq(tableResultMock), any()); + + try (MockedStatic telemetry = + Mockito.mockStatic(TelemetryManager.class, Mockito.CALLS_REAL_METHODS)) { + userSpy.executeQuery("SELECT 1"); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(any(), anyLong()), Mockito.times(1)); + } + } + + /** Runs {@code action} and asserts how many times {@code feature} was recorded by it. */ + private static void assertFeatureRecorded( + SqlRunnable action, DriverFeature feature, int expectedCount) throws SQLException { + try (MockedStatic telemetry = Mockito.mockStatic(TelemetryManager.class)) { + action.run(); + telemetry.verify( + () -> TelemetryManager.recordFeatureUsage(feature), Mockito.times(expectedCount)); + } + } + + @FunctionalInterface + private interface SqlRunnable { + void run() throws SQLException; + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java index 3fb7a816cf5e..663bd025e4db 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITStatementTest.java @@ -229,9 +229,15 @@ public void testExecuteUpdate() throws SQLException { public void testScript() throws SQLException { Properties withReadApi = new Properties(); withReadApi.setProperty("EnableHighThroughputAPI", "1"); - String longQuery = String.format(GENERATE_ARRAY_QUERY, 500000); - String longerQuery = String.format(GENERATE_ARRAY_QUERY, 700000); - String longerQuery2 = String.format(GENERATE_ARRAY_QUERY, 900000); + // Each child result must exceed the Read API activation point (>= HighThroughputMinTableSize + // rows, default 10,000, and > HighThroughputActivationRatio pages, default 2, at the default + // MaxResults page size of 10,000), so >= 30,000 rows per child takes the Arrow path. Keep the + // counts modest: the presubmit suite runs all IT classes in one JVM and Arrow buffers are + // off-heap, so multi-million-row scripts have pushed the build container past its memory + // limit (exit 137). + String longQuery = String.format(GENERATE_ARRAY_QUERY, 30000); + String longerQuery = String.format(GENERATE_ARRAY_QUERY, 40000); + String longerQuery2 = String.format(GENERATE_ARRAY_QUERY, 50000); try (Connection connection = DriverManager.getConnection(ITBase.connectionUrl, withReadApi); Statement statement = connection.createStatement()) { @@ -239,17 +245,17 @@ public void testScript() throws SQLException { ResultSet arrowResultSet = statement.getResultSet(); assertArrowResultSet(arrowResultSet); - assertEquals(500000, resultSetRowCount(arrowResultSet)); + assertEquals(30000, resultSetRowCount(arrowResultSet)); assertTrue(statement.getMoreResults()); arrowResultSet = statement.getResultSet(); assertArrowResultSet(arrowResultSet); - assertEquals(700000, resultSetRowCount(arrowResultSet)); + assertEquals(40000, resultSetRowCount(arrowResultSet)); assertTrue(statement.getMoreResults()); arrowResultSet = statement.getResultSet(); assertArrowResultSet(arrowResultSet); - assertEquals(900000, resultSetRowCount(arrowResultSet)); + assertEquals(50000, resultSetRowCount(arrowResultSet)); assertFalse(statement.getMoreResults()); }