Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion java-bigquery-jdbc/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,18 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<!--
The presubmit suite runs every IT class in one forked JVM and Arrow result sets allocate
off-heap. Without bounds, a memory overrun kills the build container (exit 137) with no
test attributed. Capping heap and direct memory turns that into an OutOfMemoryError on
the offending test, and ExitOnOutOfMemoryError stops the fork instead of letting it limp
on. Applies on every JDK, including the java11 CI image where the [17,) profile is off.
-->
<argLine>-Xmx1536m -XX:MaxDirectMemorySize=1g -XX:+ExitOnOutOfMemoryError</argLine>
<systemPropertyVariables>
<JDBC_TESTS>true</JDBC_TESTS>
<!-- Test-only: keeps integration runs from emitting driver usage telemetry. -->
<BIGQUERY_JDBC_TELEMETRY_ENABLED>false</BIGQUERY_JDBC_TELEMETRY_ENABLED>
</systemPropertyVariables>
</configuration>
<dependencies>
Expand Down Expand Up @@ -518,7 +528,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<argLine>--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED</argLine>
<!-- Profile argLine replaces the base one; repeat the memory bounds from above. -->
<argLine>-Xmx1536m -XX:MaxDirectMemorySize=1g -XX:+ExitOnOutOfMemoryError --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED</argLine>
</configuration>
</plugin>
</plugins>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ public abstract class BigQueryBaseResultSet extends BigQueryNoOpsResultSet
protected final boolean isNested;
protected boolean isClosed = false;
protected boolean wasNull = false;
// getMetaData() is commonly called inside row loops; record the adoption signal once per
// result set. A ResultSet is single-threaded by JDBC contract, so a plain field suffices.
private boolean metadataRetrievalRecorded;
private int fetchSize = -1;
private Job job;
private SQLWarning warnings;
Expand Down Expand Up @@ -268,9 +271,15 @@ public ResultSetMetaData getMetaData() throws SQLException {
}
}

TelemetryManager.recordFeatureUsage(
DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL,
"DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL");
// Always a user call: no driver code invokes ResultSet.getMetaData(), so this is recorded
// regardless of whether the owning statement is internal (e.g. the result set that
// DatabaseMetaData.getExportedKeys hands back to the user).
if (!this.metadataRetrievalRecorded) {
this.metadataRetrievalRecorded = true;
TelemetryManager.recordFeatureUsage(
DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL,
"DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL");
}

return BigQueryJdbcContextProxy.wrap(metaData, ResultSetMetaData.class, connectionId);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,10 @@
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Level;
import javax.annotation.concurrent.GuardedBy;

/**
* An implementation of {@link java.sql.Connection} for establishing a connection with BigQuery and
Expand Down Expand Up @@ -249,7 +251,13 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
private Context otelContext;
Tracer tracer =
OpenTelemetry.noop().getTracer(BigQueryJdbcOpenTelemetry.INSTRUMENTATION_SCOPE_NAME);
DatabaseMetaData databaseMetaData;

@GuardedBy("this")
private DatabaseMetaData databaseMetaData;

// A Connection may be used from several threads; compareAndSet keeps the adoption signal at
// exactly one record per connection without a lock.
private final AtomicBoolean metadataRetrievalRecorded = new AtomicBoolean();
Boolean reqGoogleDriveScope;
private final Properties clientInfo = new Properties();
private boolean isReadOnlyTokenUsed = false;
Expand Down Expand Up @@ -511,6 +519,34 @@ public Statement createStatement() throws SQLException {
return currentStatement;
}

/**
* Creates a {@code Statement} for the driver's own use (for example the {@code isValid}
* health-check query). Identical to {@link #createStatement()} except that neither the statement
* nor anything it executes is reported as user activity in usage telemetry.
*/
BigQueryStatement createStatementInternal() throws SQLException {
checkClosed();
BigQueryStatement statement = new BigQueryStatement(this);
statement.markInternal();
LOG.fine("Internal Statement %s created.", statement);
addOpenStatements(statement);
return statement;
}

/**
* Creates a {@code PreparedStatement} for the driver's own metadata queries. Identical to {@link
* #prepareStatement(String)} except that neither the statement, its parameter bindings, nor
* anything it executes is reported as user activity in usage telemetry.
*/
BigQueryPreparedStatement prepareStatementInternal(String sql) throws SQLException {
checkClosed();
BigQueryPreparedStatement statement = new BigQueryPreparedStatement(this, sql);
statement.markInternal();
LOG.fine("Internal Prepared Statement %s created.", statement);
addOpenStatements(statement);
return statement;
}

/**
* Creates and returns a new {@code Statement} object for executing BigQuery SQL queries. This
* method is similar to {@link BigQueryConnection#createStatement()}, but it overrides the type
Expand Down Expand Up @@ -883,7 +919,8 @@ public boolean isValid(int timeout) throws SQLException {
throw new BigQueryJdbcException("timeout must be >= 0");
}
if (!isClosed()) {
try (Statement statement = createStatement();
// Internal statement: a pool health-check must not count as user statements or SELECTs.
try (Statement statement = createStatementInternal();
ResultSet rs = statement.executeQuery("SELECT 1")) {
LOG.finer("Running validation query");
if (rs.next()) {
Expand Down Expand Up @@ -1031,12 +1068,26 @@ private void rollbackImpl() throws SQLException {

@Override
public DatabaseMetaData getMetaData() throws SQLException {
DatabaseMetaData metaData = getMetaDataInternal();
// Adoption signal, recorded once per connection on the first user call. Tied to this flag
// rather than to instance creation so the driver's own startup lookup does not consume it.
if (this.metadataRetrievalRecorded.compareAndSet(false, true)) {
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL);
}
return metaData;
}

/**
* Same instance as {@link #getMetaData()} without the usage-telemetry record. Driver-internal.
*
* <p>Synchronized: a Connection may be shared across threads, and without the lock two threads
* could each construct a BigQueryDatabaseMetaData, or one could observe a partially constructed
* instance. The lock is uncontended after first use and this is not a hot path.
*/
synchronized DatabaseMetaData getMetaDataInternal() {
if (databaseMetaData == null) {
databaseMetaData = new BigQueryDatabaseMetaData(this);
}

TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL);

return databaseMetaData;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2610,7 +2610,7 @@ public ResultSet getExportedKeys(String catalog, String schema, String table)
validateSchemaIdentifier(schema);
String sql = getExportedKeysSqlContent();
String formattedSql = replaceSqlParameters(sql, catalog, schema, table);
PreparedStatement stmt = this.connection.prepareStatement(formattedSql);
PreparedStatement stmt = this.connection.prepareStatementInternal(formattedSql);
if (stmt == null) {
throw new BigQueryJdbcException("Failed to prepare statement for getExportedKeys");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import io.opentelemetry.api.OpenTelemetry;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.DriverPropertyInfo;
Expand Down Expand Up @@ -196,10 +197,12 @@ public Connection connect(String url, Properties info) throws SQLException {
LOG.log(Level.SEVERE, "Failed to parse connection URL", e);
throw new BigQueryJdbcException("Failed to parse connection URL", e);
}
authType = TelemetryManager.toAuthenticationType(ds.getOAuthType());
if (customOpenTelemetryObj instanceof OpenTelemetry) {
ds.setCustomOpenTelemetry((OpenTelemetry) customOpenTelemetryObj);
}
BigQueryConnection connection = new BigQueryConnection(connectionUri, ds);
DatabaseMetaData metaData = connection.getMetaDataInternal();
LOG.info(
"Driver info : { {Database Product Name : %s}, "
+ "{Database Product Version : %s}, "
Expand All @@ -208,17 +211,16 @@ public Connection connect(String url, Properties info) throws SQLException {
+ "{LogLevel : %s}, "
+ "{LogPath : %s}, "
+ "{Driver Instance : %s} }",
connection.getMetaData().getDatabaseProductName(),
connection.getMetaData().getDatabaseProductVersion(),
connection.getMetaData().getDriverName(),
connection.getMetaData().getDriverVersion(),
metaData.getDatabaseProductName(),
metaData.getDatabaseProductVersion(),
metaData.getDriverName(),
metaData.getDriverVersion(),
logLevel,
logPath,
this.toString());

Connection wrapped = BigQueryJdbcContextProxy.wrap(connection, Connection.class);

authType = TelemetryManager.toAuthenticationType(ds.getOAuthType());
TelemetryManager.recordConnectionAttempt(Status.STATUS_SUCCESS, 0, authType);
return wrapped;
} else {
Expand All @@ -227,8 +229,7 @@ public Connection connect(String url, Properties info) throws SQLException {
} catch (IOException e) {
TelemetryManager.recordConnectionAttempt(
Status.STATUS_ERROR, TelemetryManager.extractErrorCode(e), authType);
LOG.warning("Getting a warning: %s", e.getMessage());
return null;
throw new BigQueryJdbcException("Failed to establish connection", e);
} catch (Throwable t) {
TelemetryManager.recordConnectionAttempt(
Status.STATUS_ERROR, TelemetryManager.extractErrorCode(t), authType);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,28 @@ class BigQueryParameterHandler {
new BigQueryJdbcCustomLogger(BigQueryParameterHandler.class.getName());
private final int parametersArraySize;
private final boolean enableTimestampPicos;

/** False when the owning statement is driver-internal; see BigQueryStatement#markInternal. */
private boolean recordTelemetry = true;

/**
* DRIVER_FEATURE_PARAMETER_BINDING is an adoption signal, recorded once per handler (i.e. per
* prepared statement) rather than once per setX call. Handlers are confined to one statement,
* which JDBC defines as single-threaded, so a plain field is sufficient.
*/
private boolean bindingRecorded;

void markInternal() {
this.recordTelemetry = false;
}

private void recordBinding() {
if (this.recordTelemetry && !this.bindingRecorded) {
this.bindingRecorded = true;
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING);
}
}

final ArrayList<BigQueryJdbcParameter> parametersList;
// TIMESTAMP is rendered in UTC with a trailing 'Z': the instant is preserved, the output does not
// depend on the JVM zone, and QueryParameterValue's validator accepts 'Z' but rejects "-05:00".
Expand Down Expand Up @@ -271,7 +293,7 @@ void setNullParameter(int parameterIndex, Class<?> declaredJavaType) {
parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED);
parameter.setScale(-1);

TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING);
recordBinding();

LOG.finest("Parameter set { %s }", parameter.toString());
}
Expand All @@ -289,6 +311,8 @@ void setParameter(int parameterIndex, Object value, Class type) {
parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED);
parameter.setScale(-1);

recordBinding();

LOG.finest("Parameter set { %s }", parameter.toString());
}

Expand Down Expand Up @@ -383,7 +407,7 @@ void setParameter(
parametersList.add(parameter);
}

TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING);
recordBinding();

LOG.finest("Parameter set { %s }", parameter.toString());
}
Expand All @@ -409,7 +433,7 @@ void setParameter(
parameter.setParamType(paramType);
parameter.setScale(scale);

TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING);
recordBinding();

LOG.finest("Parameter set { %s }", parameter.toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@

package com.google.cloud.bigquery.jdbc;

import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature;
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
import com.google.common.annotations.VisibleForTesting;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
Expand Down Expand Up @@ -237,7 +235,6 @@ public void rollback() throws SQLException {

@Override
public DatabaseMetaData getMetaData() throws SQLException {
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL);
return bqConnectionDelegate.getMetaData();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta
Schema resultSchema = null; // Feeds getMetaData().
Schema insertSchema = null; // Feeds the Storage Write API.
private TableName insertTableName = null;
// Per-object adoption signal for getMetaData(); see BigQueryParameterHandler#bindingRecorded.
private boolean resultSetMetadataRecorded;

BigQueryPreparedStatement(BigQueryConnection connection, String query) throws SQLException {
super(connection);
Expand All @@ -97,6 +99,12 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta
}
}

@Override
void markInternal() {
super.markInternal();
this.parameterHandler.markInternal();
}

private int resolveParameterCount(String query, QueryStatistics queryStatistics) {
if (queryStatistics != null) {
return queryStatistics.getQueryParameters() == null
Expand Down Expand Up @@ -426,6 +434,9 @@ public int[] executeBatch() throws SQLException {

writeApiExecutionBuilder.setStatus(
com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_SUCCESS);
// Success-only, matching BigQueryStatement.executeBatch; failures are already captured
// by the StatementExecution status recorded in the finally block.
recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS);

return insertArray;

Expand All @@ -440,10 +451,17 @@ public int[] executeBatch() throws SQLException {
.setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR)
.setErrorCode(TelemetryManager.extractErrorCode(e));
throw new BigQueryJdbcException("Failed to execute batch with Write API", e);
} catch (SQLException | RuntimeException e) {
// bulkInsertWithWriteAPI throws BigQueryJdbcException itself, and the Storage Write API
// client throws unchecked ApiException; neither matched the catches above, so these
// failures used to be recorded as STATUS_UNSPECIFIED.
writeApiExecutionBuilder
.setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR)
.setErrorCode(TelemetryManager.extractErrorCode(e));
throw e;
} finally {
long durationMs = System.currentTimeMillis() - startTime;
TelemetryManager.recordStatementExecution(writeApiExecutionBuilder, durationMs);
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS);
recordExecution(writeApiExecutionBuilder, durationMs);
}

} else {
Expand All @@ -469,11 +487,10 @@ public int[] executeBatch() throws SQLException {
getMoreResults();
i++;
}
recordFeature(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS);
return result;
} catch (SQLException e) {
throw new BigQueryJdbcException("SQL error during individual INSERT batch", e);
} finally {
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS);
}
}
}
Expand Down Expand Up @@ -628,9 +645,12 @@ public ResultSetMetaData getMetaData() throws SQLException {
return null;
}

TelemetryManager.recordFeatureUsage(
DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL,
"DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL");
if (!this.resultSetMetadataRecorded && isRecordingTelemetry()) {
this.resultSetMetadataRecorded = true;
TelemetryManager.recordFeatureUsage(
DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL,
"DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL");
}

return BigQueryResultSetMetadata.of(this.resultSchema.getFields(), this);
}
Expand Down
Loading
Loading