diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
index bd6041a6f7f4..16d23a571da7 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
@@ -22,11 +22,16 @@
/** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */
class BigQueryArrowBatchWrapper {
+ private static final long ROW_COUNT_NOT_REPORTED = -1;
+
// Reference to the current arrowBatch
private final ArrowRecordBatch currentArrowBatch;
// Arrow schema carried with this batch, set only when the source sent one (e.g. the first
// ReadRowsResponse of a stream)
private final ArrowSchema arrowSchema;
+ // Rows the source reported for this message (ReadRowsResponse.row_count), or -1 when the wrapper
+ // was not built from a stream message (markers, nested records)
+ private final long reportedRowCount;
// Reference to the nested Records, set as null otherwise (Arrays)
private final JsonStringArrayList nestedRecords;
@@ -38,11 +43,13 @@ class BigQueryArrowBatchWrapper {
private BigQueryArrowBatchWrapper(
ArrowRecordBatch currentArrowBatch,
ArrowSchema arrowSchema,
+ long reportedRowCount,
JsonStringArrayList nestedRecords,
boolean isLast,
Throwable exception) {
this.currentArrowBatch = currentArrowBatch;
this.arrowSchema = arrowSchema;
+ this.reportedRowCount = reportedRowCount;
this.nestedRecords = nestedRecords;
this.isLast = isLast;
this.exception = exception;
@@ -50,25 +57,30 @@ private BigQueryArrowBatchWrapper(
static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
- return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, isLastFlag, null);
+ return new BigQueryArrowBatchWrapper(
+ currentArrowBatch, null, ROW_COUNT_NOT_REPORTED, null, isLastFlag, null);
}
/**
- * Wraps a record batch together with the schema sent by the source. Either may be null (a
- * schema-only message carries no batch; later messages carry no schema).
+ * Wraps one stream message: its record batch, the schema sent with it, and the row count it
+ * reports. The batch and the schema may each be null (a schema-only message carries no batch;
+ * later messages carry no schema).
*/
- static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema) {
- return new BigQueryArrowBatchWrapper(currentArrowBatch, arrowSchema, null, false, null);
+ static BigQueryArrowBatchWrapper of(
+ ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema, long reportedRowCount) {
+ return new BigQueryArrowBatchWrapper(
+ currentArrowBatch, arrowSchema, reportedRowCount, null, false, null);
}
static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper(
JsonStringArrayList nestedRecords, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
- return new BigQueryArrowBatchWrapper(null, null, nestedRecords, isLastFlag, null);
+ return new BigQueryArrowBatchWrapper(
+ null, null, ROW_COUNT_NOT_REPORTED, nestedRecords, isLastFlag, null);
}
static BigQueryArrowBatchWrapper ofError(Throwable exception) {
- return new BigQueryArrowBatchWrapper(null, null, null, true, exception);
+ return new BigQueryArrowBatchWrapper(null, null, ROW_COUNT_NOT_REPORTED, null, true, exception);
}
ArrowRecordBatch getCurrentArrowBatch() {
@@ -79,6 +91,16 @@ ArrowSchema getArrowSchema() {
return this.arrowSchema;
}
+ /** True if this wrapper was built from a stream message, which always reports a row count. */
+ boolean hasReportedRowCount() {
+ return this.reportedRowCount != ROW_COUNT_NOT_REPORTED;
+ }
+
+ /** Rows the source reported for this message. See {@link #hasReportedRowCount()}. */
+ long getReportedRowCount() {
+ return this.reportedRowCount;
+ }
+
JsonStringArrayList getNestedRecords() {
return this.nestedRecords;
}
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
index afac21f9c456..ecdefe60d7fc 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
@@ -291,6 +291,10 @@ public boolean next() throws SQLException {
this.rowCount++;
return true;
} catch (SQLException ex) {
+ // A stream error ends iteration: the next call returns false and no row stays readable.
+ // The failed message is never skipped, even if the caller ignores this exception.
+ this.hasReachedEnd = true;
+ this.currentBatchRowIndex = -1;
throw new BigQueryJdbcException(
"Error occurred while advancing the cursor. This could happen when connection is closed while the next method is being called.",
ex);
@@ -320,16 +324,24 @@ private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLExcepti
}
try {
this.arrowDeserializer = new ArrowDeserializer(arrowSchema);
- } catch (IOException ex) {
- throw new BigQueryJdbcException("IOException during ArrowDeserializer creation", ex);
+ } catch (RuntimeException | IOException ex) {
+ // Arrow throws unchecked exceptions on malformed schema bytes.
+ throw new BigQueryJdbcException("Failed to create the Arrow deserializer", ex);
}
}
/**
* Loads the next batch that has rows into the vectors. The reader queues every message as-is, so
- * this is the single place that decides what to use: messages without rows (schema, statistics or
- * throttling updates) are skipped, and the stream must deliver all {@code totalRows} rows.
- * Returns false at the end of the stream.
+ * this is the single place that decides what to use:
+ *
+ *
+ * - A message without batch data (a schema, statistics or throttling update) is skipped. It
+ * must not report rows.
+ *
- A message with batch data is decoded, and must hold exactly the rows it reports.
+ *
- The stream must deliver all {@code totalRows} rows.
+ *
+ *
+ * Returns false at the end of the stream. A failure ends iteration, see {@link #next()}.
*/
private boolean loadNextBatch() throws SQLException, InterruptedException {
while (true) {
@@ -354,7 +366,11 @@ private boolean loadNextBatch() throws SQLException, InterruptedException {
}
initDeserializerIfNeeded(batchWrapper.getArrowSchema());
ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch();
+ long reportedRows = batchWrapper.getReportedRowCount();
if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) {
+ if (reportedRows > 0) {
+ throw rowCountMismatch(reportedRows, 0);
+ }
continue; // no rows in this message
}
if (this.arrowDeserializer == null) {
@@ -362,12 +378,23 @@ private boolean loadNextBatch() throws SQLException, InterruptedException {
}
// Populates vectorSchemaRoot
this.arrowDeserializer.deserializeArrowBatch(arrowBatch);
- if (this.vectorSchemaRoot.getRowCount() > 0) {
+ int decodedRows = this.vectorSchemaRoot.getRowCount();
+ if (batchWrapper.hasReportedRowCount() && reportedRows != decodedRows) {
+ throw rowCountMismatch(reportedRows, decodedRows);
+ }
+ if (decodedRows > 0) {
return true;
}
}
}
+ private static SQLException rowCountMismatch(long reportedRows, long decodedRows) {
+ return new BigQueryJdbcException(
+ String.format(
+ "An Arrow stream message reported %d rows but its batch has %d.",
+ reportedRows, decodedRows));
+ }
+
@Override
Object getObjectInternal(int columnIndex) throws SQLException {
LOG.finestTrace("getObjectInternal");
@@ -562,7 +589,8 @@ public boolean isFirst() throws SQLException {
if (this.isNested) {
return !this.afterLast && this.nestedRowIndex == this.fromIndex;
} else {
- return !this.afterLast && this.rowCount == 1;
+ // The index is negative after a stream error: the cursor is on no row then.
+ return !this.afterLast && this.rowCount == 1 && this.currentBatchRowIndex >= 0;
}
}
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
index 34903ac6ffb9..ad8e7f5cc737 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
@@ -62,6 +62,7 @@
import com.google.cloud.bigquery.storage.v1.ReadSession;
import com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import io.opentelemetry.api.common.AttributeKey;
@@ -90,6 +91,7 @@
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
+import java.util.function.Supplier;
import java.util.logging.Level;
/**
@@ -1003,12 +1005,87 @@ private TableReadOptions buildTableReadOptions() {
.build();
}
+ /**
+ * Arrow options for a job stream. They are sent with every request instead of left to the
+ * stream's defaults, because they change the schema the stream returns: all requests for one
+ * result must use the same values. Compression stays unset, which means none.
+ */
+ @VisibleForTesting
+ ArrowSerializationOptions buildArrowSerializationOptions() {
+ return ArrowSerializationOptions.newBuilder()
+ .setPicosTimestampPrecision(
+ isEnableTimestampPicos()
+ ? ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS
+ : ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS)
+ .build();
+ }
+
+ /**
+ * Name of the stream that carries a query job's results. The API does not return it, so it is
+ * built from the job reference. The location must be the job's own: the stream does not exist in
+ * any other location.
+ */
+ @VisibleForTesting
+ static String jobStreamName(JobId jobId) throws BigQueryJdbcException {
+ if (jobId == null
+ || Strings.isNullOrEmpty(jobId.getProject())
+ || Strings.isNullOrEmpty(jobId.getLocation())
+ || Strings.isNullOrEmpty(jobId.getJob())) {
+ throw new BigQueryJdbcException(
+ "A job stream needs the job's project, location and job ID, got: " + jobId);
+ }
+ return String.format(
+ "projects/%s/locations/%s/jobs/%s/streams/_default",
+ jobId.getProject(), jobId.getLocation(), jobId.getJob());
+ }
+
/** Asynchronously reads results and populates an arrow record queue */
@InternalApi
Future> populateArrowBufferedQueue(
ReadSession readSession,
BlockingQueue arrowBatchWrapperBlockingQueue,
BigQueryReadClient bqReadClient) {
+ // Use the first stream to perform reading. The session already carries the read options:
+ // ReadRows accepts Arrow options for job streams only.
+ return readRowsIntoArrowQueue(
+ () ->
+ ReadRowsRequest.newBuilder().setReadStream(readSession.getStreams(0).getName()).build(),
+ arrowBatchWrapperBlockingQueue,
+ bqReadClient);
+ }
+
+ /**
+ * Asynchronously reads a query job's results from its {@code _default} stream, starting at row
+ * {@code startOffset}, and populates an arrow record queue. No read session is created.
+ */
+ @InternalApi
+ Future> populateArrowBufferedQueueFromJobStream(
+ JobId jobId,
+ long startOffset,
+ BlockingQueue arrowBatchWrapperBlockingQueue,
+ BigQueryReadClient bqReadClient)
+ throws BigQueryJdbcException {
+ if (startOffset < 0) {
+ throw new BigQueryJdbcException("A job stream offset can't be negative: " + startOffset);
+ }
+ ReadRowsRequest firstRequest =
+ ReadRowsRequest.newBuilder()
+ .setReadStream(jobStreamName(jobId))
+ .setOffset(startOffset)
+ .setArrowSerializationOptions(buildArrowSerializationOptions())
+ .build();
+ return readRowsIntoArrowQueue(() -> firstRequest, arrowBatchWrapperBlockingQueue, bqReadClient);
+ }
+
+ /**
+ * Reads one stream into the queue. Every attempt sends the first request moved forward by the
+ * rows already read, so a resumed request keeps the stream name and the Arrow options. The first
+ * request is resolved on the worker, where a failure reaches the queue as an error.
+ */
+ private Future> readRowsIntoArrowQueue(
+ Supplier firstRequestSupplier,
+ BlockingQueue arrowBatchWrapperBlockingQueue,
+ BigQueryReadClient bqReadClient) {
LOG.finer("++enter++");
ExecutorService executor = connection.getExecutorService();
@@ -1020,17 +1097,15 @@ Future> populateArrowBufferedQueue(
long rowsRead = 0;
int retryCount = 0;
try {
- // Use the first stream to perform reading.
- String streamName = readSession.getStreams(0).getName();
+ ReadRowsRequest firstRequest = firstRequestSupplier.get();
while (true) {
ServerStream stream = null;
boolean streamCompleted = false;
try {
ReadRowsRequest readRowsRequest =
- ReadRowsRequest.newBuilder()
- .setReadStream(streamName)
- .setOffset(rowsRead)
+ firstRequest.toBuilder()
+ .setOffset(firstRequest.getOffset() + rowsRead)
.build();
// Process each block of rows as they arrive and decode using our simple row
@@ -1048,7 +1123,8 @@ Future> populateArrowBufferedQueue(
response.hasArrowRecordBatch()
? response.getArrowRecordBatch()
: null,
- response.hasArrowSchema() ? response.getArrowSchema() : null));
+ response.hasArrowSchema() ? response.getArrowSchema() : null,
+ response.getRowCount()));
rowsRead += response.getRowCount();
}
if (isWorkerShuttingDown()) {
@@ -1059,7 +1135,7 @@ Future> populateArrowBufferedQueue(
break;
} catch (ApiException e) {
if (e.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) {
- LOG.warning("Read session expired or not found: %s", e.getMessage());
+ LOG.warning("Read stream expired or not found: %s", e.getMessage());
enqueueError(arrowBatchWrapperBlockingQueue, e);
break;
}
@@ -1077,7 +1153,7 @@ Future> populateArrowBufferedQueue(
LOG.warning(
"Connection interrupted during arrow stream read, retrying. attempt: %d",
retryCount);
- Thread.sleep(RETRY_DELAY_MS);
+ Thread.sleep(getArrowRetryDelayMs());
} finally {
if (stream != null && !streamCompleted) {
stream.cancel();
@@ -1572,6 +1648,12 @@ int getArrowBufferSize() {
return DEFAULT_ARROW_BUFFER_SIZE;
}
+ /** Wait between two attempts to read an Arrow stream. */
+ @VisibleForTesting
+ long getArrowRetryDelayMs() {
+ return RETRY_DELAY_MS;
+ }
+
/** Returns the destinationTable from jobId by calling `jobs.get` API */
TableId getDestinationTable(JobId jobId) {
Job job = this.bigQuery.getJob(jobId);
diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
index 33c5084a1cf1..43ffbdb2019e 100644
--- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
+++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
@@ -36,6 +36,7 @@
import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
import com.google.cloud.bigquery.storage.v1.ArrowSchema;
import com.google.common.collect.ImmutableList;
+import com.google.protobuf.ByteString;
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Array;
@@ -859,12 +860,20 @@ private static ArrowSchema idArrowSchema() throws IOException {
.build();
}
+ private static ArrowRecordBatch idBatch(int... ids) throws IOException {
+ return ArrowRecordBatch.newBuilder()
+ .setSerializedRecordBatch(serializeVectorSchemaRoot(idRoot(ids)))
+ .build();
+ }
+
private static BigQueryArrowBatchWrapper idWrapper(int... ids) throws IOException {
- ArrowRecordBatch batch =
- ArrowRecordBatch.newBuilder()
- .setSerializedRecordBatch(serializeVectorSchemaRoot(idRoot(ids)))
- .build();
- return BigQueryArrowBatchWrapper.of(batch);
+ return BigQueryArrowBatchWrapper.of(idBatch(ids));
+ }
+
+ /** A stream message: the ids as an Arrow batch, and the row count the message reports. */
+ private static BigQueryArrowBatchWrapper idMessage(long reportedRows, int... ids)
+ throws IOException {
+ return BigQueryArrowBatchWrapper.of(idBatch(ids), null, reportedRows);
}
private BigQueryArrowResultSet idResultSet(
@@ -876,6 +885,96 @@ private BigQueryArrowResultSet idResultSet(
ID_SCHEMA, arrowSchema, totalRows, statement, queue, mock(Future.class), null);
}
+ /**
+ * Reads a stream of one good row, then {@code failing}, then two more good rows. next() must fail
+ * on {@code failing}, leave no row readable, and not move on to the rows behind it. The row limit
+ * (3) is below the stream's total, so if {@code failing} were skipped, the rows behind it would
+ * fill the limit and nothing would report the gap.
+ */
+ private SQLException assertStreamFailsOn(BigQueryArrowBatchWrapper failing)
+ throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ idArrowSchema(),
+ 3,
+ idMessage(1, 1),
+ failing,
+ idMessage(2, 3, 4),
+ arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isTrue();
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(rs.isFirst()).isFalse();
+ assertThrows(SQLException.class, () -> rs.getLong(1));
+ assertThat(rs.next()).isFalse();
+ return e;
+ }
+
+ private void assertRowCountMismatch(BigQueryArrowBatchWrapper message, String expected)
+ throws SQLException, IOException {
+ assertThat(assertStreamFailsOn(message).getCause().getMessage()).contains(expected);
+ }
+
+ @Test
+ public void testRowsReportedWithoutDataThrows() throws SQLException, IOException {
+ assertRowCountMismatch(
+ BigQueryArrowBatchWrapper.of(null, null, 2), "reported 2 rows but its batch has 0");
+ assertRowCountMismatch(
+ BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 2),
+ "reported 2 rows but its batch has 0");
+ }
+
+ @Test
+ public void testRowsReportedBeforeSchemaThrows() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(null, 2, BigQueryArrowBatchWrapper.of(null, null, 2), arrowBatchWrapperLast);
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("reported 2 rows but its batch has 0");
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testUndecodableBatchEndsIteration() throws SQLException, IOException {
+ ArrowRecordBatch whole = idBatch(8, 9);
+ ArrowRecordBatch truncated =
+ ArrowRecordBatch.newBuilder()
+ .setSerializedRecordBatch(
+ whole
+ .getSerializedRecordBatch()
+ .substring(0, whole.getSerializedRecordBatch().size() / 2))
+ .build();
+
+ assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated));
+ }
+
+ @Test
+ public void testBatchRowCountMismatchThrows() throws SQLException, IOException {
+ assertRowCountMismatch(idMessage(3, 1, 2), "reported 3 rows but its batch has 2");
+ assertRowCountMismatch(idMessage(0, 1, 2), "reported 0 rows but its batch has 2");
+ assertRowCountMismatch(idMessage(2), "reported 2 rows but its batch has 0");
+ }
+
+ @Test
+ public void testMatchingReportedRowCountsAreAccepted() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ null,
+ 3,
+ BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0),
+ idMessage(2, 1, 2),
+ BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 0),
+ idMessage(0),
+ idMessage(1, 3),
+ arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.getLong(1)).isEqualTo(3L);
+ assertThat(rs.next()).isFalse();
+ }
+
@Test
public void testMessagesWithoutRowsAreSkipped() throws SQLException, IOException {
BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null);
@@ -925,7 +1024,7 @@ public void testSchemaFromStreamMessage() throws SQLException, IOException {
idResultSet(
null,
1,
- BigQueryArrowBatchWrapper.of(null, idArrowSchema()),
+ BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0),
idWrapper(7),
arrowBatchWrapperLast);
@@ -936,8 +1035,40 @@ public void testSchemaFromStreamMessage() throws SQLException, IOException {
@Test
public void testBatchBeforeSchemaThrows() throws SQLException, IOException {
- BigQueryArrowResultSet rs = idResultSet(null, 1, idWrapper(1), arrowBatchWrapperLast);
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ null,
+ 1,
+ idWrapper(1),
+ BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1),
+ arrowBatchWrapperLast);
assertThrows(SQLException.class, rs::next);
+ // The message behind the failure has a schema and a row. It must not be served.
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testUndecodableSchemaEndsIteration() throws SQLException, IOException {
+ // A negative length prefix: Arrow fails on it with an unchecked exception.
+ ArrowSchema undecodable =
+ ArrowSchema.newBuilder()
+ .setSerializedSchema(
+ ByteString.copyFrom(
+ new byte[] {(byte) 0xFE, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}))
+ .build();
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ null,
+ 1,
+ BigQueryArrowBatchWrapper.of(null, undecodable, 0),
+ BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1),
+ arrowBatchWrapperLast);
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("Failed to create the Arrow deserializer");
+ assertThat(e.getCause().getCause()).isInstanceOf(RuntimeException.class);
+ // The message behind the failure has a schema and a row. It must not be served.
+ assertThat(rs.next()).isFalse();
}
}
diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java
index 96cb173af3a6..82981c53048c 100644
--- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java
+++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java
@@ -162,6 +162,12 @@ public class BigQueryStatementTest {
private static final TableId TABLE_ID = TableId.of(DEFAULT_TEST_DATASET, PROJECT);
+ private static final JobId STREAM_JOB_ID =
+ JobId.newBuilder().setProject("p").setLocation("us-central1").setJob("j").build();
+
+ private static final String JOB_STREAM_NAME =
+ "projects/p/locations/us-central1/jobs/j/streams/_default";
+
private static ArrowSchema arrowSchema;
private final Map LABELS =
@@ -1589,6 +1595,255 @@ public ReadRowsResponse next() {
"Stopping early due to worker shutdown must enqueue an error marker rather than normal end-of-stream");
}
+ private static ReadRowsResponse rowsResponse(long rowCount) {
+ return ReadRowsResponse.newBuilder()
+ .setRowCount(rowCount)
+ .setArrowRecordBatch(ArrowRecordBatch.getDefaultInstance())
+ .build();
+ }
+
+ /** One ReadRows attempt: delivers {@code responses}, then throws {@code failure} unless null. */
+ private static ServerStream readRowsAttempt(
+ RuntimeException failure, ReadRowsResponse... responses) {
+ Iterator delivered = Arrays.asList(responses).iterator();
+ Iterator attempt =
+ new Iterator() {
+ @Override
+ public boolean hasNext() {
+ if (!delivered.hasNext() && failure != null) {
+ throw failure;
+ }
+ return delivered.hasNext();
+ }
+
+ @Override
+ public ReadRowsResponse next() {
+ return delivered.next();
+ }
+ };
+ @SuppressWarnings("unchecked")
+ ServerStream stream = mock(ServerStream.class);
+ doReturn(attempt).when(stream).iterator();
+ return stream;
+ }
+
+ /** A read client that serves {@code attempts} in order and records each request it is sent. */
+ @SafeVarargs
+ private static BigQueryReadClient readClientServing(
+ List requests, ServerStream... attempts) {
+ Iterator> remaining = Arrays.asList(attempts).iterator();
+ @SuppressWarnings("unchecked")
+ ServerStreamingCallable callable =
+ mock(ServerStreamingCallable.class);
+ Mockito.doAnswer(
+ invocation -> {
+ requests.add(invocation.getArgument(0));
+ return remaining.next();
+ })
+ .when(callable)
+ .call(any(ReadRowsRequest.class));
+ BigQueryReadClient readClient = mock(BigQueryReadClient.class);
+ doReturn(callable).when(readClient).readRowsCallable();
+ return readClient;
+ }
+
+ /** A statement with picosecond timestamps on, whose reader retries without waiting. */
+ private BigQueryStatement picosStatementWithoutRetryDelay() throws SQLException {
+ doReturn(true).when(bigQueryConnection).isEnableTimestampPicos();
+ BigQueryStatement statement = Mockito.spy(new BigQueryStatement(bigQueryConnection));
+ doReturn(0L).when(statement).getArrowRetryDelayMs();
+ return statement;
+ }
+
+ @Test
+ public void testJobStreamNameUsesJobReference() throws SQLException {
+ assertEquals(JOB_STREAM_NAME, BigQueryStatement.jobStreamName(STREAM_JOB_ID));
+ }
+
+ @Test
+ public void testJobStreamNameRejectsMissingParts() {
+ for (JobId incomplete :
+ Arrays.asList(
+ null,
+ STREAM_JOB_ID.toBuilder().setProject(null).build(),
+ STREAM_JOB_ID.toBuilder().setLocation(null).build(),
+ STREAM_JOB_ID.toBuilder().setLocation("").build(),
+ STREAM_JOB_ID.toBuilder().setJob(null).build())) {
+ assertThrows(
+ BigQueryJdbcException.class,
+ () -> BigQueryStatement.jobStreamName(incomplete),
+ "Expected no stream name for " + incomplete);
+ }
+ }
+
+ @Test
+ public void testArrowSerializationOptionsDefaultToMicros() {
+ ArrowSerializationOptions options = bigQueryStatement.buildArrowSerializationOptions();
+
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS,
+ options.getPicosTimestampPrecision());
+ assertEquals(
+ ArrowSerializationOptions.CompressionCodec.COMPRESSION_UNSPECIFIED,
+ options.getBufferCompression());
+ }
+
+ @Test
+ public void testArrowSerializationOptionsUsePicosWhenEnabled() throws SQLException {
+ doReturn(true).when(bigQueryConnection).isEnableTimestampPicos();
+ BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
+
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS,
+ statement.buildArrowSerializationOptions().getPicosTimestampPrecision());
+ }
+
+ @Test
+ public void testJobStreamReaderSendsNameOffsetAndOptions() throws Exception {
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient =
+ readClientServing(requests, readRowsAttempt(null, rowsResponse(3)));
+
+ bigQueryStatement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ assertEquals(1, requests.size());
+ ReadRowsRequest request = requests.get(0);
+ assertEquals(JOB_STREAM_NAME, request.getReadStream());
+ assertEquals(1000L, request.getOffset());
+ assertTrue(request.hasArrowSerializationOptions());
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS,
+ request.getArrowSerializationOptions().getPicosTimestampPrecision());
+ }
+
+ @Test
+ public void testJobStreamReaderResumeKeepsOptionsAndOffset() throws Exception {
+ BigQueryStatement statement = picosStatementWithoutRetryDelay();
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient =
+ readClientServing(
+ requests,
+ readRowsAttempt(
+ mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2), rowsResponse(1)),
+ readRowsAttempt(null, rowsResponse(4)));
+
+ statement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ assertEquals(2, requests.size());
+ ReadRowsRequest first = requests.get(0);
+ ReadRowsRequest resumed = requests.get(1);
+ assertEquals(1000L, first.getOffset());
+ assertEquals(1003L, resumed.getOffset());
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS,
+ resumed.getArrowSerializationOptions().getPicosTimestampPrecision());
+ // The resumed request differs from the first one in its offset only.
+ assertEquals(first, resumed.toBuilder().setOffset(first.getOffset()).build());
+
+ // Three messages, then a normal end of stream.
+ List queued = new ArrayList<>(queue);
+ assertEquals(4, queued.size());
+ assertTrue(queued.get(3).isLast());
+ assertNull(queued.get(3).getException());
+ }
+
+ @Test
+ public void testReadSessionReaderSendsNoArrowOptions() throws Exception {
+ // Picoseconds are on, so the session carries the Arrow options. ReadRows on a session stream
+ // must not: the request accepts them for job streams only.
+ BigQueryStatement statement = picosStatementWithoutRetryDelay();
+ String sessionStream = "projects/p/locations/l/sessions/s/streams/0";
+ ReadSession readSession =
+ ReadSession.newBuilder().addStreams(ReadStream.newBuilder().setName(sessionStream)).build();
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient =
+ readClientServing(
+ requests,
+ readRowsAttempt(mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2)),
+ readRowsAttempt(null, rowsResponse(4)));
+
+ statement.populateArrowBufferedQueue(readSession, queue, readClient).get(5, TimeUnit.SECONDS);
+
+ assertEquals(
+ Arrays.asList(
+ ReadRowsRequest.newBuilder().setReadStream(sessionStream).build(),
+ ReadRowsRequest.newBuilder().setReadStream(sessionStream).setOffset(2).build()),
+ requests);
+ }
+
+ @Test
+ public void testJobStreamReaderQueuesMessagesAsReceived() throws Exception {
+ ReadRowsResponse schemaAndRows =
+ rowsResponse(3).toBuilder().setArrowSchema(arrowSchema).build();
+ ReadRowsResponse noRows = ReadRowsResponse.getDefaultInstance();
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ BigQueryReadClient readClient =
+ readClientServing(new ArrayList<>(), readRowsAttempt(null, schemaAndRows, noRows));
+
+ bigQueryStatement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ BigQueryArrowBatchWrapper first = queue.poll();
+ assertEquals(arrowSchema, first.getArrowSchema());
+ assertNotNull(first.getCurrentArrowBatch());
+ assertEquals(3L, first.getReportedRowCount());
+ BigQueryArrowBatchWrapper second = queue.poll();
+ assertNull(second.getArrowSchema());
+ assertNull(second.getCurrentArrowBatch());
+ assertEquals(0L, second.getReportedRowCount());
+ BigQueryArrowBatchWrapper end = queue.poll();
+ assertTrue(end.isLast());
+ assertNull(end.getException());
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testJobStreamReaderDoesNotRetryNotFound() throws Exception {
+ ApiException notFound = mockApiException(StatusCode.Code.NOT_FOUND);
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient = readClientServing(requests, readRowsAttempt(notFound));
+
+ bigQueryStatement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ assertEquals(1, requests.size());
+ assertEquals(notFound, queue.poll().getException());
+ BigQueryArrowBatchWrapper end = queue.poll();
+ assertTrue(end.isLast());
+ assertNull(end.getException());
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testJobStreamReaderRejectsBadArgumentsBeforeStartingAWorker() {
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ JobId withoutProjectAndLocation = JobId.of("j");
+
+ assertThrows(
+ BigQueryJdbcException.class,
+ () ->
+ bigQueryStatement.populateArrowBufferedQueueFromJobStream(
+ STREAM_JOB_ID, -1L, queue, storageReadClient));
+ assertThrows(
+ BigQueryJdbcException.class,
+ () ->
+ bigQueryStatement.populateArrowBufferedQueueFromJobStream(
+ withoutProjectAndLocation, 0L, queue, storageReadClient));
+
+ verify(bigQueryConnection, Mockito.never()).getExecutorService();
+ assertTrue(queue.isEmpty());
+ }
+
private static FieldValueList createTestRow(boolean first, long second) {
return FieldValueList.of(
Arrays.asList(