diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java index bd6041a6f7f4..16d23a571da7 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java @@ -22,11 +22,16 @@ /** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */ class BigQueryArrowBatchWrapper { + private static final long ROW_COUNT_NOT_REPORTED = -1; + // Reference to the current arrowBatch private final ArrowRecordBatch currentArrowBatch; // Arrow schema carried with this batch, set only when the source sent one (e.g. the first // ReadRowsResponse of a stream) private final ArrowSchema arrowSchema; + // Rows the source reported for this message (ReadRowsResponse.row_count), or -1 when the wrapper + // was not built from a stream message (markers, nested records) + private final long reportedRowCount; // Reference to the nested Records, set as null otherwise (Arrays) private final JsonStringArrayList nestedRecords; @@ -38,11 +43,13 @@ class BigQueryArrowBatchWrapper { private BigQueryArrowBatchWrapper( ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema, + long reportedRowCount, JsonStringArrayList nestedRecords, boolean isLast, Throwable exception) { this.currentArrowBatch = currentArrowBatch; this.arrowSchema = arrowSchema; + this.reportedRowCount = reportedRowCount; this.nestedRecords = nestedRecords; this.isLast = isLast; this.exception = exception; @@ -50,25 +57,30 @@ private BigQueryArrowBatchWrapper( static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) { boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0]; - return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, isLastFlag, null); + return new BigQueryArrowBatchWrapper( + currentArrowBatch, null, ROW_COUNT_NOT_REPORTED, null, isLastFlag, null); } /** - * Wraps a record batch together with the schema sent by the source. Either may be null (a - * schema-only message carries no batch; later messages carry no schema). + * Wraps one stream message: its record batch, the schema sent with it, and the row count it + * reports. The batch and the schema may each be null (a schema-only message carries no batch; + * later messages carry no schema). */ - static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema) { - return new BigQueryArrowBatchWrapper(currentArrowBatch, arrowSchema, null, false, null); + static BigQueryArrowBatchWrapper of( + ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema, long reportedRowCount) { + return new BigQueryArrowBatchWrapper( + currentArrowBatch, arrowSchema, reportedRowCount, null, false, null); } static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper( JsonStringArrayList nestedRecords, boolean... isLast) { boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0]; - return new BigQueryArrowBatchWrapper(null, null, nestedRecords, isLastFlag, null); + return new BigQueryArrowBatchWrapper( + null, null, ROW_COUNT_NOT_REPORTED, nestedRecords, isLastFlag, null); } static BigQueryArrowBatchWrapper ofError(Throwable exception) { - return new BigQueryArrowBatchWrapper(null, null, null, true, exception); + return new BigQueryArrowBatchWrapper(null, null, ROW_COUNT_NOT_REPORTED, null, true, exception); } ArrowRecordBatch getCurrentArrowBatch() { @@ -79,6 +91,16 @@ ArrowSchema getArrowSchema() { return this.arrowSchema; } + /** True if this wrapper was built from a stream message, which always reports a row count. */ + boolean hasReportedRowCount() { + return this.reportedRowCount != ROW_COUNT_NOT_REPORTED; + } + + /** Rows the source reported for this message. See {@link #hasReportedRowCount()}. */ + long getReportedRowCount() { + return this.reportedRowCount; + } + JsonStringArrayList getNestedRecords() { return this.nestedRecords; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java index afac21f9c456..ecdefe60d7fc 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java @@ -291,6 +291,10 @@ public boolean next() throws SQLException { this.rowCount++; return true; } catch (SQLException ex) { + // A stream error ends iteration: the next call returns false and no row stays readable. + // The failed message is never skipped, even if the caller ignores this exception. + this.hasReachedEnd = true; + this.currentBatchRowIndex = -1; throw new BigQueryJdbcException( "Error occurred while advancing the cursor. This could happen when connection is closed while the next method is being called.", ex); @@ -320,16 +324,24 @@ private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLExcepti } try { this.arrowDeserializer = new ArrowDeserializer(arrowSchema); - } catch (IOException ex) { - throw new BigQueryJdbcException("IOException during ArrowDeserializer creation", ex); + } catch (RuntimeException | IOException ex) { + // Arrow throws unchecked exceptions on malformed schema bytes. + throw new BigQueryJdbcException("Failed to create the Arrow deserializer", ex); } } /** * Loads the next batch that has rows into the vectors. The reader queues every message as-is, so - * this is the single place that decides what to use: messages without rows (schema, statistics or - * throttling updates) are skipped, and the stream must deliver all {@code totalRows} rows. - * Returns false at the end of the stream. + * this is the single place that decides what to use: + * + * + * + * Returns false at the end of the stream. A failure ends iteration, see {@link #next()}. */ private boolean loadNextBatch() throws SQLException, InterruptedException { while (true) { @@ -354,7 +366,11 @@ private boolean loadNextBatch() throws SQLException, InterruptedException { } initDeserializerIfNeeded(batchWrapper.getArrowSchema()); ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch(); + long reportedRows = batchWrapper.getReportedRowCount(); if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) { + if (reportedRows > 0) { + throw rowCountMismatch(reportedRows, 0); + } continue; // no rows in this message } if (this.arrowDeserializer == null) { @@ -362,12 +378,23 @@ private boolean loadNextBatch() throws SQLException, InterruptedException { } // Populates vectorSchemaRoot this.arrowDeserializer.deserializeArrowBatch(arrowBatch); - if (this.vectorSchemaRoot.getRowCount() > 0) { + int decodedRows = this.vectorSchemaRoot.getRowCount(); + if (batchWrapper.hasReportedRowCount() && reportedRows != decodedRows) { + throw rowCountMismatch(reportedRows, decodedRows); + } + if (decodedRows > 0) { return true; } } } + private static SQLException rowCountMismatch(long reportedRows, long decodedRows) { + return new BigQueryJdbcException( + String.format( + "An Arrow stream message reported %d rows but its batch has %d.", + reportedRows, decodedRows)); + } + @Override Object getObjectInternal(int columnIndex) throws SQLException { LOG.finestTrace("getObjectInternal"); @@ -562,7 +589,8 @@ public boolean isFirst() throws SQLException { if (this.isNested) { return !this.afterLast && this.nestedRowIndex == this.fromIndex; } else { - return !this.afterLast && this.rowCount == 1; + // The index is negative after a stream error: the cursor is on no row then. + return !this.afterLast && this.rowCount == 1 && this.currentBatchRowIndex >= 0; } } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 34903ac6ffb9..ad8e7f5cc737 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -62,6 +62,7 @@ import com.google.cloud.bigquery.storage.v1.ReadSession; import com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.opentelemetry.api.common.AttributeKey; @@ -90,6 +91,7 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; +import java.util.function.Supplier; import java.util.logging.Level; /** @@ -1003,12 +1005,87 @@ private TableReadOptions buildTableReadOptions() { .build(); } + /** + * Arrow options for a job stream. They are sent with every request instead of left to the + * stream's defaults, because they change the schema the stream returns: all requests for one + * result must use the same values. Compression stays unset, which means none. + */ + @VisibleForTesting + ArrowSerializationOptions buildArrowSerializationOptions() { + return ArrowSerializationOptions.newBuilder() + .setPicosTimestampPrecision( + isEnableTimestampPicos() + ? ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS + : ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS) + .build(); + } + + /** + * Name of the stream that carries a query job's results. The API does not return it, so it is + * built from the job reference. The location must be the job's own: the stream does not exist in + * any other location. + */ + @VisibleForTesting + static String jobStreamName(JobId jobId) throws BigQueryJdbcException { + if (jobId == null + || Strings.isNullOrEmpty(jobId.getProject()) + || Strings.isNullOrEmpty(jobId.getLocation()) + || Strings.isNullOrEmpty(jobId.getJob())) { + throw new BigQueryJdbcException( + "A job stream needs the job's project, location and job ID, got: " + jobId); + } + return String.format( + "projects/%s/locations/%s/jobs/%s/streams/_default", + jobId.getProject(), jobId.getLocation(), jobId.getJob()); + } + /** Asynchronously reads results and populates an arrow record queue */ @InternalApi Future populateArrowBufferedQueue( ReadSession readSession, BlockingQueue arrowBatchWrapperBlockingQueue, BigQueryReadClient bqReadClient) { + // Use the first stream to perform reading. The session already carries the read options: + // ReadRows accepts Arrow options for job streams only. + return readRowsIntoArrowQueue( + () -> + ReadRowsRequest.newBuilder().setReadStream(readSession.getStreams(0).getName()).build(), + arrowBatchWrapperBlockingQueue, + bqReadClient); + } + + /** + * Asynchronously reads a query job's results from its {@code _default} stream, starting at row + * {@code startOffset}, and populates an arrow record queue. No read session is created. + */ + @InternalApi + Future populateArrowBufferedQueueFromJobStream( + JobId jobId, + long startOffset, + BlockingQueue arrowBatchWrapperBlockingQueue, + BigQueryReadClient bqReadClient) + throws BigQueryJdbcException { + if (startOffset < 0) { + throw new BigQueryJdbcException("A job stream offset can't be negative: " + startOffset); + } + ReadRowsRequest firstRequest = + ReadRowsRequest.newBuilder() + .setReadStream(jobStreamName(jobId)) + .setOffset(startOffset) + .setArrowSerializationOptions(buildArrowSerializationOptions()) + .build(); + return readRowsIntoArrowQueue(() -> firstRequest, arrowBatchWrapperBlockingQueue, bqReadClient); + } + + /** + * Reads one stream into the queue. Every attempt sends the first request moved forward by the + * rows already read, so a resumed request keeps the stream name and the Arrow options. The first + * request is resolved on the worker, where a failure reaches the queue as an error. + */ + private Future readRowsIntoArrowQueue( + Supplier firstRequestSupplier, + BlockingQueue arrowBatchWrapperBlockingQueue, + BigQueryReadClient bqReadClient) { LOG.finer("++enter++"); ExecutorService executor = connection.getExecutorService(); @@ -1020,17 +1097,15 @@ Future populateArrowBufferedQueue( long rowsRead = 0; int retryCount = 0; try { - // Use the first stream to perform reading. - String streamName = readSession.getStreams(0).getName(); + ReadRowsRequest firstRequest = firstRequestSupplier.get(); while (true) { ServerStream stream = null; boolean streamCompleted = false; try { ReadRowsRequest readRowsRequest = - ReadRowsRequest.newBuilder() - .setReadStream(streamName) - .setOffset(rowsRead) + firstRequest.toBuilder() + .setOffset(firstRequest.getOffset() + rowsRead) .build(); // Process each block of rows as they arrive and decode using our simple row @@ -1048,7 +1123,8 @@ Future populateArrowBufferedQueue( response.hasArrowRecordBatch() ? response.getArrowRecordBatch() : null, - response.hasArrowSchema() ? response.getArrowSchema() : null)); + response.hasArrowSchema() ? response.getArrowSchema() : null, + response.getRowCount())); rowsRead += response.getRowCount(); } if (isWorkerShuttingDown()) { @@ -1059,7 +1135,7 @@ Future populateArrowBufferedQueue( break; } catch (ApiException e) { if (e.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) { - LOG.warning("Read session expired or not found: %s", e.getMessage()); + LOG.warning("Read stream expired or not found: %s", e.getMessage()); enqueueError(arrowBatchWrapperBlockingQueue, e); break; } @@ -1077,7 +1153,7 @@ Future populateArrowBufferedQueue( LOG.warning( "Connection interrupted during arrow stream read, retrying. attempt: %d", retryCount); - Thread.sleep(RETRY_DELAY_MS); + Thread.sleep(getArrowRetryDelayMs()); } finally { if (stream != null && !streamCompleted) { stream.cancel(); @@ -1572,6 +1648,12 @@ int getArrowBufferSize() { return DEFAULT_ARROW_BUFFER_SIZE; } + /** Wait between two attempts to read an Arrow stream. */ + @VisibleForTesting + long getArrowRetryDelayMs() { + return RETRY_DELAY_MS; + } + /** Returns the destinationTable from jobId by calling `jobs.get` API */ TableId getDestinationTable(JobId jobId) { Job job = this.bigQuery.getJob(jobId); diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java index 33c5084a1cf1..43ffbdb2019e 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java @@ -36,6 +36,7 @@ import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; import java.io.IOException; import java.math.BigDecimal; import java.sql.Array; @@ -859,12 +860,20 @@ private static ArrowSchema idArrowSchema() throws IOException { .build(); } + private static ArrowRecordBatch idBatch(int... ids) throws IOException { + return ArrowRecordBatch.newBuilder() + .setSerializedRecordBatch(serializeVectorSchemaRoot(idRoot(ids))) + .build(); + } + private static BigQueryArrowBatchWrapper idWrapper(int... ids) throws IOException { - ArrowRecordBatch batch = - ArrowRecordBatch.newBuilder() - .setSerializedRecordBatch(serializeVectorSchemaRoot(idRoot(ids))) - .build(); - return BigQueryArrowBatchWrapper.of(batch); + return BigQueryArrowBatchWrapper.of(idBatch(ids)); + } + + /** A stream message: the ids as an Arrow batch, and the row count the message reports. */ + private static BigQueryArrowBatchWrapper idMessage(long reportedRows, int... ids) + throws IOException { + return BigQueryArrowBatchWrapper.of(idBatch(ids), null, reportedRows); } private BigQueryArrowResultSet idResultSet( @@ -876,6 +885,96 @@ private BigQueryArrowResultSet idResultSet( ID_SCHEMA, arrowSchema, totalRows, statement, queue, mock(Future.class), null); } + /** + * Reads a stream of one good row, then {@code failing}, then two more good rows. next() must fail + * on {@code failing}, leave no row readable, and not move on to the rows behind it. The row limit + * (3) is below the stream's total, so if {@code failing} were skipped, the rows behind it would + * fill the limit and nothing would report the gap. + */ + private SQLException assertStreamFailsOn(BigQueryArrowBatchWrapper failing) + throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet( + idArrowSchema(), + 3, + idMessage(1, 1), + failing, + idMessage(2, 3, 4), + arrowBatchWrapperLast); + + assertThat(rs.next()).isTrue(); + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(rs.isFirst()).isFalse(); + assertThrows(SQLException.class, () -> rs.getLong(1)); + assertThat(rs.next()).isFalse(); + return e; + } + + private void assertRowCountMismatch(BigQueryArrowBatchWrapper message, String expected) + throws SQLException, IOException { + assertThat(assertStreamFailsOn(message).getCause().getMessage()).contains(expected); + } + + @Test + public void testRowsReportedWithoutDataThrows() throws SQLException, IOException { + assertRowCountMismatch( + BigQueryArrowBatchWrapper.of(null, null, 2), "reported 2 rows but its batch has 0"); + assertRowCountMismatch( + BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 2), + "reported 2 rows but its batch has 0"); + } + + @Test + public void testRowsReportedBeforeSchemaThrows() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet(null, 2, BigQueryArrowBatchWrapper.of(null, null, 2), arrowBatchWrapperLast); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("reported 2 rows but its batch has 0"); + assertThat(rs.next()).isFalse(); + } + + @Test + public void testUndecodableBatchEndsIteration() throws SQLException, IOException { + ArrowRecordBatch whole = idBatch(8, 9); + ArrowRecordBatch truncated = + ArrowRecordBatch.newBuilder() + .setSerializedRecordBatch( + whole + .getSerializedRecordBatch() + .substring(0, whole.getSerializedRecordBatch().size() / 2)) + .build(); + + assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated)); + } + + @Test + public void testBatchRowCountMismatchThrows() throws SQLException, IOException { + assertRowCountMismatch(idMessage(3, 1, 2), "reported 3 rows but its batch has 2"); + assertRowCountMismatch(idMessage(0, 1, 2), "reported 0 rows but its batch has 2"); + assertRowCountMismatch(idMessage(2), "reported 2 rows but its batch has 0"); + } + + @Test + public void testMatchingReportedRowCountsAreAccepted() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet( + null, + 3, + BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0), + idMessage(2, 1, 2), + BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 0), + idMessage(0), + idMessage(1, 3), + arrowBatchWrapperLast); + + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + assertThat(rs.getLong(1)).isEqualTo(3L); + assertThat(rs.next()).isFalse(); + } + @Test public void testMessagesWithoutRowsAreSkipped() throws SQLException, IOException { BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null); @@ -925,7 +1024,7 @@ public void testSchemaFromStreamMessage() throws SQLException, IOException { idResultSet( null, 1, - BigQueryArrowBatchWrapper.of(null, idArrowSchema()), + BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0), idWrapper(7), arrowBatchWrapperLast); @@ -936,8 +1035,40 @@ public void testSchemaFromStreamMessage() throws SQLException, IOException { @Test public void testBatchBeforeSchemaThrows() throws SQLException, IOException { - BigQueryArrowResultSet rs = idResultSet(null, 1, idWrapper(1), arrowBatchWrapperLast); + BigQueryArrowResultSet rs = + idResultSet( + null, + 1, + idWrapper(1), + BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1), + arrowBatchWrapperLast); assertThrows(SQLException.class, rs::next); + // The message behind the failure has a schema and a row. It must not be served. + assertThat(rs.next()).isFalse(); + } + + @Test + public void testUndecodableSchemaEndsIteration() throws SQLException, IOException { + // A negative length prefix: Arrow fails on it with an unchecked exception. + ArrowSchema undecodable = + ArrowSchema.newBuilder() + .setSerializedSchema( + ByteString.copyFrom( + new byte[] {(byte) 0xFE, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF})) + .build(); + BigQueryArrowResultSet rs = + idResultSet( + null, + 1, + BigQueryArrowBatchWrapper.of(null, undecodable, 0), + BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1), + arrowBatchWrapperLast); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("Failed to create the Arrow deserializer"); + assertThat(e.getCause().getCause()).isInstanceOf(RuntimeException.class); + // The message behind the failure has a schema and a row. It must not be served. + assertThat(rs.next()).isFalse(); } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index 96cb173af3a6..82981c53048c 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -162,6 +162,12 @@ public class BigQueryStatementTest { private static final TableId TABLE_ID = TableId.of(DEFAULT_TEST_DATASET, PROJECT); + private static final JobId STREAM_JOB_ID = + JobId.newBuilder().setProject("p").setLocation("us-central1").setJob("j").build(); + + private static final String JOB_STREAM_NAME = + "projects/p/locations/us-central1/jobs/j/streams/_default"; + private static ArrowSchema arrowSchema; private final Map LABELS = @@ -1589,6 +1595,255 @@ public ReadRowsResponse next() { "Stopping early due to worker shutdown must enqueue an error marker rather than normal end-of-stream"); } + private static ReadRowsResponse rowsResponse(long rowCount) { + return ReadRowsResponse.newBuilder() + .setRowCount(rowCount) + .setArrowRecordBatch(ArrowRecordBatch.getDefaultInstance()) + .build(); + } + + /** One ReadRows attempt: delivers {@code responses}, then throws {@code failure} unless null. */ + private static ServerStream readRowsAttempt( + RuntimeException failure, ReadRowsResponse... responses) { + Iterator delivered = Arrays.asList(responses).iterator(); + Iterator attempt = + new Iterator() { + @Override + public boolean hasNext() { + if (!delivered.hasNext() && failure != null) { + throw failure; + } + return delivered.hasNext(); + } + + @Override + public ReadRowsResponse next() { + return delivered.next(); + } + }; + @SuppressWarnings("unchecked") + ServerStream stream = mock(ServerStream.class); + doReturn(attempt).when(stream).iterator(); + return stream; + } + + /** A read client that serves {@code attempts} in order and records each request it is sent. */ + @SafeVarargs + private static BigQueryReadClient readClientServing( + List requests, ServerStream... attempts) { + Iterator> remaining = Arrays.asList(attempts).iterator(); + @SuppressWarnings("unchecked") + ServerStreamingCallable callable = + mock(ServerStreamingCallable.class); + Mockito.doAnswer( + invocation -> { + requests.add(invocation.getArgument(0)); + return remaining.next(); + }) + .when(callable) + .call(any(ReadRowsRequest.class)); + BigQueryReadClient readClient = mock(BigQueryReadClient.class); + doReturn(callable).when(readClient).readRowsCallable(); + return readClient; + } + + /** A statement with picosecond timestamps on, whose reader retries without waiting. */ + private BigQueryStatement picosStatementWithoutRetryDelay() throws SQLException { + doReturn(true).when(bigQueryConnection).isEnableTimestampPicos(); + BigQueryStatement statement = Mockito.spy(new BigQueryStatement(bigQueryConnection)); + doReturn(0L).when(statement).getArrowRetryDelayMs(); + return statement; + } + + @Test + public void testJobStreamNameUsesJobReference() throws SQLException { + assertEquals(JOB_STREAM_NAME, BigQueryStatement.jobStreamName(STREAM_JOB_ID)); + } + + @Test + public void testJobStreamNameRejectsMissingParts() { + for (JobId incomplete : + Arrays.asList( + null, + STREAM_JOB_ID.toBuilder().setProject(null).build(), + STREAM_JOB_ID.toBuilder().setLocation(null).build(), + STREAM_JOB_ID.toBuilder().setLocation("").build(), + STREAM_JOB_ID.toBuilder().setJob(null).build())) { + assertThrows( + BigQueryJdbcException.class, + () -> BigQueryStatement.jobStreamName(incomplete), + "Expected no stream name for " + incomplete); + } + } + + @Test + public void testArrowSerializationOptionsDefaultToMicros() { + ArrowSerializationOptions options = bigQueryStatement.buildArrowSerializationOptions(); + + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS, + options.getPicosTimestampPrecision()); + assertEquals( + ArrowSerializationOptions.CompressionCodec.COMPRESSION_UNSPECIFIED, + options.getBufferCompression()); + } + + @Test + public void testArrowSerializationOptionsUsePicosWhenEnabled() throws SQLException { + doReturn(true).when(bigQueryConnection).isEnableTimestampPicos(); + BigQueryStatement statement = new BigQueryStatement(bigQueryConnection); + + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS, + statement.buildArrowSerializationOptions().getPicosTimestampPrecision()); + } + + @Test + public void testJobStreamReaderSendsNameOffsetAndOptions() throws Exception { + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = + readClientServing(requests, readRowsAttempt(null, rowsResponse(3))); + + bigQueryStatement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + assertEquals(1, requests.size()); + ReadRowsRequest request = requests.get(0); + assertEquals(JOB_STREAM_NAME, request.getReadStream()); + assertEquals(1000L, request.getOffset()); + assertTrue(request.hasArrowSerializationOptions()); + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS, + request.getArrowSerializationOptions().getPicosTimestampPrecision()); + } + + @Test + public void testJobStreamReaderResumeKeepsOptionsAndOffset() throws Exception { + BigQueryStatement statement = picosStatementWithoutRetryDelay(); + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = + readClientServing( + requests, + readRowsAttempt( + mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2), rowsResponse(1)), + readRowsAttempt(null, rowsResponse(4))); + + statement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + assertEquals(2, requests.size()); + ReadRowsRequest first = requests.get(0); + ReadRowsRequest resumed = requests.get(1); + assertEquals(1000L, first.getOffset()); + assertEquals(1003L, resumed.getOffset()); + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS, + resumed.getArrowSerializationOptions().getPicosTimestampPrecision()); + // The resumed request differs from the first one in its offset only. + assertEquals(first, resumed.toBuilder().setOffset(first.getOffset()).build()); + + // Three messages, then a normal end of stream. + List queued = new ArrayList<>(queue); + assertEquals(4, queued.size()); + assertTrue(queued.get(3).isLast()); + assertNull(queued.get(3).getException()); + } + + @Test + public void testReadSessionReaderSendsNoArrowOptions() throws Exception { + // Picoseconds are on, so the session carries the Arrow options. ReadRows on a session stream + // must not: the request accepts them for job streams only. + BigQueryStatement statement = picosStatementWithoutRetryDelay(); + String sessionStream = "projects/p/locations/l/sessions/s/streams/0"; + ReadSession readSession = + ReadSession.newBuilder().addStreams(ReadStream.newBuilder().setName(sessionStream)).build(); + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = + readClientServing( + requests, + readRowsAttempt(mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2)), + readRowsAttempt(null, rowsResponse(4))); + + statement.populateArrowBufferedQueue(readSession, queue, readClient).get(5, TimeUnit.SECONDS); + + assertEquals( + Arrays.asList( + ReadRowsRequest.newBuilder().setReadStream(sessionStream).build(), + ReadRowsRequest.newBuilder().setReadStream(sessionStream).setOffset(2).build()), + requests); + } + + @Test + public void testJobStreamReaderQueuesMessagesAsReceived() throws Exception { + ReadRowsResponse schemaAndRows = + rowsResponse(3).toBuilder().setArrowSchema(arrowSchema).build(); + ReadRowsResponse noRows = ReadRowsResponse.getDefaultInstance(); + BlockingQueue queue = new LinkedBlockingDeque<>(); + BigQueryReadClient readClient = + readClientServing(new ArrayList<>(), readRowsAttempt(null, schemaAndRows, noRows)); + + bigQueryStatement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + BigQueryArrowBatchWrapper first = queue.poll(); + assertEquals(arrowSchema, first.getArrowSchema()); + assertNotNull(first.getCurrentArrowBatch()); + assertEquals(3L, first.getReportedRowCount()); + BigQueryArrowBatchWrapper second = queue.poll(); + assertNull(second.getArrowSchema()); + assertNull(second.getCurrentArrowBatch()); + assertEquals(0L, second.getReportedRowCount()); + BigQueryArrowBatchWrapper end = queue.poll(); + assertTrue(end.isLast()); + assertNull(end.getException()); + assertTrue(queue.isEmpty()); + } + + @Test + public void testJobStreamReaderDoesNotRetryNotFound() throws Exception { + ApiException notFound = mockApiException(StatusCode.Code.NOT_FOUND); + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = readClientServing(requests, readRowsAttempt(notFound)); + + bigQueryStatement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + assertEquals(1, requests.size()); + assertEquals(notFound, queue.poll().getException()); + BigQueryArrowBatchWrapper end = queue.poll(); + assertTrue(end.isLast()); + assertNull(end.getException()); + assertTrue(queue.isEmpty()); + } + + @Test + public void testJobStreamReaderRejectsBadArgumentsBeforeStartingAWorker() { + BlockingQueue queue = new LinkedBlockingDeque<>(); + JobId withoutProjectAndLocation = JobId.of("j"); + + assertThrows( + BigQueryJdbcException.class, + () -> + bigQueryStatement.populateArrowBufferedQueueFromJobStream( + STREAM_JOB_ID, -1L, queue, storageReadClient)); + assertThrows( + BigQueryJdbcException.class, + () -> + bigQueryStatement.populateArrowBufferedQueueFromJobStream( + withoutProjectAndLocation, 0L, queue, storageReadClient)); + + verify(bigQueryConnection, Mockito.never()).getExecutorService(); + assertTrue(queue.isEmpty()); + } + private static FieldValueList createTestRow(boolean first, long second) { return FieldValueList.of( Arrays.asList(