Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,16 @@

/** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */
class BigQueryArrowBatchWrapper {
private static final long ROW_COUNT_NOT_REPORTED = -1;

// Reference to the current arrowBatch
private final ArrowRecordBatch currentArrowBatch;
// Arrow schema carried with this batch, set only when the source sent one (e.g. the first
// ReadRowsResponse of a stream)
private final ArrowSchema arrowSchema;
// Rows the source reported for this message (ReadRowsResponse.row_count), or -1 when the wrapper
// was not built from a stream message (markers, nested records)
private final long reportedRowCount;
// Reference to the nested Records, set as null otherwise (Arrays)
private final JsonStringArrayList nestedRecords;

Expand All @@ -38,37 +43,44 @@ class BigQueryArrowBatchWrapper {
private BigQueryArrowBatchWrapper(
ArrowRecordBatch currentArrowBatch,
ArrowSchema arrowSchema,
long reportedRowCount,
JsonStringArrayList nestedRecords,
boolean isLast,
Throwable exception) {
this.currentArrowBatch = currentArrowBatch;
this.arrowSchema = arrowSchema;
this.reportedRowCount = reportedRowCount;
this.nestedRecords = nestedRecords;
this.isLast = isLast;
this.exception = exception;
}

static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, isLastFlag, null);
return new BigQueryArrowBatchWrapper(
currentArrowBatch, null, ROW_COUNT_NOT_REPORTED, null, isLastFlag, null);
}

/**
* Wraps a record batch together with the schema sent by the source. Either may be null (a
* schema-only message carries no batch; later messages carry no schema).
* Wraps one stream message: its record batch, the schema sent with it, and the row count it
* reports. The batch and the schema may each be null (a schema-only message carries no batch;
* later messages carry no schema).
*/
static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema) {
return new BigQueryArrowBatchWrapper(currentArrowBatch, arrowSchema, null, false, null);
static BigQueryArrowBatchWrapper of(
ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema, long reportedRowCount) {
return new BigQueryArrowBatchWrapper(
currentArrowBatch, arrowSchema, reportedRowCount, null, false, null);
}

static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper(
JsonStringArrayList nestedRecords, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
return new BigQueryArrowBatchWrapper(null, null, nestedRecords, isLastFlag, null);
return new BigQueryArrowBatchWrapper(
null, null, ROW_COUNT_NOT_REPORTED, nestedRecords, isLastFlag, null);
}

static BigQueryArrowBatchWrapper ofError(Throwable exception) {
return new BigQueryArrowBatchWrapper(null, null, null, true, exception);
return new BigQueryArrowBatchWrapper(null, null, ROW_COUNT_NOT_REPORTED, null, true, exception);
}

ArrowRecordBatch getCurrentArrowBatch() {
Expand All @@ -79,6 +91,16 @@ ArrowSchema getArrowSchema() {
return this.arrowSchema;
}

/** True if this wrapper was built from a stream message, which always reports a row count. */
boolean hasReportedRowCount() {
return this.reportedRowCount != ROW_COUNT_NOT_REPORTED;
}

/** Rows the source reported for this message. See {@link #hasReportedRowCount()}. */
long getReportedRowCount() {
return this.reportedRowCount;
}

JsonStringArrayList getNestedRecords() {
return this.nestedRecords;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,10 @@ public boolean next() throws SQLException {
this.rowCount++;
return true;
} catch (SQLException ex) {
// A stream error ends iteration: the next call returns false and no row stays readable.
// The failed message is never skipped, even if the caller ignores this exception.
this.hasReachedEnd = true;
this.currentBatchRowIndex = -1;
throw new BigQueryJdbcException(
"Error occurred while advancing the cursor. This could happen when connection is closed while the next method is being called.",
ex);
Expand Down Expand Up @@ -320,16 +324,24 @@ private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLExcepti
}
try {
this.arrowDeserializer = new ArrowDeserializer(arrowSchema);
} catch (IOException ex) {
throw new BigQueryJdbcException("IOException during ArrowDeserializer creation", ex);
} catch (RuntimeException | IOException ex) {
// Arrow throws unchecked exceptions on malformed schema bytes.
throw new BigQueryJdbcException("Failed to create the Arrow deserializer", ex);
}
}

/**
* Loads the next batch that has rows into the vectors. The reader queues every message as-is, so
* this is the single place that decides what to use: messages without rows (schema, statistics or
* throttling updates) are skipped, and the stream must deliver all {@code totalRows} rows.
* Returns false at the end of the stream.
* this is the single place that decides what to use:
*
* <ul>
* <li>A message without batch data (a schema, statistics or throttling update) is skipped. It
* must not report rows.
* <li>A message with batch data is decoded, and must hold exactly the rows it reports.
* <li>The stream must deliver all {@code totalRows} rows.
* </ul>
*
* Returns false at the end of the stream. A failure ends iteration, see {@link #next()}.
*/
private boolean loadNextBatch() throws SQLException, InterruptedException {
while (true) {
Expand All @@ -354,20 +366,35 @@ private boolean loadNextBatch() throws SQLException, InterruptedException {
}
initDeserializerIfNeeded(batchWrapper.getArrowSchema());
ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch();
long reportedRows = batchWrapper.getReportedRowCount();
if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) {
if (reportedRows > 0) {
throw rowCountMismatch(reportedRows, 0);
}
continue; // no rows in this message
}
if (this.arrowDeserializer == null) {
throw new BigQueryJdbcException("Received an Arrow record batch before its schema.");
}
// Populates vectorSchemaRoot
this.arrowDeserializer.deserializeArrowBatch(arrowBatch);
if (this.vectorSchemaRoot.getRowCount() > 0) {
int decodedRows = this.vectorSchemaRoot.getRowCount();
if (batchWrapper.hasReportedRowCount() && reportedRows != decodedRows) {
throw rowCountMismatch(reportedRows, decodedRows);
}
if (decodedRows > 0) {
return true;
}
}
}

private static SQLException rowCountMismatch(long reportedRows, long decodedRows) {
return new BigQueryJdbcException(
String.format(
"An Arrow stream message reported %d rows but its batch has %d.",
reportedRows, decodedRows));
}

@Override
Object getObjectInternal(int columnIndex) throws SQLException {
LOG.finestTrace("getObjectInternal");
Expand Down Expand Up @@ -562,7 +589,8 @@ public boolean isFirst() throws SQLException {
if (this.isNested) {
return !this.afterLast && this.nestedRowIndex == this.fromIndex;
} else {
return !this.afterLast && this.rowCount == 1;
// The index is negative after a stream error: the cursor is on no row then.
return !this.afterLast && this.rowCount == 1 && this.currentBatchRowIndex >= 0;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import com.google.cloud.bigquery.storage.v1.ReadSession;
import com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import io.opentelemetry.api.common.AttributeKey;
Expand Down Expand Up @@ -90,6 +91,7 @@
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
import java.util.function.Supplier;
import java.util.logging.Level;

/**
Expand Down Expand Up @@ -1003,12 +1005,87 @@ private TableReadOptions buildTableReadOptions() {
.build();
}

/**
* Arrow options for a job stream. They are sent with every request instead of left to the
* stream's defaults, because they change the schema the stream returns: all requests for one
* result must use the same values. Compression stays unset, which means none.
*/
@VisibleForTesting
ArrowSerializationOptions buildArrowSerializationOptions() {
return ArrowSerializationOptions.newBuilder()
.setPicosTimestampPrecision(
isEnableTimestampPicos()
? ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS
: ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS)
.build();
}

/**
* Name of the stream that carries a query job's results. The API does not return it, so it is
* built from the job reference. The location must be the job's own: the stream does not exist in
* any other location.
*/
@VisibleForTesting
static String jobStreamName(JobId jobId) throws BigQueryJdbcException {
if (jobId == null
|| Strings.isNullOrEmpty(jobId.getProject())
|| Strings.isNullOrEmpty(jobId.getLocation())
|| Strings.isNullOrEmpty(jobId.getJob())) {
throw new BigQueryJdbcException(
"A job stream needs the job's project, location and job ID, got: " + jobId);
}
return String.format(
"projects/%s/locations/%s/jobs/%s/streams/_default",
jobId.getProject(), jobId.getLocation(), jobId.getJob());
}

/** Asynchronously reads results and populates an arrow record queue */
@InternalApi
Future<?> populateArrowBufferedQueue(
ReadSession readSession,
BlockingQueue<BigQueryArrowBatchWrapper> arrowBatchWrapperBlockingQueue,
BigQueryReadClient bqReadClient) {
// Use the first stream to perform reading. The session already carries the read options:
// ReadRows accepts Arrow options for job streams only.
return readRowsIntoArrowQueue(
() ->
ReadRowsRequest.newBuilder().setReadStream(readSession.getStreams(0).getName()).build(),
arrowBatchWrapperBlockingQueue,
bqReadClient);
}

/**
* Asynchronously reads a query job's results from its {@code _default} stream, starting at row
* {@code startOffset}, and populates an arrow record queue. No read session is created.
*/
@InternalApi
Future<?> populateArrowBufferedQueueFromJobStream(
JobId jobId,
long startOffset,
BlockingQueue<BigQueryArrowBatchWrapper> arrowBatchWrapperBlockingQueue,
BigQueryReadClient bqReadClient)
throws BigQueryJdbcException {
if (startOffset < 0) {
throw new BigQueryJdbcException("A job stream offset can't be negative: " + startOffset);
}
ReadRowsRequest firstRequest =
ReadRowsRequest.newBuilder()
.setReadStream(jobStreamName(jobId))
.setOffset(startOffset)
.setArrowSerializationOptions(buildArrowSerializationOptions())
.build();
return readRowsIntoArrowQueue(() -> firstRequest, arrowBatchWrapperBlockingQueue, bqReadClient);
}

/**
* Reads one stream into the queue. Every attempt sends the first request moved forward by the
* rows already read, so a resumed request keeps the stream name and the Arrow options. The first
* request is resolved on the worker, where a failure reaches the queue as an error.
*/
private Future<?> readRowsIntoArrowQueue(
Supplier<ReadRowsRequest> firstRequestSupplier,
BlockingQueue<BigQueryArrowBatchWrapper> arrowBatchWrapperBlockingQueue,
BigQueryReadClient bqReadClient) {
LOG.finer("++enter++");

ExecutorService executor = connection.getExecutorService();
Expand All @@ -1020,17 +1097,15 @@ Future<?> populateArrowBufferedQueue(
long rowsRead = 0;
int retryCount = 0;
try {
// Use the first stream to perform reading.
String streamName = readSession.getStreams(0).getName();
ReadRowsRequest firstRequest = firstRequestSupplier.get();

while (true) {
ServerStream<ReadRowsResponse> stream = null;
boolean streamCompleted = false;
try {
ReadRowsRequest readRowsRequest =
ReadRowsRequest.newBuilder()
.setReadStream(streamName)
.setOffset(rowsRead)
firstRequest.toBuilder()
.setOffset(firstRequest.getOffset() + rowsRead)
.build();

// Process each block of rows as they arrive and decode using our simple row
Expand All @@ -1048,7 +1123,8 @@ Future<?> populateArrowBufferedQueue(
response.hasArrowRecordBatch()
? response.getArrowRecordBatch()
: null,
response.hasArrowSchema() ? response.getArrowSchema() : null));
response.hasArrowSchema() ? response.getArrowSchema() : null,
response.getRowCount()));
rowsRead += response.getRowCount();
}
if (isWorkerShuttingDown()) {
Expand All @@ -1059,7 +1135,7 @@ Future<?> populateArrowBufferedQueue(
break;
} catch (ApiException e) {
if (e.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) {
LOG.warning("Read session expired or not found: %s", e.getMessage());
LOG.warning("Read stream expired or not found: %s", e.getMessage());
enqueueError(arrowBatchWrapperBlockingQueue, e);
break;
}
Expand All @@ -1077,7 +1153,7 @@ Future<?> populateArrowBufferedQueue(
LOG.warning(
"Connection interrupted during arrow stream read, retrying. attempt: %d",
retryCount);
Thread.sleep(RETRY_DELAY_MS);
Thread.sleep(getArrowRetryDelayMs());
} finally {
if (stream != null && !streamCompleted) {
stream.cancel();
Expand Down Expand Up @@ -1572,6 +1648,12 @@ int getArrowBufferSize() {
return DEFAULT_ARROW_BUFFER_SIZE;
}

/** Wait between two attempts to read an Arrow stream. */
@VisibleForTesting
long getArrowRetryDelayMs() {
return RETRY_DELAY_MS;
}

/** Returns the destinationTable from jobId by calling `jobs.get` API */
TableId getDestinationTable(JobId jobId) {
Job job = this.bigQuery.getJob(jobId);
Expand Down
Loading
Loading