diff --git a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java index 34a9c4ec03..a052d1b02e 100644 --- a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java +++ b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java @@ -64,7 +64,23 @@ public boolean loadNextBatch() throws IOException { if (array.snapshot().release == NULL) { return false; } - Data.importIntoVectorSchemaRoot(allocator, array, getVectorSchemaRoot(), provider); + try { + // Keep the array open, so the failure handling below can still read it: it is closed by + // the enclosing try-with-resources. + Data.importIntoVectorSchemaRoot( + allocator, array, getVectorSchemaRoot(), provider, /* closeImportedStructs= */ false); + } catch (Throwable t) { + // If the import failed before moving the batch out of the array, e.g. allocating the + // schema, the batch is still in the array: release it. + if (array.snapshot().release != NULL) { + try { + array.release(); + } catch (Throwable suppressed) { + t.addSuppressed(suppressed); + } + } + throw t; + } return true; } } diff --git a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java index 7c099f2ef0..9979318cba 100644 --- a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java +++ b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java @@ -16,11 +16,14 @@ */ package org.apache.arrow.c; +import static org.apache.arrow.c.NativeUtil.NULL; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; @@ -28,10 +31,16 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Regression test: a mid-import {@link OutOfMemoryException} must not leak the imported array. @@ -99,6 +108,70 @@ public void importOomDoesNotLeakExportedArray() { } } + @ParameterizedTest(name = "schemaFirst = {0}") + @ValueSource(booleans = {false, true}) + public void streamImportOomDoesNotLeakExportedBatch(boolean schemaFirst) throws Exception { + // Reading the schema lazily, the first batch can fail allocating the schema struct. Reading the + // schema first, it can fail allocating the struct the batch moves into. Each limit fails the + // import at a later allocation. Whichever fails, the exported batch must be released, which + // drains the producer. + boolean imported = false; + for (long limit = 0; !imported; limit += 8) { + try (BufferAllocator producer = root.newChildAllocator("producer", 0, Long.MAX_VALUE)) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(producer)) { + Data.exportArrayStream(producer, streamOfOneBatch(producer), stream); + try (BufferAllocator consumer = root.newChildAllocator("consumer", 0, limit)) { + imported = importStream(consumer, stream, schemaFirst); + } finally { + // The stream is still ours if importing it failed. + if (stream.snapshot().release != NULL) { + stream.release(); + } + } + } + assertEquals( + 0L, + producer.getAllocatedMemory(), + "import OOM leaked the exported batch with a consumer limit of " + + limit + + (schemaFirst ? ", reading the schema first" : "")); + } + } + } + + /** Reads {@code stream} in {@code consumer}; returns false if it ran out of memory. */ + private static boolean importStream( + BufferAllocator consumer, ArrowArrayStream stream, boolean schemaFirst) throws Exception { + // Keep the struct open, so the caller can tell whether the import took the stream. + try (ArrowReader reader = Data.importArrayStream(consumer, stream, false)) { + if (schemaFirst) { + reader.getVectorSchemaRoot(); + } + assertTrue(reader.loadNextBatch()); + assertEquals(ROWS, reader.getVectorSchemaRoot().getRowCount()); + return !reader.loadNextBatch(); + } catch (Exception e) { + assertTrue(hasOutOfMemoryCause(e), "import failure must be an allocator OOM: " + e); + return false; + } + } + + /** A reader of one VarChar batch allocated on {@code alloc}. */ + private static ArrowReader streamOfOneBatch(BufferAllocator alloc) { + VarCharVector vector = new VarCharVector("col", alloc); + vector.allocateNew((long) ROWS * 8, ROWS); + for (int r = 0; r < ROWS; r++) { + vector.setSafe(r, ("v" + r).getBytes(StandardCharsets.UTF_8)); + } + vector.setValueCount(ROWS); + try (VectorSchemaRoot source = new VectorSchemaRoot(Collections.singletonList(vector))) { + List batches = new ArrayList<>(); + batches.add(new VectorUnloader(source).getRecordBatch()); + return new StreamTest.InMemoryArrowReader( + alloc, source.getSchema(), batches, new DictionaryProvider.MapDictionaryProvider()); + } + } + /** True if {@code t} is, or is caused by, an Arrow {@link OutOfMemoryException}. */ private static boolean hasOutOfMemoryCause(Throwable t) { for (Throwable cause = t; cause != null; cause = cause.getCause()) {