From 0a3ba9dbc2e2586e3d2bd03de8fbd05441067473 Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Mon, 16 Nov 2026 10:15:10 -0800 Subject: [PATCH 1/3] GH-1491: Release the batch when a C Data stream import fails before taking it ArrowArrayStreamReader.loadNextBatch now releases the batch when the import fails before moving it out of the array, e.g. allocating the schema struct or the struct the batch moves into. Before, the struct was closed without calling its release callback, which leaked the producer's batch. Co-Authored-By: Claude Opus 5.5 --- .../arrow/c/ArrowArrayStreamReader.java | 11 ++- .../apache/arrow/c/ImportOutOfMemoryTest.java | 75 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java index 34a9c4ec03..c1c4910afd 100644 --- a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java +++ b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java @@ -64,7 +64,16 @@ public boolean loadNextBatch() throws IOException { if (array.snapshot().release == NULL) { return false; } - Data.importIntoVectorSchemaRoot(allocator, array, getVectorSchemaRoot(), provider); + try { + Data.importIntoVectorSchemaRoot(allocator, array, getVectorSchemaRoot(), provider); + } catch (Throwable t) { + // If the import failed before moving the batch out of the array, e.g. allocating the + // schema, the batch is still in the array: release it. + if (array.snapshot().release != NULL) { + array.release(); + } + throw t; + } return true; } } diff --git a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java index 7c099f2ef0..1e5a25ae1d 100644 --- a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java +++ b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java @@ -16,11 +16,14 @@ */ package org.apache.arrow.c; +import static org.apache.arrow.c.NativeUtil.NULL; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; @@ -28,6 +31,10 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -99,6 +106,74 @@ public void importOomDoesNotLeakExportedArray() { } } + @Test + public void streamImportOomDoesNotLeakExportedBatch() throws Exception { + // Reading the schema lazily, the first batch can fail allocating the schema struct. + streamImportOomDoesNotLeakExportedBatch(false); + // Reading the schema first, it can fail allocating the struct the batch moves into. + streamImportOomDoesNotLeakExportedBatch(true); + } + + private void streamImportOomDoesNotLeakExportedBatch(boolean schemaFirst) throws Exception { + // Each limit fails the import at a later allocation. Whichever fails, the exported batch must + // be released, which drains the producer. + boolean imported = false; + for (long limit = 0; !imported; limit += 8) { + try (BufferAllocator producer = root.newChildAllocator("producer", 0, Long.MAX_VALUE)) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(producer)) { + Data.exportArrayStream(producer, streamOfOneBatch(producer), stream); + try (BufferAllocator consumer = root.newChildAllocator("consumer", 0, limit)) { + imported = importStream(consumer, stream, schemaFirst); + } finally { + // The stream is still ours if importing it failed. + if (stream.snapshot().release != NULL) { + stream.release(); + } + } + } + assertEquals( + 0L, + producer.getAllocatedMemory(), + "import OOM leaked the exported batch with a consumer limit of " + + limit + + (schemaFirst ? ", reading the schema first" : "")); + } + } + } + + /** Reads {@code stream} in {@code consumer}; returns false if it ran out of memory. */ + private static boolean importStream( + BufferAllocator consumer, ArrowArrayStream stream, boolean schemaFirst) throws Exception { + // Keep the struct open, so the caller can tell whether the import took the stream. + try (ArrowReader reader = Data.importArrayStream(consumer, stream, false)) { + if (schemaFirst) { + reader.getVectorSchemaRoot(); + } + assertTrue(reader.loadNextBatch()); + assertEquals(ROWS, reader.getVectorSchemaRoot().getRowCount()); + return !reader.loadNextBatch(); + } catch (Exception e) { + assertTrue(hasOutOfMemoryCause(e), "import failure must be an allocator OOM: " + e); + return false; + } + } + + /** A reader of one VarChar batch allocated on {@code alloc}. */ + private static ArrowReader streamOfOneBatch(BufferAllocator alloc) { + VarCharVector vector = new VarCharVector("col", alloc); + vector.allocateNew((long) ROWS * 8, ROWS); + for (int r = 0; r < ROWS; r++) { + vector.setSafe(r, ("v" + r).getBytes(StandardCharsets.UTF_8)); + } + vector.setValueCount(ROWS); + try (VectorSchemaRoot source = new VectorSchemaRoot(Collections.singletonList(vector))) { + List batches = new ArrayList<>(); + batches.add(new VectorUnloader(source).getRecordBatch()); + return new StreamTest.InMemoryArrowReader( + alloc, source.getSchema(), batches, new DictionaryProvider.MapDictionaryProvider()); + } + } + /** True if {@code t} is, or is caused by, an Arrow {@link OutOfMemoryException}. */ private static boolean hasOutOfMemoryCause(Throwable t) { for (Throwable cause = t; cause != null; cause = cause.getCause()) { From dc152437c2021a8591884c89416235b495698ef3 Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Mon, 16 Nov 2026 20:54:13 -0800 Subject: [PATCH 2/3] Parameterize the stream import OOM test with JUnit 5 Co-Authored-By: Claude Opus 5.5 --- .../apache/arrow/c/ImportOutOfMemoryTest.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java index 1e5a25ae1d..9979318cba 100644 --- a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java +++ b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java @@ -39,6 +39,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Regression test: a mid-import {@link OutOfMemoryException} must not leak the imported array. @@ -106,17 +108,13 @@ public void importOomDoesNotLeakExportedArray() { } } - @Test - public void streamImportOomDoesNotLeakExportedBatch() throws Exception { - // Reading the schema lazily, the first batch can fail allocating the schema struct. - streamImportOomDoesNotLeakExportedBatch(false); - // Reading the schema first, it can fail allocating the struct the batch moves into. - streamImportOomDoesNotLeakExportedBatch(true); - } - - private void streamImportOomDoesNotLeakExportedBatch(boolean schemaFirst) throws Exception { - // Each limit fails the import at a later allocation. Whichever fails, the exported batch must - // be released, which drains the producer. + @ParameterizedTest(name = "schemaFirst = {0}") + @ValueSource(booleans = {false, true}) + public void streamImportOomDoesNotLeakExportedBatch(boolean schemaFirst) throws Exception { + // Reading the schema lazily, the first batch can fail allocating the schema struct. Reading the + // schema first, it can fail allocating the struct the batch moves into. Each limit fails the + // import at a later allocation. Whichever fails, the exported batch must be released, which + // drains the producer. boolean imported = false; for (long limit = 0; !imported; limit += 8) { try (BufferAllocator producer = root.newChildAllocator("producer", 0, Long.MAX_VALUE)) { From 2d037af457faa6b6ec4743ec090099955f202697 Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Wed, 18 Nov 2026 08:56:05 -0800 Subject: [PATCH 3/3] Keep the imported array open until the stream import failure is handled The 4-argument importIntoVectorSchemaRoot closes the array before unloading and loading the batch, so a failure there made the failure handling read a closed array: the NullPointerException masked the original exception. Import with closeImportedStructs = false and let the enclosing try-with-resources close the array, and add a failure to release the batch to the original exception as suppressed. Co-Authored-By: Claude Opus 5.5 --- .../org/apache/arrow/c/ArrowArrayStreamReader.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java index c1c4910afd..a052d1b02e 100644 --- a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java +++ b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java @@ -65,12 +65,19 @@ public boolean loadNextBatch() throws IOException { return false; } try { - Data.importIntoVectorSchemaRoot(allocator, array, getVectorSchemaRoot(), provider); + // Keep the array open, so the failure handling below can still read it: it is closed by + // the enclosing try-with-resources. + Data.importIntoVectorSchemaRoot( + allocator, array, getVectorSchemaRoot(), provider, /* closeImportedStructs= */ false); } catch (Throwable t) { // If the import failed before moving the batch out of the array, e.g. allocating the // schema, the batch is still in the array: release it. if (array.snapshot().release != NULL) { - array.release(); + try { + array.release(); + } catch (Throwable suppressed) { + t.addSuppressed(suppressed); + } } throw t; }