From 42541e084e39fdb6ec109869199828cbd21ac9e0 Mon Sep 17 00:00:00 2001 From: henriquejsza Date: Fri, 30 Oct 2026 11:40:47 -0300 Subject: [PATCH] GH-551: [Java] Add TimeStamp*TZ support to ComplexCopier ComplexCopier skipped every minor type that has type parameters except Decimal, so copying a list or struct containing TimeStampSecTZ, TimeStampMilliTZ, TimeStampMicroTZ, or TimeStampNanoTZ threw UnsupportedOperationException. Generate the copy and the list and struct writer-lookup cases for the timezone-aware timestamp types, and write the values with their timezone so the copy keeps it. Map keys and values of these types still throw, since UnionMapWriter has no TZ overrides for them yet. Closes #551. Signed-off-by: henriquejsza --- .../main/codegen/templates/ComplexCopier.java | 26 +++- .../complex/impl/TestComplexCopier.java | 142 ++++++++++++++++++ 2 files changed, 166 insertions(+), 2 deletions(-) diff --git a/vector/src/main/codegen/templates/ComplexCopier.java b/vector/src/main/codegen/templates/ComplexCopier.java index f4711339aa..db892154e0 100644 --- a/vector/src/main/codegen/templates/ComplexCopier.java +++ b/vector/src/main/codegen/templates/ComplexCopier.java @@ -30,6 +30,10 @@ <#include "/@includes/vv_imports.ftl" /> +<#function is_timestamp_tz type> + <#return type?starts_with("TimeStamp") && type?ends_with("TZ")> + + /* * This class is generated using freemarker and the ${.template_name} template. */ @@ -123,14 +127,21 @@ public static void copy(FieldReader reader, FieldWriter writer) { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class?starts_with("Decimal") > + <#if !minor.typeParams?? || minor.class?starts_with("Decimal") || is_timestamp_tz(minor.class) > case ${name?upper_case}: if (reader.isSet()) { Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder(); reader.read(${uncappedName}Holder); if (${uncappedName}Holder.isSet == 1) { + <#if is_timestamp_tz(minor.class)> + ${name}Holder holder = new ${name}Holder(); + holder.value = ${uncappedName}Holder.value; + holder.timezone = ${uncappedName}Holder.timezone; + writer.write(holder); + <#else> writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, <#if minor.class?starts_with("Decimal")>, new ArrowType.Decimal(${uncappedName}Holder.precision, ${uncappedName}Holder.scale, ${name}Holder.WIDTH * 8)); + } } else { writer.writeNull(); @@ -160,6 +171,17 @@ private static FieldWriter getStructWriterForReader(FieldReader reader, StructWr return (FieldWriter) writer.${uncappedName}(name); } + <#if is_timestamp_tz(minor.class)> + case ${name?upper_case}: + if (reader.getField().getType() instanceof ArrowType.Timestamp) { + ArrowType.Timestamp type = (ArrowType.Timestamp) reader.getField().getType(); + return (FieldWriter) writer.${uncappedName}(name, type.getTimezone()); + } else { + throw new UnsupportedOperationException( + "Cannot copy " + reader.getMinorType() + " from a field of type " + + reader.getField().getType()); + } + case STRUCT: @@ -184,7 +206,7 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class?starts_with("Decimal") > + <#if !minor.typeParams?? || minor.class?starts_with("Decimal") || is_timestamp_tz(minor.class) > case ${name?upper_case}: return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}(); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index e108894e49..40aec20f2c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -25,6 +25,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.compare.VectorEqualsVisitor; import org.apache.arrow.vector.complex.FixedSizeListVector; @@ -39,6 +41,8 @@ import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.TimeStampNanoTZHolder; +import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; @@ -1006,4 +1010,142 @@ public void testCopyExtensionTypeFromNullValuedNullableUuidHolderReader() { assertTrue(target.isNull(0)); } } + + @Test + public void testCopyListOfTimeStampNanoTZVector() { + try (ListVector from = ListVector.empty("v", allocator); + ListVector to = ListVector.empty("v", allocator)) { + + UnionListWriter listWriter = from.getWriter(); + listWriter.allocate(); + + TimeStampNanoTZHolder holder = new TimeStampNanoTZHolder(); + holder.timezone = "America/Sao_Paulo"; + for (int i = 0; i < COUNT; i++) { + listWriter.setPosition(i); + listWriter.startList(); + holder.value = i * 1_000_000L; + listWriter.timeStampNanoTZ().write(holder); + holder.value = i * 2_000_000L; + listWriter.timeStampNanoTZ().write(holder); + listWriter.endList(); + } + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out); + } + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } + + @Test + public void testCopyStructOfTimeStampNanoTZVector() { + try (final StructVector from = StructVector.empty("v", allocator); + final StructVector to = StructVector.empty("v", allocator)) { + + from.allocateNewSafe(); + + NullableStructWriter structWriter = from.getWriter(); + TimeStampNanoTZHolder holder = new TimeStampNanoTZHolder(); + holder.timezone = "America/Sao_Paulo"; + for (int i = 0; i < COUNT; i++) { + structWriter.setPosition(i); + structWriter.start(); + holder.value = i * 1_000_000L; + structWriter.timeStampNanoTZ("ts", holder.timezone).write(holder); + structWriter.end(); + } + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out); + } + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } + + @Test + public void testCopyStructOfTimeStampNanoTZInUnionThrows() { + try (final StructVector from = StructVector.empty("v", allocator); + final StructVector to = StructVector.empty("v", allocator)) { + + from.allocateNewSafe(); + + NullableStructWriter structWriter = from.getWriter(); + TimeStampNanoTZHolder holder = new TimeStampNanoTZHolder(); + holder.timezone = "America/Sao_Paulo"; + holder.value = 1_000_000L; + structWriter.setPosition(0); + structWriter.start(); + structWriter.timeStampNanoTZ("ts", holder.timezone).write(holder); + structWriter.end(); + // writing an int to the same field promotes it to a union + structWriter.setPosition(1); + structWriter.start(); + structWriter.integer("ts").writeInt(1); + structWriter.end(); + from.setValueCount(2); + + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + in.setPosition(0); + out.setPosition(0); + assertThrows(UnsupportedOperationException.class, () -> ComplexCopier.copy(in, out)); + } + } + + @Test + public void testCopyMapOfTimeStampNanoTZVectorThrows() { + try (final MapVector from = MapVector.empty("v", allocator, false); + final MapVector to = MapVector.empty("v", allocator, false)) { + + StructVector entries = + (StructVector) + from.addOrGetVector(FieldType.notNullable(ArrowType.Struct.INSTANCE)).getVector(); + IntVector keys = + entries.addOrGet( + MapVector.KEY_NAME, + FieldType.notNullable(Types.MinorType.INT.getType()), + IntVector.class); + TimeStampNanoTZVector values = + entries.addOrGet( + MapVector.VALUE_NAME, + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "America/Sao_Paulo")), + TimeStampNanoTZVector.class); + from.allocateNew(); + + int offset = from.startNewValue(0); + keys.setSafe(offset, 1); + values.setSafe(offset, 1_000_000L); + entries.setIndexDefined(offset); + from.endValue(0, 1); + from.setValueCount(1); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + in.setPosition(0); + out.setPosition(0); + UnsupportedOperationException e = + assertThrows(UnsupportedOperationException.class, () -> ComplexCopier.copy(in, out)); + assertEquals("TIMESTAMPNANOTZ", e.getMessage()); + } + } }