diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 722ed7788..202937983 100644 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -10,7 +10,7 @@ from databricks.sql import * from databricks.sql.exc import OperationalError from databricks.sql.thrift_backend import ThriftBackend -from databricks.sql.utils import ExecuteResponse, ParamEscaper, inject_parameters +from databricks.sql.utils import ExecuteResponse, ParamEscaper, convert_arrow_table_to_pandas, inject_parameters from databricks.sql.types import Row from databricks.sql.auth.auth import get_python_sql_connector_auth_provider from databricks.sql.experimental.oauth_persistence import OAuthPersistence @@ -821,32 +821,8 @@ def _convert_arrow_table(self, table): return [ ResultRow(*[v.as_py() for v in r]) for r in zip(*table.itercolumns()) ] - - # Need to use nullable types, as otherwise type can change when there are missing values. - # See https://arrow.apache.org/docs/python/pandas.html#nullable-types - # NOTE: This api is epxerimental https://pandas.pydata.org/pandas-docs/stable/user_guide/integer_na.html - dtype_mapping = { - pyarrow.int8(): pandas.Int8Dtype(), - pyarrow.int16(): pandas.Int16Dtype(), - pyarrow.int32(): pandas.Int32Dtype(), - pyarrow.int64(): pandas.Int64Dtype(), - pyarrow.uint8(): pandas.UInt8Dtype(), - pyarrow.uint16(): pandas.UInt16Dtype(), - pyarrow.uint32(): pandas.UInt32Dtype(), - pyarrow.uint64(): pandas.UInt64Dtype(), - pyarrow.bool_(): pandas.BooleanDtype(), - pyarrow.float32(): pandas.Float32Dtype(), - pyarrow.float64(): pandas.Float64Dtype(), - pyarrow.string(): pandas.StringDtype(), - } - - # Need to rename columns, as the to_pandas function cannot handle duplicate column names - table_renamed = table.rename_columns([str(c) for c in range(table.num_columns)]) - df = table_renamed.to_pandas( - types_mapper=dtype_mapping.get, - date_as_object=True, - timestamp_as_object=True, - ) + + df = convert_arrow_table_to_pandas(table) res = df.to_numpy(na_value=None) return [ResultRow(*v) for v in res] diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index ed5581367..bed417bd7 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -3,6 +3,7 @@ import datetime, decimal from enum import Enum from typing import Dict +import pandas import pyarrow from databricks.sql import exc @@ -183,3 +184,29 @@ def escape_item(self, item): def inject_parameters(operation: str, parameters: Dict[str, str]): return operation % parameters + + +def convert_arrow_table_to_pandas(table: pyarrow.Table) -> pandas.DataFrame: + # Need to use nullable types, as otherwise type can change when there are missing values. + # See https://arrow.apache.org/docs/python/pandas.html#nullable-types + # NOTE: This api is epxerimental https://pandas.pydata.org/pandas-docs/stable/user_guide/integer_na.html + dtype_mapping = { + pyarrow.int8(): pandas.Int8Dtype(), + pyarrow.int16(): pandas.Int16Dtype(), + pyarrow.int32(): pandas.Int32Dtype(), + pyarrow.int64(): pandas.Int64Dtype(), + pyarrow.uint8(): pandas.UInt8Dtype(), + pyarrow.uint16(): pandas.UInt16Dtype(), + pyarrow.uint32(): pandas.UInt32Dtype(), + pyarrow.uint64(): pandas.UInt64Dtype(), + pyarrow.bool_(): pandas.BooleanDtype(), + pyarrow.float32(): pandas.Float32Dtype(), + pyarrow.float64(): pandas.Float64Dtype(), + pyarrow.string(): pandas.StringDtype(), + } + + return table.to_pandas( + types_mapper=dtype_mapping.get, + date_as_object=True, + timestamp_as_object=True, + )