From 2593b4e3b6b7c90dd4dfe3046a348c519ebabbca Mon Sep 17 00:00:00 2001 From: Gaurav Tarlok Kakkar Date: Sun, 25 Jun 2023 23:07:29 -0700 Subject: [PATCH 1/3] fix: migrate to sqlalchemy 2.0 --- evadb/storage/sqlite_storage_engine.py | 13 +++++++------ setup.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/evadb/storage/sqlite_storage_engine.py b/evadb/storage/sqlite_storage_engine.py index 7b4003b75..b32557f62 100644 --- a/evadb/storage/sqlite_storage_engine.py +++ b/evadb/storage/sqlite_storage_engine.py @@ -59,14 +59,14 @@ def _dict_to_sql_row(self, dict_row: dict, columns: List[ColumnCatalogEntry]): dict_row[col.name] = dict_row[col.name].tolist() return dict_row - def _sql_row_to_dict(self, sql_row: tuple, columns: List[ColumnCatalogEntry]): + def _deserialize_sql_row(self, sql_row: dict, columns: List[ColumnCatalogEntry]): # Deserialize numpy data dict_row = {} for idx, col in enumerate(columns): if col.type == ColumnType.NDARRAY: - dict_row[col.name] = self._serializer.deserialize(sql_row[idx]) + dict_row[col.name] = self._serializer.deserialize(sql_row[col.name]) else: - dict_row[col.name] = sql_row[idx] + dict_row[col.name] = sql_row[col.name] return dict_row def _try_loading_table_via_reflection(self, table_name: str): @@ -177,14 +177,15 @@ def read( """ try: table_to_read = self._try_loading_table_via_reflection(table.name) - result = self._sql_engine.execute(table_to_read.select()) + result = self._sql_session.execute(table_to_read.select()) data_batch = [] row_size = None for row in result: - # Todo: Verify the order of columns in row matches the table.columns # For table read, we provide row_id so that user can also retrieve # row_id from the table. - data_batch.append(self._sql_row_to_dict(row, table.columns)) + data_batch.append( + self._deserialize_sql_row(row._asdict(), table.columns) + ) if row_size is None: row_size = 0 row_size = get_size(data_batch) diff --git a/setup.py b/setup.py index 537e76dba..0ce20993f 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def read(path, encoding="utf-8"): minimal_requirements = [ "numpy>=1.19.5", "pandas>=1.1.5", - "sqlalchemy>=1.4.0,<2.0.0", # BREAKING CHANGES IN 2.0.0 + "sqlalchemy>=2.0.0", "sqlalchemy-utils>=0.36.6", "lark>=1.0.0", "pyyaml>=5.1", From 32abd05b5dfbb29be00d53b57b7f0fad289925e5 Mon Sep 17 00:00:00 2001 From: Gaurav Tarlok Kakkar Date: Tue, 27 Jun 2023 14:54:10 -0700 Subject: [PATCH 2/3] fix delete testcase for sqlalchemy 2 --- evadb/storage/sqlite_storage_engine.py | 2 +- test/integration_tests/test_delete_executor.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/evadb/storage/sqlite_storage_engine.py b/evadb/storage/sqlite_storage_engine.py index b32557f62..1cb59ad0b 100644 --- a/evadb/storage/sqlite_storage_engine.py +++ b/evadb/storage/sqlite_storage_engine.py @@ -213,7 +213,7 @@ def delete( try: table_to_delete_from = self._try_loading_table_via_reflection(table.name) d = table_to_delete_from.delete().where(sqlalchemy_filter_clause) - self._sql_engine.execute(d) + self._sql_session.execute(d) self._sql_session.commit() except Exception as e: err_msg = ( diff --git a/test/integration_tests/test_delete_executor.py b/test/integration_tests/test_delete_executor.py index c59cf7c2d..5c87180f5 100644 --- a/test/integration_tests/test_delete_executor.py +++ b/test/integration_tests/test_delete_executor.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import unittest -from test.markers import macos_skip_marker from test.util import ( file_remove, get_evadb_for_testing, @@ -123,7 +122,6 @@ def test_should_delete_single_image_in_table(self): ) ) - @macos_skip_marker def test_should_delete_tuple_in_table(self): delete_query = """DELETE FROM testDeleteOne WHERE id < 20 OR dummyfloat < 2 AND id < 5 AND 20 > id From 00e173f84f7eec61d96dda674dbbdf0c55f99f59 Mon Sep 17 00:00:00 2001 From: Gaurav Tarlok Kakkar Date: Wed, 28 Jun 2023 01:08:19 -0700 Subject: [PATCH 3/3] minor fix --- evadb/storage/sqlite_storage_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evadb/storage/sqlite_storage_engine.py b/evadb/storage/sqlite_storage_engine.py index 1cb59ad0b..33cd5d715 100644 --- a/evadb/storage/sqlite_storage_engine.py +++ b/evadb/storage/sqlite_storage_engine.py @@ -201,7 +201,7 @@ def read( raise Exception(err_msg) def delete( - self, table: TableCatalogEntry, sqlalchemy_filter_clause: ColumnElement[bool] + self, table: TableCatalogEntry, sqlalchemy_filter_clause: "ColumnElement[bool]" ): """Delete tuples from the table where rows satisfy the where_clause. The current implementation only handles equality predicates.