feat: メモリを効率よく使用するため、SELECT結果をジェネレータにして返すように修正(_to_dataframe以外)

ecs/jskult-webapp/src/db/database.py

@@ -4,6 +4,7 @@ from sqlalchemy.engine.url import URL
 from sqlalchemy.orm import Session, sessionmaker
 from sqlalchemy.pool import Pool
 
+from src.db.db_row_generator import DBRowGenerator
 from src.error.exceptions import DBException
 from src.logging.get_logger import get_logger
 from src.system_var import environment
@@ -125,7 +126,7 @@ class DatabaseClient:
             DBException: DBエラー
 
         Returns:
-            list[dict]: カラム名: 値の辞書リスト
+            DBRowGenerator: カラム名: 値の辞書リストを返すジェネレータオブジェクト
         """
         result = None
         try:
@@ -138,7 +139,7 @@ class DatabaseClient:
         except Exception as e:
             raise DBException(e)
 
-        result_rows = result.mappings().all()
+        result_rows = DBRowGenerator(result.mappings())
         return result_rows
 
     def execute(self, query: str, parameters=None) -> CursorResult:

ecs/jskult-webapp/src/db/db_row_generator.py

@@ -0,0 +1,14 @@
+from sqlalchemy import MappingResult
+
+
+class DBRowGenerator:
+    """DBの検索結果を指定行数ごとに返すジェネレータ
+    """
+    FETCH_MANY_SIZE = 2000
+
+    def __init__(self, mapping_result: MappingResult) -> None:
+        self.mapping_result = mapping_result
+
+    def __iter__(self):
+        yield_per = self.mapping_result.yield_per(self.FETCH_MANY_SIZE)
+        yield from yield_per

ecs/jskult-webapp/src/repositories/bio_sales_lot_repository.py

@@ -71,8 +71,8 @@ class BioSalesLotRepository(BaseRepository):
             query = self.FETCH_SQL.format(where_clause=where_clause, limit=environment.BIO_SEARCH_RESULT_MAX_COUNT + 1)
             logger.debug(f'SQL: {query}')
             result = self._database.execute_select(query, parameter.model_dump())
-            logger.debug(f'count= {len(result)}')
             models = [BioSalesLotDBModel(**r) for r in result]
+            logger.debug(f'count= {len(models)}')
             return models
         except Exception as e:
             logger.exception(f"DB Error : Exception={e.args}")