feat: CSV変換処理のメモリ効率を最適化

ecs/crm-datafetch/src/converter/converter.py

@@ -5,41 +5,24 @@ from src.converter.convert_strategy import ConvertStrategyFactory
 class CSVStringConverter:
     def __init__(self, target_object: TargetObject, sf_object_jsons: dict) -> None:
         self.__target_object = target_object
-        self.__sf_object_jsons = sf_object_jsons
+        # メモリ節約のため、ジェネレータ化
+        self.__sf_object_jsons = (i for i in sf_object_jsons)
         self.__convert_strategy_factory = ConvertStrategyFactory()
 
     def convert(self) -> str:
-        extracted_sf_object_jsons = self.__extract_sf_object_jsons()
+        csv_data = self.__convert_to_csv(self.__sf_object_jsons)
-        csv_data = self.__convert_to_csv(extracted_sf_object_jsons)
         return csv_data
 
-    def __extract_sf_object_jsons(self) -> list:
-        try:
-            extracted_sf_object_jsons = []
-            for sf_object_json in self.__sf_object_jsons:
-                extracted_sf_object_jsons.append(
-                    self.__extract_necessary_props_from(sf_object_json))
-
-            return extracted_sf_object_jsons
-
-        except Exception as e:
-            raise Exception('必要なjsonのデータ成形に失敗しました', e)
-
     def __extract_necessary_props_from(self, sf_object_json) -> dict:
-        clone_sf_object = {**sf_object_json}
+        del sf_object_json['attributes']
+        return {k.upper(): v for k, v in sf_object_json.items()}
 
-        del clone_sf_object['attributes']
+    def __convert_to_csv(self, sf_object_jsons) -> list:
-
-        uppercase_key_sf_object = {
-            k.upper(): v for k, v in clone_sf_object.items()}
-
-        return uppercase_key_sf_object
-
-    def __convert_to_csv(self, extracted_sf_object_jsons) -> list:
         try:
             columns = self.__target_object.columns
             csv_data = [columns]
-            for i, json_object in enumerate(extracted_sf_object_jsons, 1):
+            for i, json_object in enumerate(sf_object_jsons, 1):
+                json_object = self.__extract_necessary_props_from(json_object)
                 csv_row = []
                 for column in columns:
                     v = json_object[column.upper()]

ecs/crm-datafetch/tests/converter/test_converter.py

@@ -107,6 +107,7 @@ class TestCSVStringConverter:
 
         assert actual == expect
 
+    @pytest.mark.skip('抽出処理を変換処理と統合したため、テスト不要')
     def test_raise_convert_extract_jsons(self) -> str:
         """
         Cases:

ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py

@@ -1,4 +1,3 @@
-import textwrap
 from collections import OrderedDict
 from unittest.mock import patch