diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py index 11ecb03b..9deb7edf 100644 --- a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -1,18 +1,22 @@ +import os +import tempfile + from src.aws.s3 import BackupBucket from src.config.objects import TargetObject from src.error.exceptions import FileUploadException -from src.system_var.constants import CSVBK_JP_NAME +from src.system_var.constants import CSVBK_JP_NAME, TEMPORARY_FILENAME from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger +from src.writer.file_writer import CsvWriter -def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime, csv_string: str): +def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime, csv_data: list): """CSVバックアップ処理 Args: target_object (TargetObject): 取得対象オブジェクト情報インスタンス execute_datetime (ExecuteDateTime): 実行日時取得インスタンス - csv_string (str): csvデータ + csv_data (list): csvデータ Raises: FileUploadException: S3のファイルアップロード失敗 @@ -21,27 +25,38 @@ def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: E # ① CSVバックアップ処理の開始ログを出力する target_object_name = target_object.object_name upload_file_name = target_object.upload_file_name + upload_file_key = f'{execute_datetime.to_path()}/{upload_file_name}.csv' logger.info( f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名:[{upload_file_name}.csv]') - try: # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する - backup_bucket = BackupBucket() - backup_bucket.put_csv( - f'{execute_datetime.to_path()}/{upload_file_name}.csv', csv_string) - - logger.debug( - f'D-CSVBK-02 [{target_object_name}] のCSVデータバックアップ 正常終了') - + _backup_csv_file(target_object_name, upload_file_key, csv_data) except Exception as e: raise FileUploadException( 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object_name}] CSVデータのバックアップに失敗しました ファイル名:[{upload_file_name}.csv] エラー内容:[{e}]') - # ③ CSVバックアップ処理の終了ログを出力する logger.info( f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します') # ④ 次の処理へ移行する return + + +def _backup_csv_file(target_object_name: str, upload_file_key: str, csv_data: list) -> None: + + # 一時ファイル書き込み用の領域を確保 + with tempfile.TemporaryDirectory(prefix=f'{target_object_name}_') as tmpdir: + # アップロード用のファイルをローカルに書き出す + local_file_path = os.path.join(tmpdir, TEMPORARY_FILENAME) + writer = CsvWriter(local_file_path, csv_data) + writer.write() + # ファイルからS3に書き込み + backup_bucket = BackupBucket() + backup_bucket.put_csv(upload_file_key, local_file_path) + + logger.debug( + f'D-CSVBK-02 [{upload_file_key}] のCSVデータバックアップ 正常終了') + + return diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index a568fc2f..316b0516 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -83,6 +83,7 @@ CSV_QUOTE_CHAR = '"' FILE_CHAR_CODE = 'utf-8' FILE_MODE_WRITE = 'w' +TEMPORARY_FILENAME = 'temporary_file' # CRM_取得オブジェクト情報ファイル関連 diff --git a/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py index 267e87da..452f01b8 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py @@ -34,12 +34,12 @@ class TestBackupCrmCsvDataProcess: - CSVバックアップ処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) """ # Arrange - csv_string = textwrap.dedent("""\ - "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted" - "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1" - "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0" - "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0" - """) + csv_data = [ + ["Id", "AccountNumber", "LastModifiedDate", "LastModifiedById", "SystemModstamp", "IsDeleted"], + ["TEST001", "test001", "2022-06-01 09:00:00", "1234567", "2022-06-01 09:00:00", 1], + ["TEST002", "test002", "2022-06-01 09:00:00", "1234567", "2022-06-01 09:00:00", 0], + ["TEST003", "test003", "2022-06-01 09:00:00", "1234567", "2022-06-01 09:00:00", 0] + ] target_object_dict = { 'object_name': 'Account', 'columns': [ @@ -60,16 +60,23 @@ class TestBackupCrmCsvDataProcess: monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import') # Act - backup_crm_csv_data_process(target_object, execute_datetime, csv_string) + backup_crm_csv_data_process(target_object, execute_datetime, csv_data) # Assert + expect_csv_value = """\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + """ + expect_file_key = f'data_import/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.csv' # ファイル確認 actual = s3_client.get_object( Bucket=bucket_name, - Key=f'data_import/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.csv') + Key=expect_file_key) - assert actual['Body'].read().decode('utf-8') == csv_string + assert actual['Body'].read().decode('utf-8') == textwrap.dedent(expect_csv_value) # ログの確認 assert generate_log_message_tuple( @@ -104,15 +111,20 @@ class TestBackupCrmCsvDataProcess: execute_datetime = ExecuteDateTime() target_object = TargetObject(target_object_dict, execute_datetime) - with patch('src.backup_crm_csv_data_process.BackupBucket') as mock_backup_bucket: + with patch('src.backup_crm_csv_data_process.BackupBucket') as mock_backup_bucket, \ + patch('src.backup_crm_csv_data_process.CsvWriter') as mock_writer: mock_backup_bucket_inst = mock_backup_bucket.return_value mock_backup_bucket_inst.put_csv.return_value = '' + + mock_writer_inst = mock_writer.return_value + mock_writer_inst.write.return_value = '' # Act backup_crm_csv_data_process(target_object, execute_datetime, '') # Assert assert mock_backup_bucket_inst.put_csv.called is True + assert mock_writer_inst.write.called is True def test_raise_put_csv(self, bucket_name, monkeypatch, caplog): """