From 373f758e76f9ee378434afaa28d0ed24b32a8aa1 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 12 Oct 2022 22:01:14 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9C=80=E7=B5=82=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=97=A5=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=A2=E3=83=83?= =?UTF-8?q?=E3=83=97=E3=83=AD=E3=83=BC=E3=83=89=E5=87=A6=E7=90=86=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/upload_last_fetch_datetime_process.py | 40 ++++++--- ...test_upload_last_fetch_datetime_process.py | 85 +++++++++++++++++-- 2 files changed, 104 insertions(+), 21 deletions(-) diff --git a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index e6b34c53..3028ad89 100644 --- a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,10 +1,12 @@ -import json +import os +import tempfile from src.aws.s3 import ConfigBucket from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileUploadException -from src.system_var.constants import UPD_JP_NAME +from src.system_var.constants import TEMPORARY_FILENAME, UPD_JP_NAME from src.util.logger import logger_instance as logger +from src.writer.file_writer import JsonWriter def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): @@ -18,15 +20,16 @@ def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_d FileUploadException: S3のファイルアップロード失敗 """ + object_name = target_object.object_name # ① 前回取得日時ファイル更新処理の開始ログを出力する logger.info( - f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') + f'I-UPD-01 [{object_name}] の前回取得日時ファイルの更新処理を開始します') try: if target_object.is_update_last_fetch_datetime is False: # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする logger.info( - f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') + f'I-UPD-02 [{object_name}] の前回取得日時ファイルの更新処理をスキップします') return # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する @@ -35,23 +38,32 @@ def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_d 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, 'last_fetch_datetime_to': '' } - - config_bucket = ConfigBucket() - config_bucket.put_last_fetch_datetime_file( - target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) - - logger.info( - f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') - + _upload_last_fetch_datetime(target_object, last_fetch_datetime_dict) except Exception as e: raise FileUploadException( 'E-UPD-01', UPD_JP_NAME, - f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') + f'[{object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') # ④ 前回取得日時ファイル更新処理の終了ログを出力する logger.info( - f'I-UPD-04 [{target_object.object_name}] の前回取得日時ファイルの更新処理を終了します') + f'I-UPD-04 [{object_name}] の前回取得日時ファイルの更新処理を終了します') # ⑤ 次の処理へ移行する return + + +def _upload_last_fetch_datetime(target_object: TargetObject, last_fetch_datetime_dict: dict) -> None: + # 一時ファイル書き込み用の領域を確保 + with tempfile.TemporaryDirectory(prefix=f'{target_object.object_name}_') as tmpdir: + # アップロード用のファイルをローカルに書き出す + local_file_path = os.path.join(tmpdir, TEMPORARY_FILENAME) + writer = JsonWriter(local_file_path, last_fetch_datetime_dict) + writer.write() + # ファイルからS3に書き込み + config_bucket = ConfigBucket() + config_bucket.put_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name, local_file_path) + + logger.info( + f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') diff --git a/ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py index e04c7943..6334e77d 100644 --- a/ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py @@ -118,11 +118,19 @@ class TestUploadLastFetchDatetimeProcess: last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) # Act - with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket): + with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \ + patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer: + mock_writer_inst = mock_writer.return_value + mock_writer_inst.write.return_value = '' + + mock_config_bucket_inst = mock_config_bucket.return_value + mock_config_bucket_inst.put_last_fetch_datetime_file.return_value = '' + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) # Assert # 処理実行確認 + assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is False assert mock_config_bucket.called is False # ログの確認 @@ -143,7 +151,6 @@ class TestUploadLastFetchDatetimeProcess: """ # Arrange - mock_config_bucket = MagicMock(return_value=None) target_objects_dict = { 'object_name': 'Account', @@ -165,11 +172,19 @@ class TestUploadLastFetchDatetimeProcess: last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) # Act - with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket): + with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \ + patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer: + mock_writer_inst = mock_writer.return_value + mock_writer_inst.write.return_value = '' + + mock_config_bucket_inst = mock_config_bucket.return_value + mock_config_bucket_inst.put_last_fetch_datetime_file.return_value = '' + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) # Assert - assert mock_config_bucket.called is True + assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is True + assert mock_writer_inst.write.called is True def test_raise_put_last_fetch_datetime_file(self, monkeypatch): """ @@ -185,7 +200,6 @@ class TestUploadLastFetchDatetimeProcess: """ # Arrange - mock_config_bucket = MagicMock(side_effect=Exception('ファイルアップロードエラー')) target_objects_dict = { 'object_name': 'Account', @@ -207,12 +221,69 @@ class TestUploadLastFetchDatetimeProcess: last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) # Act - with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket): + with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \ + patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer: + mock_writer_inst = mock_writer.return_value + mock_writer_inst.write.return_value = '' + mock_config_bucket_inst = mock_config_bucket.return_value + mock_config_bucket_inst.put_last_fetch_datetime_file.side_effect = Exception('ファイルアップロードエラー') with pytest.raises(FileUploadException) as e: upload_last_fetch_datetime_process(target_object, last_fetch_datetime) # Assert - assert mock_config_bucket.called is True + assert mock_writer_inst.write.called is True + assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is True assert e.value.error_id == 'E-UPD-01' assert e.value.func_name == UPD_JP_NAME assert e.value.args[0] == f'[Account] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[Account.json] エラー内容:[ファイルアップロードエラー]' + + def test_raise_put_last_fetch_datetime_file_write_local_file(self, monkeypatch): + """ + Cases: + 前回取得日時ファイルをアップロードするためのローカルファイルの書き込みに失敗した場合、エラーが発生すること + Arranges: + - 前回取得日時ファイル更新処理で例外を発生させるモックを生成する + - 取得処理実施結果を準備する + - 実行日時取得インスタンスを生成する + Expects: + - 例外が発生する + - ファイルが書き込めないエラーが返却される + """ + + # Arrange + + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ], + 'is_update_last_fetch_datetime': True + } + + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1999-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2100-12-31T23:59:59.000Z", + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \ + patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer: + mock_writer_inst = mock_writer.return_value + mock_writer_inst.write.side_effect = Exception('ファイル書き込みエラー') + mock_config_bucket_inst = mock_config_bucket.return_value + mock_config_bucket_inst.put_last_fetch_datetime_file.return_value = '' + with pytest.raises(FileUploadException) as e: + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # Assert + assert mock_writer_inst.write.called is True + assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is False + assert e.value.error_id == 'E-UPD-01' + assert e.value.func_name == UPD_JP_NAME + assert e.value.args[0] == f'[Account] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[Account.json] エラー内容:[ファイル書き込みエラー]'