nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 結果ファイルアップロード処理を修正

Browse Source
This commit is contained in:
shimoda.m@nds-tyo.co.jp 2022-10-12 22:12:07 +09:00
parent 373f758e76
commit d16b38d8b6
2 changed files with 79 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
ecs/crm-datafetch/src/upload_result_data_process.py
View File

@ -1,9 +1,13 @@
import os
import tempfile
from src.aws.s3 import BackupBucket
from src.error.exceptions import FileUploadException
from src.system_var.constants import END_JP_NAME
from src.system_var.constants import END_JP_NAME, TEMPORARY_FILENAME
from src.system_var.environments import PROCESS_RESULT_FILENAME
from src.util.execute_datetime import ExecuteDateTime
from src.util.logger import logger_instance as logger
from src.writer.file_writer import JsonWriter
def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime):
@ -23,12 +27,7 @@ def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDa
try:
# ② CRMバックアップ保管用バケットに、取得処理実施結果のJSONデータを保管する
backup_bucket = BackupBucket()
backup_bucket.put_result_json(
f'{execute_datetime.to_path()}/{PROCESS_RESULT_FILENAME}', process_result)
logger.debug(f'D-END-02 取得処理実施結果アップロード 正常終了')
_upload_result(execute_datetime, process_result)
except Exception as e:
raise FileUploadException(
'E-END-01', END_JP_NAME, f'取得処理実施結果のアップロードに失敗しました　ファイル名：[{PROCESS_RESULT_FILENAME}] エラー内容：[{e}]')
@ -37,3 +36,18 @@ def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDa
logger.info(f'I-END-03 取得処理実施結果アップロード処理を終了します')
return
def _upload_result(execute_datetime: ExecuteDateTime, process_result: dict) -> None:
# 一時ファイル書き込み用の領域を確保
with tempfile.TemporaryDirectory(prefix=f'{PROCESS_RESULT_FILENAME}_') as tmpdir:
# アップロード用のファイルをローカルに書き出す
local_file_path = os.path.join(tmpdir, TEMPORARY_FILENAME)
writer = JsonWriter(local_file_path, process_result)
writer.write()
# ファイルからS3に書き込み
backup_bucket = BackupBucket()
backup_bucket.put_result_json(
f'{execute_datetime.to_path()}/{PROCESS_RESULT_FILENAME}', local_file_path)
logger.debug(f'D-END-02 取得処理実施結果アップロード 正常終了')

65
ecs/crm-datafetch/tests/test_upload_result_data_process.py
View File

@ -1,4 +1,4 @@
from unittest.mock import MagicMock, patch
from unittest.mock import patch
import pytest
from src.error.exceptions import FileUploadException
@ -72,7 +72,6 @@ class TestUploadResultDataProcess:
"""
# Arrange
mock_backup_bucket = MagicMock(return_value=None)
process_result = {
"Account": "success",
@ -82,11 +81,18 @@ class TestUploadResultDataProcess:
execute_datetime = ExecuteDateTime()
# Act
with patch('src.aws.s3.BackupBucket.put_result_json', mock_backup_bucket):
with patch('src.upload_result_data_process.BackupBucket') as mock_backup_bucket, \
patch('src.upload_result_data_process.JsonWriter') as mock_writer:
mock_writer_inst = mock_writer.return_value
mock_writer_inst.write.return_value = ''
mock_backup_bucket_inst = mock_backup_bucket.return_value
mock_backup_bucket_inst.put_result_json.return_value = ''
upload_result_data_process(process_result, execute_datetime)
# Assert
assert mock_backup_bucket.called is True
assert mock_writer_inst.write.called is True
assert mock_backup_bucket_inst.put_result_json.called is True
def test_raise_put_result_json(self, monkeypatch):
"""
@ -102,7 +108,6 @@ class TestUploadResultDataProcess:
"""
# Arrange
mock_backup_bucket = MagicMock(side_effect=Exception('ファイルアップロードエラー'))
process_result = {
"Account": "success",
@ -112,13 +117,59 @@ class TestUploadResultDataProcess:
execute_datetime = ExecuteDateTime()
# Act
with patch('src.aws.s3.BackupBucket.put_result_json', mock_backup_bucket):
with patch('src.upload_result_data_process.BackupBucket') as mock_backup_bucket, \
patch('src.upload_result_data_process.JsonWriter') as mock_writer:
mock_writer_inst = mock_writer.return_value
mock_writer_inst.write.return_value = ''
mock_backup_bucket_inst = mock_backup_bucket.return_value
mock_backup_bucket_inst.put_result_json.side_effect = Exception('ファイルアップロードエラー')
with pytest.raises(FileUploadException) as e:
upload_result_data_process(process_result, execute_datetime)
# Assert
assert mock_writer_inst.write.called is True
assert mock_backup_bucket.called is True
assert e.value.error_id == 'E-END-01'
assert e.value.func_name == END_JP_NAME
assert e.value.args[0] == f'取得処理実施結果のアップロードに失敗しました　ファイル名：[process_result.json] エラー内容：[ファイルアップロードエラー]'
def test_raise_put_result_json_write_local_file(self, monkeypatch):
"""
Cases:
取得処理実施結果をアップロードできない場合、エラーが発生すること
Arranges:
- 取得処理実施結果アップロード処理で例外を発生させるモックを生成する
- 取得処理実施結果を準備する
- 実行日時取得インスタンスを生成する
Expects:
- 例外が発生する
- ファイルがアップロードできないエラーが返却される
"""
# Arrange
process_result = {
"Account": "success",
"Contact": "fail"
}
execute_datetime = ExecuteDateTime()
# Act
with patch('src.upload_result_data_process.BackupBucket') as mock_backup_bucket, \
patch('src.upload_result_data_process.JsonWriter') as mock_writer:
mock_writer_inst = mock_writer.return_value
mock_writer_inst.write.side_effect = Exception('ファイル書き込みエラー')
mock_backup_bucket_inst = mock_backup_bucket.return_value
mock_backup_bucket_inst.put_result_json.return_value = ''
with pytest.raises(FileUploadException) as e:
upload_result_data_process(process_result, execute_datetime)
# Assert
assert mock_writer_inst.write.called is True
assert mock_backup_bucket.called is False
assert e.value.error_id == 'E-END-01'
assert e.value.func_name == END_JP_NAME
assert e.value.args[0] == f'取得処理実施結果のアップロードに失敗しました　ファイル名：[process_result.json] エラー内容：[ファイル書き込みエラー]'