nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 最終更新日ファイルアップロード処理修正

Browse Source
This commit is contained in:
shimoda.m@nds-tyo.co.jp 2022-10-12 22:01:14 +09:00
parent 6f601b1340
commit 373f758e76
2 changed files with 104 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py
View File

@ -1,10 +1,12 @@
import json
import os
import tempfile
from src.aws.s3 import ConfigBucket
from src.config.objects import LastFetchDatetime, TargetObject
from src.error.exceptions import FileUploadException
from src.system_var.constants import UPD_JP_NAME
from src.system_var.constants import TEMPORARY_FILENAME, UPD_JP_NAME
from src.util.logger import logger_instance as logger
from src.writer.file_writer import JsonWriter
def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime):
@ -18,15 +20,16 @@ def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_d
FileUploadException: S3のファイルアップロード失敗
"""
object_name = target_object.object_name
# ① 前回取得日時ファイル更新処理の開始ログを出力する
logger.info(
f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します')
f'I-UPD-01 [{object_name}] の前回取得日時ファイルの更新処理を開始します')
try:
if target_object.is_update_last_fetch_datetime is False:
# ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする
logger.info(
f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします')
f'I-UPD-02 [{object_name}] の前回取得日時ファイルの更新処理をスキップします')
return
# ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する
@ -35,23 +38,32 @@ def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_d
'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to,
'last_fetch_datetime_to': ''
}
config_bucket = ConfigBucket()
config_bucket.put_last_fetch_datetime_file(
target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict))
logger.info(
f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了')
_upload_last_fetch_datetime(target_object, last_fetch_datetime_dict)
except Exception as e:
raise FileUploadException(
'E-UPD-01',
UPD_JP_NAME,
f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]')
f'[{object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]')
# ④ 前回取得日時ファイル更新処理の終了ログを出力する
logger.info(
f'I-UPD-04 [{target_object.object_name}] の前回取得日時ファイルの更新処理を終了します')
f'I-UPD-04 [{object_name}] の前回取得日時ファイルの更新処理を終了します')
# ⑤ 次の処理へ移行する
return
def _upload_last_fetch_datetime(target_object: TargetObject, last_fetch_datetime_dict: dict) -> None:
# 一時ファイル書き込み用の領域を確保
with tempfile.TemporaryDirectory(prefix=f'{target_object.object_name}_') as tmpdir:
# アップロード用のファイルをローカルに書き出す
local_file_path = os.path.join(tmpdir, TEMPORARY_FILENAME)
writer = JsonWriter(local_file_path, last_fetch_datetime_dict)
writer.write()
# ファイルからS3に書き込み
config_bucket = ConfigBucket()
config_bucket.put_last_fetch_datetime_file(
target_object.last_fetch_datetime_file_name, local_file_path)
logger.info(
f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了')

85
ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py
View File

@ -118,11 +118,19 @@ class TestUploadLastFetchDatetimeProcess:
last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime)
# Act
with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket):
with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \
patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer:
mock_writer_inst = mock_writer.return_value
mock_writer_inst.write.return_value = ''
mock_config_bucket_inst = mock_config_bucket.return_value
mock_config_bucket_inst.put_last_fetch_datetime_file.return_value = ''
upload_last_fetch_datetime_process(target_object, last_fetch_datetime)
# Assert
# 処理実行確認
assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is False
assert mock_config_bucket.called is False
# ログの確認
@ -143,7 +151,6 @@ class TestUploadLastFetchDatetimeProcess:
"""
# Arrange
mock_config_bucket = MagicMock(return_value=None)
target_objects_dict = {
'object_name': 'Account',
@ -165,11 +172,19 @@ class TestUploadLastFetchDatetimeProcess:
last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime)
# Act
with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket):
with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \
patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer:
mock_writer_inst = mock_writer.return_value
mock_writer_inst.write.return_value = ''
mock_config_bucket_inst = mock_config_bucket.return_value
mock_config_bucket_inst.put_last_fetch_datetime_file.return_value = ''
upload_last_fetch_datetime_process(target_object, last_fetch_datetime)
# Assert
assert mock_config_bucket.called is True
assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is True
assert mock_writer_inst.write.called is True
def test_raise_put_last_fetch_datetime_file(self, monkeypatch):
"""
@ -185,7 +200,6 @@ class TestUploadLastFetchDatetimeProcess:
"""
# Arrange
mock_config_bucket = MagicMock(side_effect=Exception('ファイルアップロードエラー'))
target_objects_dict = {
'object_name': 'Account',
@ -207,12 +221,69 @@ class TestUploadLastFetchDatetimeProcess:
last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime)
# Act
with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket):
with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \
patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer:
mock_writer_inst = mock_writer.return_value
mock_writer_inst.write.return_value = ''
mock_config_bucket_inst = mock_config_bucket.return_value
mock_config_bucket_inst.put_last_fetch_datetime_file.side_effect = Exception('ファイルアップロードエラー')
with pytest.raises(FileUploadException) as e:
upload_last_fetch_datetime_process(target_object, last_fetch_datetime)
# Assert
assert mock_config_bucket.called is True
assert mock_writer_inst.write.called is True
assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is True
assert e.value.error_id == 'E-UPD-01'
assert e.value.func_name == UPD_JP_NAME
assert e.value.args[0] == f'[Account] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[Account.json] エラー内容:[ファイルアップロードエラー]'
def test_raise_put_last_fetch_datetime_file_write_local_file(self, monkeypatch):
"""
Cases:
前回取得日時ファイルをアップロードするためのローカルファイルの書き込みに失敗した場合、エラーが発生すること
Arranges:
- 前回取得日時ファイル更新処理で例外を発生させるモックを生成する
- 取得処理実施結果を準備する
- 実行日時取得インスタンスを生成する
Expects:
- 例外が発生する
- ファイルが書き込めないエラーが返却される
"""
# Arrange
target_objects_dict = {
'object_name': 'Account',
'columns': [
'Id',
'Name'
],
'is_update_last_fetch_datetime': True
}
last_fetch_datetime_dict = {
"last_fetch_datetime_from": "1999-01-01T00:00:00.000Z",
"last_fetch_datetime_to": "2100-12-31T23:59:59.000Z",
}
execute_datetime = ExecuteDateTime()
target_object = TargetObject(target_objects_dict, execute_datetime)
last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime)
# Act
with patch('src.upload_last_fetch_datetime_process.ConfigBucket') as mock_config_bucket, \
patch('src.upload_last_fetch_datetime_process.JsonWriter') as mock_writer:
mock_writer_inst = mock_writer.return_value
mock_writer_inst.write.side_effect = Exception('ファイル書き込みエラー')
mock_config_bucket_inst = mock_config_bucket.return_value
mock_config_bucket_inst.put_last_fetch_datetime_file.return_value = ''
with pytest.raises(FileUploadException) as e:
upload_last_fetch_datetime_process(target_object, last_fetch_datetime)
# Assert
assert mock_writer_inst.write.called is True
assert mock_config_bucket_inst.put_last_fetch_datetime_file.called is False
assert e.value.error_id == 'E-UPD-01'
assert e.value.func_name == UPD_JP_NAME
assert e.value.args[0] == f'[Account] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[Account.json] エラー内容:[ファイル書き込みエラー]'