feat: CSVバックアップ処理内のファイルアップロードの実装を修正
This commit is contained in:
shimoda.m@nds-tyo.co.jp
parent
fed4883680
commit
899ee3a459
@ -1,18 +1,22 @@
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from src.aws.s3 import BackupBucket
|
||||
from src.config.objects import TargetObject
|
||||
from src.error.exceptions import FileUploadException
|
||||
from src.system_var.constants import CSVBK_JP_NAME
|
||||
from src.system_var.constants import CSVBK_JP_NAME, TEMPORARY_FILENAME
|
||||
from src.util.execute_datetime import ExecuteDateTime
|
||||
from src.util.logger import logger_instance as logger
|
||||
from src.writer.file_writer import CsvWriter
|
||||
|
||||
|
||||
def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime, csv_string: str):
|
||||
def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime, csv_data: list):
|
||||
"""CSVバックアップ処理
|
||||
|
||||
Args:
|
||||
target_object (TargetObject): 取得対象オブジェクト情報インスタンス
|
||||
execute_datetime (ExecuteDateTime): 実行日時取得インスタンス
|
||||
csv_string (str): csvデータ
|
||||
csv_data (list): csvデータ
|
||||
|
||||
Raises:
|
||||
FileUploadException: S3のファイルアップロード失敗
|
||||
@ -21,27 +25,38 @@ def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: E
|
||||
# ① CSVバックアップ処理の開始ログを出力する
|
||||
target_object_name = target_object.object_name
|
||||
upload_file_name = target_object.upload_file_name
|
||||
upload_file_key = f'{execute_datetime.to_path()}/{upload_file_name}.csv'
|
||||
|
||||
logger.info(
|
||||
f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{upload_file_name}.csv]')
|
||||
|
||||
try:
|
||||
# ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する
|
||||
backup_bucket = BackupBucket()
|
||||
backup_bucket.put_csv(
|
||||
f'{execute_datetime.to_path()}/{upload_file_name}.csv', csv_string)
|
||||
|
||||
logger.debug(
|
||||
f'D-CSVBK-02 [{target_object_name}] のCSVデータバックアップ 正常終了')
|
||||
|
||||
_backup_csv_file(target_object_name, upload_file_key, csv_data)
|
||||
except Exception as e:
|
||||
raise FileUploadException(
|
||||
'E-CSVBK-01',
|
||||
CSVBK_JP_NAME, f'[{target_object_name}] CSVデータのバックアップに失敗しました ファイル名：[{upload_file_name}.csv] エラー内容:[{e}]')
|
||||
|
||||
# ③ CSVバックアップ処理の終了ログを出力する
|
||||
logger.info(
|
||||
f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します')
|
||||
|
||||
# ④ 次の処理へ移行する
|
||||
return
|
||||
|
||||
|
||||
def _backup_csv_file(target_object_name: str, upload_file_key: str, csv_data: list) -> None:
|
||||
|
||||
# 一時ファイル書き込み用の領域を確保
|
||||
with tempfile.TemporaryDirectory(prefix=f'{target_object_name}_') as tmpdir:
|
||||
# アップロード用のファイルをローカルに書き出す
|
||||
local_file_path = os.path.join(tmpdir, TEMPORARY_FILENAME)
|
||||
writer = CsvWriter(local_file_path, csv_data)
|
||||
writer.write()
|
||||
# ファイルからS3に書き込み
|
||||
backup_bucket = BackupBucket()
|
||||
backup_bucket.put_csv(upload_file_key, local_file_path)
|
||||
|
||||
logger.debug(
|
||||
f'D-CSVBK-02 [{upload_file_key}] のCSVデータバックアップ 正常終了')
|
||||
|
||||
return
|
||||
|
||||
@ -83,6 +83,7 @@ CSV_QUOTE_CHAR = '"'
|
||||
|
||||
FILE_CHAR_CODE = 'utf-8'
|
||||
FILE_MODE_WRITE = 'w'
|
||||
TEMPORARY_FILENAME = 'temporary_file'
|
||||
|
||||
|
||||
# CRM_取得オブジェクト情報ファイル関連
|
||||
|
||||
@ -34,12 +34,12 @@ class TestBackupCrmCsvDataProcess:
|
||||
- CSVバックアップ処理の仕様に沿った正常系ログが出力されること(デバッグログは除く)
|
||||
"""
|
||||
# Arrange
|
||||
csv_string = textwrap.dedent("""\
|
||||
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"
|
||||
"TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"
|
||||
"TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"
|
||||
"TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"
|
||||
""")
|
||||
csv_data = [
|
||||
["Id", "AccountNumber", "LastModifiedDate", "LastModifiedById", "SystemModstamp", "IsDeleted"],
|
||||
["TEST001", "test001", "2022-06-01 09:00:00", "1234567", "2022-06-01 09:00:00", 1],
|
||||
["TEST002", "test002", "2022-06-01 09:00:00", "1234567", "2022-06-01 09:00:00", 0],
|
||||
["TEST003", "test003", "2022-06-01 09:00:00", "1234567", "2022-06-01 09:00:00", 0]
|
||||
]
|
||||
target_object_dict = {
|
||||
'object_name': 'Account',
|
||||
'columns': [
|
||||
@ -60,16 +60,23 @@ class TestBackupCrmCsvDataProcess:
|
||||
monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import')
|
||||
|
||||
# Act
|
||||
backup_crm_csv_data_process(target_object, execute_datetime, csv_string)
|
||||
backup_crm_csv_data_process(target_object, execute_datetime, csv_data)
|
||||
|
||||
# Assert
|
||||
|
||||
expect_csv_value = """\
|
||||
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\
|
||||
"TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\
|
||||
"TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
|
||||
"TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
|
||||
"""
|
||||
expect_file_key = f'data_import/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.csv'
|
||||
# ファイル確認
|
||||
actual = s3_client.get_object(
|
||||
Bucket=bucket_name,
|
||||
Key=f'data_import/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.csv')
|
||||
Key=expect_file_key)
|
||||
|
||||
assert actual['Body'].read().decode('utf-8') == csv_string
|
||||
assert actual['Body'].read().decode('utf-8') == textwrap.dedent(expect_csv_value)
|
||||
|
||||
# ログの確認
|
||||
assert generate_log_message_tuple(
|
||||
@ -104,15 +111,20 @@ class TestBackupCrmCsvDataProcess:
|
||||
execute_datetime = ExecuteDateTime()
|
||||
target_object = TargetObject(target_object_dict, execute_datetime)
|
||||
|
||||
with patch('src.backup_crm_csv_data_process.BackupBucket') as mock_backup_bucket:
|
||||
with patch('src.backup_crm_csv_data_process.BackupBucket') as mock_backup_bucket, \
|
||||
patch('src.backup_crm_csv_data_process.CsvWriter') as mock_writer:
|
||||
mock_backup_bucket_inst = mock_backup_bucket.return_value
|
||||
mock_backup_bucket_inst.put_csv.return_value = ''
|
||||
|
||||
mock_writer_inst = mock_writer.return_value
|
||||
mock_writer_inst.write.return_value = ''
|
||||
# Act
|
||||
backup_crm_csv_data_process(target_object, execute_datetime, '')
|
||||
|
||||
# Assert
|
||||
|
||||
assert mock_backup_bucket_inst.put_csv.called is True
|
||||
assert mock_writer_inst.write.called is True
|
||||
|
||||
def test_raise_put_csv(self, bucket_name, monkeypatch, caplog):
|
||||
"""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user