feat:ファイルのチェックと全件用取得ファイルのテスト追加

ecs/crm-datafetch/tests/test_walk_through.py

@@ -19,6 +19,7 @@ OBJECT_INFO_FOLDER = 'crm/object_info'
 LAST_FETCH_DATETIME_INFO_FOLDER = 'crm/last_fetch_datetime'
 BACKUP_DATA_IMPORT_FOLDER = 'data_import'
 BACKUP_RESPONSE_JSON_FOLDER = 'response_json'
+PROCESS_RESULT_JSON_FILE_NAME = 'process_result.json'
 
 
 @pytest.fixture
@@ -56,6 +57,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
     # 実行日時を固定する
     now = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ)
     format_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y%m%d%H%M%S')
+    path_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y/%m/%d/%H%M%S')
 
     class MockExecuteDateTime(ExecuteDateTime):
         def __init__(self):
@@ -89,24 +91,64 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
         assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages
         assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages
         assert_backup_response_process_log(log_messages, target_object_name)
+        assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json')
         assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages
         assert_convert_process_log(log_messages, target_object_name)
         assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages
         assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name)
+        assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv')
         assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages
         assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name)
+        assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv')
         assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages
         assert_upload_fetch_datetime_process_log(log_messages, target_object_name, False)
+        assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json')
         assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages
 
     # ループ終了後のログの確認
     process_result_json = {obj['object_name']: 'success' for obj in object_info_list['objects']}
     assert f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result_json}]' in log_messages
     assert f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始' in log_messages
+    assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{PROCESS_RESULT_JSON_FILE_NAME}')
     assert f'I-CTRL-19 すべてのデータの取得に成功しました' in log_messages
     assert f'I-CTRL-20 CRMデータ取得処理を終了します' in log_messages
 
-    assert 0
+    # 全件要取得処理を実行する
+
+    # 全件取得用に環境変数を設定
+    monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_all.json')
+    controller()
+    # ログ再取得
+    log_messages_all = caplog.messages
+    object_info_list_all = object_info_files[1]
+    # 開始ログなどはテスト済みなのでチェックを省く
+    for object_info in object_info_list_all['objects']:
+        target_object_name = object_info['object_name']
+        upload_file_name = f'CRM_{target_object_name}_ALL_{format_now}'
+        datetime_filename = f'{target_object_name}_ALL'
+        assert 'I-CTRL-05 オブジェクト情報形式チェック処理呼び出し' in log_messages_all
+        assert_check_process_log(log_messages_all)
+        assert f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します' in log_messages_all
+        assert f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し' in log_messages_all
+        assert_period_process_log(log_messages_all, target_object_name, datetime_filename, now)
+        assert f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し' in log_messages_all
+        assert_fetch_process_log(log_messages_all, target_object_name)
+        assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages_all
+        assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages_all
+        assert_backup_response_process_log(log_messages_all, target_object_name)
+        assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json')
+        assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages_all
+        assert_convert_process_log(log_messages_all, target_object_name)
+        assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages_all
+        assert_backup_csv_process_log(log_messages_all, target_object_name, upload_file_name)
+        assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv')
+        assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages_all
+        assert_upload_csv_process_log(log_messages_all, target_object_name, upload_file_name)
+        assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv')
+        assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages_all
+        assert_upload_fetch_datetime_process_log(log_messages_all, target_object_name, True)
+        assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json')
+        assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages_all
 
 
 """
@@ -172,6 +214,13 @@ def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, i
         assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages
 
 
+def assert_file_exist(s3_client, bucket_name, file_key):
+    try:
+        assert s3_client.head_object(Bucket=bucket_name, Key=file_key) is not None
+    except Exception:
+        raise Exception(f'ファイルが存在しません バケット名：{bucket_name}, ファイルパス：{file_key}')
+
+
 """
 以下、取得準備関数
 """
@@ -220,7 +269,7 @@ def set_environment(monkeypatch):
     monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER)
     monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER)
     monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET)
-    monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result')
+    monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', PROCESS_RESULT_JSON_FILE_NAME)
     monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER)
     monkeypatch.setattr('src.set_datetime_period_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET)
     monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER)