diff --git a/ecs/crm-datafetch/.dockerignore b/ecs/crm-datafetch/.dockerignore new file mode 100644 index 00000000..01840c61 --- /dev/null +++ b/ecs/crm-datafetch/.dockerignore @@ -0,0 +1,11 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index d4822731..873f9075 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -13,7 +13,10 @@ "python.linting.flake8Args": ["--max-line-length=150", "--ignore=F541"], "python.formatting.provider": "autopep8", "python.formatting.autopep8Args": ["--max-line-length", "150"], - "python.testing.pytestArgs": ["tests"], + "python.testing.pytestArgs": [ + "tests" + // "--walk-through" // 一気通貫テストを行いたい場合はコメントを外す + ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 82796598..da57afc8 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -6,7 +6,8 @@ name = "pypi" [scripts] test = "pytest tests/" "test:cov" = "pytest --cov=src --cov-branch --cov-report=term-missing tests/" -"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/test_result.html tests/" +"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/unit_test/test_result.html tests/" +"test:walk-through" = "pytest tests/test_walk_through.py --walk-through --cov-report=term-missing --html=.report/walk_through/test_result.html" [packages] boto3 = "*" diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index f2928706..5da3ae51 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -146,6 +146,19 @@ pipenv install --dev pipenv run test:cov ``` +#### 一気通貫テストを実行する場合の設定 + +- `.env`の以下に示す環境変数の値をメルク様提供のFullSandboxのものに書き換える + - CRM_AUTH_DOMAIN + - CRM_USER_NAME + - CRM_USER_PASSWORD + - CRM_USER_SECURITY_TOKEN +- 以下のコマンドを実行して単体テストを起動する + +```sh +pipenv run test:walk-through +``` + #### 拡張機能：Python Test Explorer UIを導入する場合 - VSCodeの拡張機能メニューから、「Python Test Explorer for Visual Studio Code」をインストール @@ -162,7 +175,11 @@ pipenv run test:cov - 標準出力とカバレッジファイル(`.coverage`)に出力される - `pipenv run test:report` - pytestのテスト終了時にテスト結果をHTMLで出力する - - `.report/test_result.html`が出力される + - `.report/unit_test/test_result.html`が出力される +- `test:walk-through` + - 一気通貫テストが実行される + - 上記のテスト以外はスキップされる + - `.report/walk_through/test_result.html`に結果のHTMLが出力される ## 単体テストの追加方法 diff --git a/ecs/crm-datafetch/pytest.ini b/ecs/crm-datafetch/pytest.ini new file mode 100644 index 00000000..5dbe2661 --- /dev/null +++ b/ecs/crm-datafetch/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(levelname)s %(asctime)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py index 42d4ecea..120b5bf2 100644 --- a/ecs/crm-datafetch/src/salesforce/soql_builder.py +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -1,6 +1,7 @@ import textwrap from src.config.objects import LastFetchDatetime, TargetObject +from src.system_var.environments import FETCH_LIMIT_CLAUSE class SOQLBuilder: @@ -9,6 +10,7 @@ class SOQLBuilder: SELECT {column_or_expression} FROM {object_name} WHERE {datetime_column} > {last_fetch_datetime_from} AND {datetime_column} <= {last_fetch_datetime_to} + {limit_clause} """) self.__target_object = target_object self.__last_fetch_datetime = last_fetch_datetime @@ -19,7 +21,8 @@ class SOQLBuilder: object_name=self.__target_object.object_name, last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, - datetime_column=self.__target_object.datetime_column + datetime_column=self.__target_object.datetime_column, + limit_clause='' ) return count_soql @@ -31,7 +34,8 @@ class SOQLBuilder: object_name=self.__target_object.object_name, last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, - datetime_column=self.__target_object.datetime_column + datetime_column=self.__target_object.datetime_column, + limit_clause=FETCH_LIMIT_CLAUSE ) return fetch_soql diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index b290cc7b..297cfb2b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -7,14 +7,6 @@ import src.system_var.constants as constants LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) -# CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) -# CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) -# CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) -# CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) # CRMのレコード件数取得処理の最大リトライ試行回数 @@ -65,9 +57,9 @@ PROCESS_RESULT_FILENAME = os.environ.get(constants.PROCESS_RESULT_FILENAME, 'pro LAST_FETCH_DATE_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_FOLDER, 'crm/last_fetch_datetime') # CRMから取得し、取込用に変換したデータを格納するフォルダ CRM_IMPORT_DATA_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_FOLDER, 'crm/target') -# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') # CRMから取得した生データのバックアップを格納するフォルダパス RESPONSE_JSON_BACKUP_FOLDER = os.environ.get(constants.RESPONSE_JSON_BACKUP_FOLDER, 'response_json') # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_BACKUP_FOLDER, 'data_import') +# 一気通貫テスト用、取得件数を絞るためのSOQL句 +FETCH_LIMIT_CLAUSE = os.environ.get('FETCH_LIMIT_CLAUSE', '') diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index 55ade88c..b86b29d9 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -57,3 +57,23 @@ def pytest_runtest_makereport(item, call): report.cases = docstring.get("Cases", '') # 「テスト内容」を`report`に追加 report.arranges = docstring.get("Arranges", '') # 「準備作業」を`report`に追加 report.expects = docstring.get("Expects", '') # 「期待結果」を`report`に追加 + + +def pytest_addoption(parser): + parser.addoption( + "--walk-through", action="store_true", default=False, help="run walk through tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "slow: mark test as slow to run") + + +def pytest_collection_modifyitems(config, items): + skip_slow = pytest.mark.skip(reason="need --walk-through option to run") + if config.getoption("--walk-through"): + [item.add_marker(skip_slow) for item in items if "walk_through" not in item.keywords] + return + for item in items: + if "walk_through" in item.keywords: + item.add_marker(skip_slow) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py new file mode 100644 index 00000000..e8c3b6e4 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -0,0 +1,326 @@ +import json +import logging +import os +import os.path as path +from datetime import datetime, timezone + +import boto3 +import pytest +from src.controller import controller +from src.parser.json_parser import JsonParser +from src.system_var.constants import YYYYMMDDTHHMMSSTZ +from src.util.execute_datetime import ExecuteDateTime + +ROOT_DIR = path.abspath(path.dirname(__name__)) + + +# ↓↓↓モックテスト用(バケットをモック化する際、コメントアウトを外す。AWS利用テスト用はコメントアウトする)↓↓↓ +# DATA_BUCKET = 'mbj-newdwh2021-staging-data' +# CONFIG_BUCKET = 'mbj-newdwh2021-staging-config' +# BACKUP_BUCKET = 'mbj-newdwh2021-staging-backup-crm' +# +# @pytest.fixture +# def s3_test(s3_client): +# s3_client.create_bucket(Bucket=DATA_BUCKET) +# s3_client.create_bucket(Bucket=CONFIG_BUCKET) +# s3_client.create_bucket(Bucket=BACKUP_BUCKET) +# yield +# ↑↑↑モックテスト用↑↑↑ + +# ↓↓↓AWS利用テスト用(AWS上のバケットを利用する際、コメントアウトを外す。モックテスト用はコメントアウトする)↓↓↓ +DATA_BUCKET = 'test-mbj-newdwh2021-test-data' +CONFIG_BUCKET = 'test-mbj-newdwh2021-test-config' +BACKUP_BUCKET = 'test-mbj-newdwh2021-test-backup-crm' + + +@pytest.fixture +def s3_test(s3_client): + yield + + +@pytest.fixture +def s3_client(): + s3_client = boto3.client("s3") + yield s3_client +# ↑↑↑AWS利用テスト用↑↑↑ + + +TARGET_FOLDER = 'crm/target' +OBJECT_INFO_FOLDER = 'crm/object_info' +LAST_FETCH_DATETIME_INFO_FOLDER = 'crm/last_fetch_datetime' +BACKUP_DATA_IMPORT_FOLDER = 'data_import' +BACKUP_RESPONSE_JSON_FOLDER = 'response_json' +PROCESS_RESULT_JSON_FILE_NAME = 'process_result.json' + + +@pytest.mark.walk_through +def test_walk_through(s3_test, s3_client, monkeypatch, caplog): + """ + Cases: + コントロール処理の頭から最後まで処理が流れきることを確認する + Arranges: + - 実行に必要なモックバケットを用意する + - 実行に必要なファイルをアップロードする + - 実行に必要な環境変数をmonkeypatchで設定する + - CRMデータカウント取得処理をモック化する（ログ出力にのみ使用しているため） + Expects: + - すべてのログが仕様どおり二出力されていること（デバッグログを除く） + - すべてのファイルが使用どおりの場所にアップロードされていること + """ + # Arrange + # バケットにファイルをアップロードしていく + object_info_files = [] + object_info_list = get_object_config_list('object_info') + for object_info in object_info_list: + json_file = read_json(object_info) + json_file = to_upload_json(json_file) + upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') + object_info_files.append(json_file) + last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') + for last_fetch_datetime in last_fetch_datetime_list: + json_file = read_json(last_fetch_datetime) + upload_json(json_file, s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{path.basename(last_fetch_datetime)}') + + # 環境変数を設定(CRMの認証情報は別途設定しておくこと) + set_environment(monkeypatch) + # 差分取得用に環境変数を設定 + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + # 一気通貫テスト用に件数の制限をかける + monkeypatch.setattr('src.salesforce.soql_builder.FETCH_LIMIT_CLAUSE', ' LIMIT 10') + # 件数取得はログ出力用なので、0件が返るようにする + monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 10) + # ログレベルをDEBUGに + monkeypatch.setattr('src.util.logger.LOG_LEVEL', 'DEBUG') + # 実行日時を固定する + now = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) + format_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y%m%d%H%M%S') + path_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y/%m/%d/%H%M%S') + + class MockExecuteDateTime(ExecuteDateTime): + def __init__(self): + super().__init__() + self._ExecuteDateTime__execute_datetime = now + + monkeypatch.setattr('src.prepare_data_fetch_process.ExecuteDateTime', MockExecuteDateTime) + + # Act + logger = logging.getLogger() + logger.setLevel(logging.INFO) + logger.info(f'##########################') + logger.info(f'# 差分データ取得処理:実行開始 #') + logger.info(f'##########################') + controller() + logger.info(f'##########################') + logger.info(f'# 差分データ取得処理:実行終了 #') + logger.info(f'##########################') + # Assertion + log_messages = caplog.messages + # ループ前のログ確認 + assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages + assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages + assert_prepare_process_log(log_messages, now) + assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages + # オブジェクト情報を取得する(diff) + object_info_list = object_info_files[0] + + for object_info in object_info_list['objects']: + target_object_name = object_info['object_name'] + upload_file_name = f'CRM_{target_object_name}_{format_now}' + assert 'I-CTRL-05 オブジェクト情報形式チェック処理呼び出し' in log_messages + assert_check_process_log(log_messages) + assert f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します' in log_messages + assert f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し' in log_messages + assert_period_process_log(log_messages, target_object_name, target_object_name, now) + assert f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し' in log_messages + assert_fetch_process_log(log_messages, target_object_name) + assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages + assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages + assert_backup_response_process_log(log_messages, target_object_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json') + assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages + assert_convert_process_log(log_messages, target_object_name) + assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages + assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv') + assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages + assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name) + assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv') + assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages + assert_upload_fetch_datetime_process_log(log_messages, target_object_name, False) + assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json') + assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages + + # ループ終了後のログの確認 + process_result_json = {obj['object_name']: 'success' for obj in object_info_list['objects']} + assert f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result_json}]' in log_messages + assert f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始' in log_messages + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{PROCESS_RESULT_JSON_FILE_NAME}') + assert f'I-CTRL-19 すべてのデータの取得に成功しました' in log_messages + assert f'I-CTRL-20 CRMデータ取得処理を終了します' in log_messages + + # 全件要取得処理を実行する + + # 全件取得用に環境変数を設定 + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_all.json') + logger.info(f'##########################') + logger.info(f'# 全件データ取得処理:実行開始 #') + logger.info(f'##########################') + controller() + logger.info(f'##########################') + logger.info(f'# 全件データ取得処理:実行終了 #') + logger.info(f'##########################') + # ログ再取得 + log_messages_all = caplog.messages + object_info_list_all = object_info_files[1] + # 開始ログなどはテスト済みなのでチェックを省く + for object_info in object_info_list_all['objects']: + target_object_name = object_info['object_name'] + upload_file_name = f'CRM_{target_object_name}_ALL_{format_now}' + datetime_filename = f'{target_object_name}_ALL' + assert 'I-CTRL-05 オブジェクト情報形式チェック処理呼び出し' in log_messages_all + assert_check_process_log(log_messages_all) + assert f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します' in log_messages_all + assert f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し' in log_messages_all + assert_period_process_log(log_messages_all, target_object_name, datetime_filename, now) + assert f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し' in log_messages_all + assert_fetch_process_log(log_messages_all, target_object_name) + assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages_all + assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages_all + assert_backup_response_process_log(log_messages_all, target_object_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json') + assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages_all + assert_convert_process_log(log_messages_all, target_object_name) + assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages_all + assert_backup_csv_process_log(log_messages_all, target_object_name, upload_file_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv') + assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages_all + assert_upload_csv_process_log(log_messages_all, target_object_name, upload_file_name) + assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv') + assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages_all + assert_upload_fetch_datetime_process_log(log_messages_all, target_object_name, True) + assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json') + assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages_all + + +""" +以下、アサーション関数 +""" + + +def assert_prepare_process_log(log_messages, now): + assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages + assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages + assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages + + +def assert_check_process_log(log_messages): + assert 'I-CHK-01 オブジェクト情報形式チェック処理を開始します' in log_messages + assert 'I-CHK-02 オブジェクト情報形式チェック処理を終了します' in log_messages + + +def assert_period_process_log(log_messages, target_object_name, datetime_file_name, now): + assert f'I-DATE-01 [{target_object_name}] のデータ取得期間設定処理を開始します' in log_messages + assert f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{LAST_FETCH_DATETIME_INFO_FOLDER}/{datetime_file_name}.json]' in log_messages + assert 'I-DATE-03 前回取得日時ファイルの取得成功しました' in log_messages + assert f'I-DATE-06 取得範囲 From: [1900-01-01T00:00:00.000Z] To: [{now}]' in log_messages + assert f'I-DATE-07 [{target_object_name}] のデータ取得期間設定処理を終了します' in log_messages + + +def assert_fetch_process_log(log_messages, target_object_name): + assert f'I-FETCH-01 [{target_object_name}] のCRMからのデータ取得処理を開始します' in log_messages + assert f'I-FETCH-02 [{target_object_name}] の件数取得を開始します' in log_messages + assert f'I-FETCH-03 [{target_object_name}] の件数：[10]' in log_messages + assert f'I-FETCH-04 [{target_object_name}] のレコード取得を開始します' in log_messages + assert f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました' in log_messages + assert f'I-FETCH-06 [{target_object_name}] のCRMからのデータ取得処理を終了します' in log_messages + + +def assert_backup_response_process_log(log_messages, target_object_name): + assert f'I-RESBK-01 [{target_object_name}] のCRM電文データバックアップ処理を開始します' in log_messages + assert f'I-RESBK-03 [{target_object_name}] のCRM電文データバックアップ処理を終了します' in log_messages + + +def assert_convert_process_log(log_messages, target_object_name): + assert f'I-CONV-01 [{target_object_name}] のCSV変換処理を開始します' in log_messages + assert f'I-CONV-03 [{target_object_name}] のCSV変換処理を終了します' in log_messages + + +def assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name): + assert f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{upload_file_name}.csv]' in log_messages + assert f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します' in log_messages + + +def assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name): + assert f'I-UPLD-01 [{target_object_name}] のCSVデータアップロード処理を開始します ファイル名：[{upload_file_name}.csv]' in log_messages + assert f'I-UPLD-03 [{target_object_name}] のCSVデータのアップロード処理を終了します' in log_messages + + +def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, is_all=False): + assert f'I-UPD-01 [{target_object_name}] の前回取得日時ファイルの更新処理を開始します' in log_messages + if is_all: + assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' in log_messages + return + assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages + + +def assert_file_exist(s3_client, bucket_name, file_key): + try: + assert s3_client.head_object(Bucket=bucket_name, Key=file_key) is not None + except Exception: + raise Exception(f'ファイルが存在しません バケット名：{bucket_name}, ファイルパス：{file_key}') + + +""" +以下、取得準備関数 +""" + + +def get_object_config_list(folder_name: str): + local_s3_path = path.join(ROOT_DIR, '..', '..', 's3', 'config', 'crm', folder_name) + config_list = [os.path.join(local_s3_path, config) for config in os.listdir(local_s3_path) if config.endswith('.json')] + return sorted(config_list, reverse=True) + + +def read_json(json_path): + with open(json_path, 'r', encoding='utf8') as f: + json_str = f.read() + JsonParser + json_file = JsonParser(json_str).parse() + + return json_file + + +def upload_json(json_file, s3_client, bucket, folder): + json_str = json.dumps(json_file) + s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) + + +def to_upload_json(json_file): + """Userオブジェクトの取得できないプロパティを取り除く + TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す + """ + for object_info in json_file['objects']: + if object_info['object_name'] != 'User': + continue + columns: list = object_info['columns'] + columns.remove('LastPasswordChangeDate') + columns.remove('NumberOfFailedLogins') + columns.remove('UserPreferencesNativeEmailClient') + + return json_file + + +def set_environment(monkeypatch): + # 環境変数を設定(CRMの認証情報は別途設定しておくこと) + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', TARGET_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) + monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', PROCESS_RESULT_JSON_FILE_NAME) + monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.set_datetime_period_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 1998012c..580bf564 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -1,9 +1,5 @@ LOG_LEVEL=INFO CRM_AUTH_TIMEOUT=100 -CRM_AUTH_MAX_RETRY_ATTEMPT=3 -CRM_AUTH_RETRY_INTERVAL=5 -CRM_AUTH_RETRY_MIN_INTERVAL=5 -CRM_AUTH_RETRY_MAX_INTERVAL=50 CRM_GET_RECORD_COUNT_TIMEOUT=300 CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5